def synchronize(obj, mode):
'''
Synchronization function.
This method retrieves the value of a future object.
Calls the runtime in order to wait for the value and returns it
when received.
:param obj: Object to synchronize.
:param mode: Direction of the object to synchronize.
:return: The value of the object requested.
'''
# TODO - CUANDO SE LLAME A compss.get_file, anadir un booleano diferenciando si es fichero u objeto
# Objetivo: mejorar el detalle de las trazas. Esto se tiene que implementar
# primero en el runtime, despues adaptar el api de C, y finalmente anadir
# el booleano aqui.
global current_id
if is_PSCO(obj):
obj_id = get_ID(obj)
if obj_id not in pending_to_synchronize:
return obj
else:
# file_path is of the form storage://pscoID or file://sys_path_to_file
file_path = compss.get_file('storage://' + str(obj_id), mode)
# TODO: Add switch on protocol
protocol, file_name = file_path.split('://')
new_obj = get_by_ID(file_name)
return new_obj
obj_id = get_object_id(obj)
if obj_id not in pending_to_synchronize:
return obj
if __debug__:
logger.debug('Synchronizing object %s with mode %s' % (obj_id, mode))
file_name = objid_to_filename[obj_id]
compss_file = compss.get_file(file_name, mode)
# Runtime can return a path or a PSCOId
if compss_file.startswith('/'):
new_obj = deserialize_from_file(compss_file)
compss.close_file(file_name, mode)
else:
new_obj = get_by_ID(compss_file)
new_obj_id = get_object_id(new_obj, True, True)
# The main program won't work with the old object anymore, update mapping
objid_to_filename[new_obj_id] = objid_to_filename[obj_id].replace(obj_id, new_obj_id)
objs_written_by_mp[new_obj_id] = objid_to_filename[new_obj_id]
if __debug__:
logger.debug('Deleting obj %s (new one is %s)' % (str(obj_id), str(new_obj_id)))
compss.delete_file(objid_to_filename[obj_id])
objid_to_filename.pop(obj_id)
pending_to_synchronize.pop(obj_id)
if __debug__:
logger.debug('Now object with id %s and %s has mapping %s' % (new_obj_id, type(new_obj), file_name))
return new_obj
def synchronize(obj, mode):
logger.debug("Synchronizing object %d with mode %s" % (id(obj), mode))
obj_id = id(obj)
if obj_id not in task_objects:
return obj
file_name = objid_to_filename[obj_id]
compss_file = compss.get_file(file_name, mode)
new_obj = deserialize_from_file(compss_file)
new_obj_id = id(new_obj)
# The main program won't work with the old object anymore, update mapping
objid_to_filename[new_obj_id] = file_name
task_objects[new_obj_id] = new_obj
# Do not let python free old objects until compss_stop, otherwise python could reuse object ids
#del objid_to_filename[obj_id]
#del task_objects[obj_id]
logger.debug("Now object with id %d and %s has mapping %s" % (new_obj_id, type(new_obj), file_name))
if mode != Direction.IN:
objs_written_by_mp[new_obj_id] = compss_file
return new_obj
def get_file(file_name, mode):
'''
Calls the external python library (that calls the bindings-common)
in order to request a file.
:return: The current name of the file requested (that may have been
renamed during runtime).
'''
if __debug__:
logger.debug('Getting file %s with mode %s' % (file_name, mode))
compss_name = compss.get_file(file_name, mode)
if __debug__:
logger.debug('COMPSs file name is %s' % compss_name)
return compss_name
def get_file(file_name, mode):
"""
Calls the external python library (that calls the bindings-common)
in order to request a file.
:param file_name: <String> File name.
:param mode: Compss mode.
:return: The current name of the file requested (that may have been renamed during runtime)
"""
if __debug__:
logger.debug("Getting file %s with mode %s" % (file_name, mode))
compss_name = compss.get_file(file_name, mode)
if __debug__:
logger.debug("COMPSs file name is %s" % compss_name)
return compss_name
def c_extension_link(in_queue, out_queue, redirect_std, out_file_name,
err_file_name):
# type: (Queue, Queue, bool, str, str) -> None
""" Main C extension process.
:param in_queue: Queue to receive messages.
:param out_queue: Queue to send messages.
:param redirect_std: Decide whether to store the stdout and stderr into
files or not.
:param out_file_name: File where to store the stdout (only required if
redirect_std is True).
:param err_file_name: File where to store the stderr (only required if
redirect_std is True).
:return: None
"""
# Import C extension within the external process
import compss
with ipython_std_redirector(out_file_name, err_file_name) \
if redirect_std else not_std_redirector():
alive = True
while alive:
message = in_queue.get()
command = message[0]
parameters = [] # type: list
if len(message) > 0:
parameters = list(message[1:])
if command == START:
compss.start_runtime()
elif command == SET_DEBUG:
compss.set_debug(*parameters)
elif command == STOP:
compss.stop_runtime(*parameters)
alive = False
elif command == CANCEL_TASKS:
compss.cancel_application_tasks(*parameters)
elif command == ACCESSED_FILE:
accessed = compss.accessed_file(*parameters)
out_queue.put(accessed)
elif command == OPEN_FILE:
compss_name = compss.open_file(*parameters)
out_queue.put(compss_name)
elif command == CLOSE_FILE:
compss.close_file(*parameters)
elif command == DELETE_FILE:
result = compss.delete_file(*parameters)
out_queue.put(result)
elif command == GET_FILE:
compss.get_file(*parameters)
elif command == GET_DIRECTORY:
compss.get_directory(*parameters)
elif command == BARRIER:
compss.barrier(*parameters)
elif command == BARRIER_GROUP:
exception_message = compss.barrier_group(*parameters)
out_queue.put(exception_message)
elif command == OPEN_TASK_GROUP:
compss.open_task_group(*parameters)
elif command == CLOSE_TASK_GROUP:
compss.close_task_group(*parameters)
elif command == GET_LOGGING_PATH:
log_path = compss.get_logging_path()
out_queue.put(log_path)
elif command == GET_NUMBER_OF_RESOURCES:
num_resources = compss.get_number_of_resources(*parameters)
out_queue.put(num_resources)
elif command == REQUEST_RESOURCES:
compss.request_resources(*parameters)
elif command == FREE_RESOURCES:
compss.free_resources(*parameters)
elif command == REGISTER_CORE_ELEMENT:
compss.register_core_element(*parameters)
elif command == PROCESS_TASK:
compss.process_task(*parameters)
elif command == PROCESS_HTTP_TASK:
compss.process_http_task(*parameters)
elif command == SET_PIPES:
compss.set_pipes(*parameters)
elif command == READ_PIPES:
compss.read_pipes(*parameters)
elif command == SET_WALL_CLOCK:
compss.set_wall_clock(*parameters)
else:
raise PyCOMPSsException("Unknown link command")
def c_extension_link(in_queue, out_queue):
# type: (..., ...) -> None
""" Main C extension process.
:param in_queue: Queue to receive messages.
:param out_queue: Queue to send messages.
:return: None
"""
import compss
alive = True
while alive:
message = in_queue.get()
command = message[0]
parameters = []
if len(message) > 0:
parameters = message[1:]
if command == START:
compss.start_runtime()
elif command == SET_DEBUG:
compss.set_debug(*parameters)
elif command == STOP:
compss.stop_runtime(*parameters)
alive = False
elif command == CANCEL_TASKS:
compss.cancel_application_tasks(*parameters)
elif command == ACCESSED_FILE:
accessed = compss.accessed_file(*parameters)
out_queue.put(accessed)
elif command == OPEN_FILE:
compss_name = compss.open_file(*parameters)
out_queue.put(compss_name)
elif command == CLOSE_FILE:
compss.close_file(*parameters)
elif command == DELETE_FILE:
result = compss.delete_file(*parameters)
out_queue.put(result)
elif command == GET_FILE:
compss.get_file(*parameters)
elif command == GET_DIRECTORY:
compss.get_directory(*parameters)
elif command == BARRIER:
compss.barrier(*parameters)
elif command == BARRIER_GROUP:
exception_message = compss.barrier_group(*parameters)
out_queue.put(exception_message)
elif command == OPEN_TASK_GROUP:
compss.open_task_group(*parameters)
elif command == CLOSE_TASK_GROUP:
compss.close_task_group(*parameters)
elif command == GET_LOGGING_PATH:
log_path = compss.get_logging_path()
out_queue.put(log_path)
elif command == GET_NUMBER_OF_RESOURCES:
num_resources = compss.get_number_of_resources(*parameters)
out_queue.put(num_resources)
elif command == REQUEST_RESOURCES:
compss.request_resources(*parameters)
elif command == FREE_RESOURCES:
compss.free_resources(*parameters)
elif command == REGISTER_CORE_ELEMENT:
compss.register_core_element(*parameters)
elif command == PROCESS_TASK:
compss.process_task(*parameters)
elif command == SET_PIPES:
compss.set_pipes(*parameters)
else:
raise Exception("Unknown link command")
def synchronize(obj, mode):
"""
Synchronization function.
This method retrieves the value of a future object.
Calls the runtime in order to wait for the value and returns it when received.
:param obj: Object to synchronize.
:param mode: Direction of the object to synchronize.
:return: The value of the object requested.
"""
# TODO: Add a boolean to differentiate between files and object on the compss.get_file call. This change pretends
# to obtain better traces. Must be implemented first in the Runtime, then in the bindings common C API and
# finally add the boolean here
global _current_id
if is_psco(obj):
obj_id = get_id(obj)
if obj_id not in pending_to_synchronize:
return obj
else:
# file_path is of the form storage://pscoId or file://sys_path_to_file
file_path = compss.get_file("storage://" + str(obj_id), mode)
# TODO: Add switch on protocol
protocol, file_name = file_path.split('://')
new_obj = get_by_id(file_name)
return new_obj
obj_id = get_object_id(obj)
if obj_id not in pending_to_synchronize:
return obj
if __debug__:
logger.debug("Synchronizing object %s with mode %s" % (obj_id, mode))
file_name = objid_to_filename[obj_id]
compss_file = compss.get_file(file_name, mode)
# Runtime can return a path or a PSCOId
if compss_file.startswith('/'):
new_obj = deserialize_from_file(compss_file)
compss.close_file(file_name, mode)
else:
new_obj = get_by_id(compss_file)
new_obj_id = get_object_id(new_obj, True, True)
# The main program won't work with the old object anymore, update mapping
objid_to_filename[new_obj_id] = objid_to_filename[obj_id].replace(
obj_id, new_obj_id)
_objs_written_by_mp[new_obj_id] = objid_to_filename[new_obj_id]
if __debug__:
logger.debug("Deleting obj %s (new one is %s)" %
(str(obj_id), str(new_obj_id)))
compss.delete_file(objid_to_filename[obj_id])
objid_to_filename.pop(obj_id)
pending_to_synchronize.pop(obj_id)
if __debug__:
logger.debug("Now object with id %s and %s has mapping %s" %
(new_obj_id, type(new_obj), file_name))
return new_obj
def get_file(file_name, mode):
logger.debug("Getting file %s with mode %s" % (file_name, mode))
compss_name = compss.get_file(file_name, mode)
logger.debug("COMPSs file name is %s" % compss_name)
return compss_name
