def synchronize(obj, mode):
'''
Synchronization function.
This method retrieves the value of a future object.
Calls the runtime in order to wait for the value and returns it
when received.
:param obj: Object to synchronize.
:param mode: Direction of the object to synchronize.
:return: The value of the object requested.
'''
# TODO - CUANDO SE LLAME A compss.get_file, anadir un booleano diferenciando si es fichero u objeto
# Objetivo: mejorar el detalle de las trazas. Esto se tiene que implementar
# primero en el runtime, despues adaptar el api de C, y finalmente anadir
# el booleano aqui.
global current_id
if is_PSCO(obj):
obj_id = get_ID(obj)
if obj_id not in pending_to_synchronize:
return obj
else:
# file_path is of the form storage://pscoID or file://sys_path_to_file
file_path = compss.get_file('storage://' + str(obj_id), mode)
# TODO: Add switch on protocol
protocol, file_name = file_path.split('://')
new_obj = get_by_ID(file_name)
return new_obj
obj_id = get_object_id(obj)
if obj_id not in pending_to_synchronize:
return obj
if __debug__:
logger.debug('Synchronizing object %s with mode %s' % (obj_id, mode))
file_name = objid_to_filename[obj_id]
compss_file = compss.get_file(file_name, mode)
# Runtime can return a path or a PSCOId
if compss_file.startswith('/'):
new_obj = deserialize_from_file(compss_file)
compss.close_file(file_name, mode)
else:
new_obj = get_by_ID(compss_file)
new_obj_id = get_object_id(new_obj, True, True)
# The main program won't work with the old object anymore, update mapping
objid_to_filename[new_obj_id] = objid_to_filename[obj_id].replace(obj_id, new_obj_id)
objs_written_by_mp[new_obj_id] = objid_to_filename[new_obj_id]
if __debug__:
logger.debug('Deleting obj %s (new one is %s)' % (str(obj_id), str(new_obj_id)))
compss.delete_file(objid_to_filename[obj_id])
objid_to_filename.pop(obj_id)
pending_to_synchronize.pop(obj_id)
if __debug__:
logger.debug('Now object with id %s and %s has mapping %s' % (new_obj_id, type(new_obj), file_name))
return new_obj
def delete_object(obj):
"""
Removes a used object from the internal structures and calls the
external python library (that calls the bindings-common)
in order to request a its corresponding file removal.
:param obj: Object to remove
:return: True if success. False otherwise
"""
obj_id = get_object_id(obj, False, False)
if obj_id is None:
return False
try:
_id2obj.pop(obj_id)
except KeyError:
pass
try:
file_name = objid_to_filename[obj_id]
compss.delete_file(file_name)
except KeyError:
pass
try:
objid_to_filename.pop(obj_id)
except KeyError:
pass
try:
pending_to_synchronize.pop(obj_id)
except KeyError:
pass
return True
def compss_delete_object(obj):
import compss
from pycompss.runtime.binding import get_object_id
from pycompss.runtime.binding import objid_to_filename
from pycompss.runtime.binding import pending_to_synchronize
from pycompss.runtime.binding import id2obj
obj_id = get_object_id(obj, False, False)
if obj_id is None:
return False
try:
id2obj.pop(obj_id)
except:
pass
try:
file_name = objid_to_filename[obj_id]
compss.delete_file(file_name)
except:
pass
try:
objid_to_filename.pop(obj_id)
except:
pass
try:
pending_to_synchronize.pop(obj_id)
except:
pass
return True
def delete_file(file_name):
'''
Calls the external python library (that calls the bindings-common)
in order to request a file removal.
:param file_name: File name to remove
:return: True if success. False on the contrary
'''
if __debug__:
logger.debug('Deleting file %s' % file_name)
compss.delete_file(file_name)
def clean_objects():
'''
Clean the objects stored in the global dictionaries:
* pending_to_synchronize dict
* id2obj dict
* objid_to_filename dict
* objs_written_by_mp dict
'''
for filename in objid_to_filename.values():
compss.delete_file(filename)
pending_to_synchronize.clear()
id2obj.clear()
objid_to_filename.clear()
objs_written_by_mp.clear()
def _clean_objects():
"""
Clean the objects stored in the global dictionaries:
* pending_to_synchronize dict
* _id2obj dict
* objid_to_filename dict
* _objs_written_by_mp dict
:return: None
"""
for filename in objid_to_filename.values():
compss.delete_file(filename)
pending_to_synchronize.clear()
_id2obj.clear()
objid_to_filename.clear()
_objs_written_by_mp.clear()
def delete_file(file_name):
"""
Calls the external python library (that calls the bindings-common)
in order to request a file removal.
:param file_name: File name to remove
:return: True if success. False otherwise
"""
if __debug__:
logger.debug("Deleting file %s" % file_name)
result = compss.delete_file(file_name) == 'true'
if __debug__:
if result:
logger.debug("File %s successfully deleted." % file_name)
else:
logger.error("Failed to remove file %s." % file_name)
return result
def c_extension_link(in_queue, out_queue, redirect_std, out_file_name,
err_file_name):
# type: (Queue, Queue, bool, str, str) -> None
""" Main C extension process.
:param in_queue: Queue to receive messages.
:param out_queue: Queue to send messages.
:param redirect_std: Decide whether to store the stdout and stderr into
files or not.
:param out_file_name: File where to store the stdout (only required if
redirect_std is True).
:param err_file_name: File where to store the stderr (only required if
redirect_std is True).
:return: None
"""
# Import C extension within the external process
import compss
with ipython_std_redirector(out_file_name, err_file_name) \
if redirect_std else not_std_redirector():
alive = True
while alive:
message = in_queue.get()
command = message[0]
parameters = [] # type: list
if len(message) > 0:
parameters = list(message[1:])
if command == START:
compss.start_runtime()
elif command == SET_DEBUG:
compss.set_debug(*parameters)
elif command == STOP:
compss.stop_runtime(*parameters)
alive = False
elif command == CANCEL_TASKS:
compss.cancel_application_tasks(*parameters)
elif command == ACCESSED_FILE:
accessed = compss.accessed_file(*parameters)
out_queue.put(accessed)
elif command == OPEN_FILE:
compss_name = compss.open_file(*parameters)
out_queue.put(compss_name)
elif command == CLOSE_FILE:
compss.close_file(*parameters)
elif command == DELETE_FILE:
result = compss.delete_file(*parameters)
out_queue.put(result)
elif command == GET_FILE:
compss.get_file(*parameters)
elif command == GET_DIRECTORY:
compss.get_directory(*parameters)
elif command == BARRIER:
compss.barrier(*parameters)
elif command == BARRIER_GROUP:
exception_message = compss.barrier_group(*parameters)
out_queue.put(exception_message)
elif command == OPEN_TASK_GROUP:
compss.open_task_group(*parameters)
elif command == CLOSE_TASK_GROUP:
compss.close_task_group(*parameters)
elif command == GET_LOGGING_PATH:
log_path = compss.get_logging_path()
out_queue.put(log_path)
elif command == GET_NUMBER_OF_RESOURCES:
num_resources = compss.get_number_of_resources(*parameters)
out_queue.put(num_resources)
elif command == REQUEST_RESOURCES:
compss.request_resources(*parameters)
elif command == FREE_RESOURCES:
compss.free_resources(*parameters)
elif command == REGISTER_CORE_ELEMENT:
compss.register_core_element(*parameters)
elif command == PROCESS_TASK:
compss.process_task(*parameters)
elif command == PROCESS_HTTP_TASK:
compss.process_http_task(*parameters)
elif command == SET_PIPES:
compss.set_pipes(*parameters)
elif command == READ_PIPES:
compss.read_pipes(*parameters)
elif command == SET_WALL_CLOCK:
compss.set_wall_clock(*parameters)
else:
raise PyCOMPSsException("Unknown link command")
def c_extension_link(in_queue, out_queue):
# type: (..., ...) -> None
""" Main C extension process.
:param in_queue: Queue to receive messages.
:param out_queue: Queue to send messages.
:return: None
"""
import compss
alive = True
while alive:
message = in_queue.get()
command = message[0]
parameters = []
if len(message) > 0:
parameters = message[1:]
if command == START:
compss.start_runtime()
elif command == SET_DEBUG:
compss.set_debug(*parameters)
elif command == STOP:
compss.stop_runtime(*parameters)
alive = False
elif command == CANCEL_TASKS:
compss.cancel_application_tasks(*parameters)
elif command == ACCESSED_FILE:
accessed = compss.accessed_file(*parameters)
out_queue.put(accessed)
elif command == OPEN_FILE:
compss_name = compss.open_file(*parameters)
out_queue.put(compss_name)
elif command == CLOSE_FILE:
compss.close_file(*parameters)
elif command == DELETE_FILE:
result = compss.delete_file(*parameters)
out_queue.put(result)
elif command == GET_FILE:
compss.get_file(*parameters)
elif command == GET_DIRECTORY:
compss.get_directory(*parameters)
elif command == BARRIER:
compss.barrier(*parameters)
elif command == BARRIER_GROUP:
exception_message = compss.barrier_group(*parameters)
out_queue.put(exception_message)
elif command == OPEN_TASK_GROUP:
compss.open_task_group(*parameters)
elif command == CLOSE_TASK_GROUP:
compss.close_task_group(*parameters)
elif command == GET_LOGGING_PATH:
log_path = compss.get_logging_path()
out_queue.put(log_path)
elif command == GET_NUMBER_OF_RESOURCES:
num_resources = compss.get_number_of_resources(*parameters)
out_queue.put(num_resources)
elif command == REQUEST_RESOURCES:
compss.request_resources(*parameters)
elif command == FREE_RESOURCES:
compss.free_resources(*parameters)
elif command == REGISTER_CORE_ELEMENT:
compss.register_core_element(*parameters)
elif command == PROCESS_TASK:
compss.process_task(*parameters)
elif command == SET_PIPES:
compss.set_pipes(*parameters)
else:
raise Exception("Unknown link command")
def synchronize(obj, mode):
"""
Synchronization function.
This method retrieves the value of a future object.
Calls the runtime in order to wait for the value and returns it when received.
:param obj: Object to synchronize.
:param mode: Direction of the object to synchronize.
:return: The value of the object requested.
"""
# TODO: Add a boolean to differentiate between files and object on the compss.get_file call. This change pretends
# to obtain better traces. Must be implemented first in the Runtime, then in the bindings common C API and
# finally add the boolean here
global _current_id
if is_psco(obj):
obj_id = get_id(obj)
if obj_id not in pending_to_synchronize:
return obj
else:
# file_path is of the form storage://pscoId or file://sys_path_to_file
file_path = compss.get_file("storage://" + str(obj_id), mode)
# TODO: Add switch on protocol
protocol, file_name = file_path.split('://')
new_obj = get_by_id(file_name)
return new_obj
obj_id = get_object_id(obj)
if obj_id not in pending_to_synchronize:
return obj
if __debug__:
logger.debug("Synchronizing object %s with mode %s" % (obj_id, mode))
file_name = objid_to_filename[obj_id]
compss_file = compss.get_file(file_name, mode)
# Runtime can return a path or a PSCOId
if compss_file.startswith('/'):
new_obj = deserialize_from_file(compss_file)
compss.close_file(file_name, mode)
else:
new_obj = get_by_id(compss_file)
new_obj_id = get_object_id(new_obj, True, True)
# The main program won't work with the old object anymore, update mapping
objid_to_filename[new_obj_id] = objid_to_filename[obj_id].replace(
obj_id, new_obj_id)
_objs_written_by_mp[new_obj_id] = objid_to_filename[new_obj_id]
if __debug__:
logger.debug("Deleting obj %s (new one is %s)" %
(str(obj_id), str(new_obj_id)))
compss.delete_file(objid_to_filename[obj_id])
objid_to_filename.pop(obj_id)
pending_to_synchronize.pop(obj_id)
if __debug__:
logger.debug("Now object with id %s and %s has mapping %s" %
(new_obj_id, type(new_obj), file_name))
return new_obj