def start_runtime(log_level="off", tracing=0, interactive=False):
# type: (str, int, bool) -> None
""" Starts the COMPSs runtime.
Starts the runtime by calling the external python library that calls
the bindings-common.
:param log_level: Log level [ "trace" | "debug" | "info" | "api" | "off" ].
:param tracing: Tracing level [0 (deactivated) | 1 (basic) | 2 (advanced)].
:param interactive: Boolean if interactive (ipython or jupyter).
:return: None
"""
if __debug__:
logger.info("Starting COMPSs...")
if tracing > 0 and not interactive:
# Enabled only if not interactive - extrae issues within jupyter.
enable_trace_master()
with event_master(START_RUNTIME_EVENT):
if interactive and context.in_master():
COMPSs.load_runtime(external_process=True)
else:
COMPSs.load_runtime(external_process=False)
if log_level == "trace":
# Could also be "debug" or True, but we only show the C extension
# debug in the maximum tracing level.
COMPSs.set_debug(True)
OT.enable_report()
COMPSs.start_runtime()
if __debug__:
logger.info("COMPSs started")
def delete_object(obj):
# type: (object) -> bool
""" Remove object.
Removes a used object from the internal structures and calls the
external python library (that calls the bindings-common)
in order to request a its corresponding file removal.
:param obj: Object to remove.
:return: True if success. False otherwise.
"""
with event_master(DELETE_OBJECT_EVENT):
app_id = 0
obj_id = OT.is_tracked(obj)
if obj_id is None:
# Not being tracked
return False
else:
try:
file_name = OT.get_file_name(obj_id)
COMPSs.delete_file(app_id, file_name, False)
OT.stop_tracking(obj)
except KeyError:
pass
return True
def testDeleteObject1(self):
obj_1 = [0]
obj_2 = increment_object(obj_1)
obj_2 = compss_wait_on(obj_2)
obj_1_id = OT.get_object_id(obj_1)
deletion_result = compss_delete_object(obj_1)
self.assertTrue(deletion_result)
self.assertFalse(obj_1_id in OT.pending_to_synchronize)
self.assertTrue(OT.get_object_id(obj_1) is "")
def testInDeleteObject4():
obj_1 = MyClass2()
obj_2 = modify_obj(obj_1)
obj_2 = compss_wait_on(obj_2)
obj_1_id = OT.get_object_id(obj_1)
if not (obj_1_id
in OT.pending_to_synchronize) and OT.get_object_id(obj_1) == "":
print("- Test 4 IN_DELETE object: OK")
else:
print("- Test 4 IN_DELETE object: ERROR")
def testInDeleteObject():
obj_1 = [0]
obj_2 = increment_object_delete(obj_1)
obj_2 = compss_wait_on(obj_2)
obj_1_id = OT.get_object_id(obj_1)
if not (obj_1_id
in OT.pending_to_synchronize) and OT.get_object_id(obj_1) == "":
print("- Test 1 IN_DELETE object: OK")
else:
print("- Test 1 IN_DELETE object: ERROR")
def stop_runtime(code=0, hard_stop=False):
# type: (int, bool) -> None
""" Stops the COMPSs runtime.
Stops the runtime by calling the external python library that calls
the bindings-common.
Also cleans objects and temporary files created during runtime.
If the code is different from 0, all running or waiting tasks will be
cancelled.
:parameter code: Stop code (if code != 0 ==> cancel application tasks).
:param hard_stop: Stop compss when runtime has died.
:return: None
"""
with event_master(STOP_RUNTIME_EVENT):
app_id = 0
if __debug__:
logger.info("Stopping runtime...")
# Stopping a possible wall clock limit
signal.alarm(0)
if code != 0:
if __debug__:
logger.info("Canceling all application tasks...")
COMPSs.cancel_application_tasks(app_id, 0)
if __debug__:
logger.info("Cleaning objects...")
_clean_objects(hard_stop=hard_stop)
if __debug__:
reporting = OT.is_report_enabled()
if reporting:
logger.info("Generating Object tracker report...")
target_path = get_log_path()
OT.generate_report(target_path)
OT.clean_report()
if __debug__:
logger.info("Stopping COMPSs...")
COMPSs.stop_runtime(code)
if __debug__:
logger.info("Cleaning temps...")
_clean_temps()
context.set_pycompss_context(context.OUT_OF_SCOPE)
if __debug__:
logger.info("COMPSs stopped")
def testInDeleteObject2():
obj_1 = [0]
count = 0
for i in range(10):
obj_1[0] = i - 1
obj_2 = increment_object_delete(obj_1)
obj_2 = compss_wait_on(obj_2)
obj_1_id = OT.get_object_id(obj_1)
if not (obj_1_id in OT.pending_to_synchronize) and OT.get_object_id(
obj_1) == "" and i == obj_2[0]:
count += 1
if count == 10:
print("- Test 2 IN_DELETE object: OK")
else:
print("- Test 2 IN_DELETE object: ERROR")
def _clean_objects(hard_stop=False):
# type: (bool) -> None
""" Clean all objects.
Clean the objects stored in the global dictionaries:
- pending_to_synchronize dict.
- _addr2id2obj dict.
- obj_id_to_filename dict.
- _objs_written_by_mp dict.
:param hard_stop: avoid call to delete_file when the runtime has died.
:return: None
"""
app_id = 0
if not hard_stop:
for filename in OT.get_all_file_names():
COMPSs.delete_file(app_id, filename, False)
OT.clean_object_tracker()
def __hard_stop__(debug, sync, logger, ipython):
# type: (bool, bool, typing.Any, typing.Any) -> None
""" The runtime has been stopped by any error and this method stops the
remaining things in the binding.
:param debug: If debugging.
:param sync: Scope variables synchronization [ True | False ].
:param logger: Logger where to put the logging messages.
:param ipython: Ipython instance.
:return: None
"""
print("The runtime is not running.")
# Check that everything is stopped as well:
# Stop streaming
if STREAMING:
stop_streaming()
# Stop persistent storage
if PERSISTENT_STORAGE:
master_stop_storage(logger)
# Clean any left object in the object tracker
OT.clean_object_tracker()
# Cleanup events and files
release_event_manager(ipython)
__clean_temp_files__()
# Stop watching stdout and stderr
STDW.stop_watching(clean=not debug)
# Retrieve the remaining messages that could have been captured.
last_messages = STDW.get_messages()
if last_messages:
for message in last_messages:
print(message)
if sync:
print("* Can not synchronize any future object.")
return None
def _wait_on_iterable(iter_obj, compss_mode):
# type: (typing.Any, int) -> typing.Any
""" Wait on an iterable object (Recursive).
Currently supports lists and dictionaries (syncs the values).
:param iter_obj: Iterable object.
:return: Synchronized object.
"""
# check if the object is in our pending_to_synchronize dictionary
obj_id = OT.is_tracked(iter_obj)
if OT.is_pending_to_synchronize(obj_id):
return _synchronize(iter_obj, compss_mode)
else:
if type(iter_obj) == list:
return [_wait_on_iterable(x, compss_mode) for x in iter_obj]
elif type(iter_obj) == dict:
return {
_wait_on_iterable(k, compss_mode):
_wait_on_iterable(v, compss_mode)
for k, v in iter_obj.items()
}
else:
return _synchronize(iter_obj, compss_mode)
def stop(sync=False, _hard_stop=False):
# type: (bool, bool) -> None
""" Runtime stop.
:param sync: Scope variables synchronization [ True | False ]
(default: False)
:param _hard_stop: Stop compss when runtime has died [ True | False ].
(default: False)
:return: None
"""
logger = logging.getLogger(__name__)
ipython = globals()["__builtins__"]["get_ipython"]()
if not context.in_pycompss():
return __hard_stop__(interactive_helpers.DEBUG, sync, logger, ipython)
from pycompss.api.api import compss_stop
print(LINE_SEPARATOR)
print("*************** STOPPING PyCOMPSs ******************")
print(LINE_SEPARATOR)
# Wait 5 seconds to give some time to process the remaining messages
# of the STDW and check if there is some error that could have stopped
# the runtime before continuing.
print("Checking if any issue happened.")
time.sleep(5)
messages = STDW.get_messages()
if messages:
for message in messages:
sys.stderr.write("".join((message, "\n")))
# Uncomment the following lines to see the ipython dictionary
# in a structured way:
# import pprint
# pprint.pprint(ipython.__dict__, width=1)
if sync and not _hard_stop:
sync_msg = "Synchronizing all future objects left on the user scope."
print(sync_msg)
logger.debug(sync_msg)
from pycompss.api.api import compss_wait_on
reserved_names = ("quit", "exit", "get_ipython", "APP_PATH",
"ipycompss", "In", "Out")
raw_code = ipython.__dict__["user_ns"]
for k in raw_code:
obj_k = raw_code[k]
if not k.startswith('_'): # not internal objects
if type(obj_k) == Future:
print("Found a future object: %s" % str(k))
logger.debug("Found a future object: %s" % str(k))
new_obj_k = compss_wait_on(obj_k)
if new_obj_k == obj_k:
print("\t - Could not retrieve object: %s" % str(k))
logger.debug("\t - Could not retrieve object: %s" %
str(k))
else:
ipython.__dict__["user_ns"][k] = new_obj_k
elif k not in reserved_names:
try:
if OT.is_pending_to_synchronize(obj_k):
print("Found an object to synchronize: %s" %
str(k)) # noqa: E501
logger.debug("Found an object to synchronize: %s" %
(k, )) # noqa: E501
ipython.__dict__["user_ns"][k] = compss_wait_on(
obj_k) # noqa: E501
except TypeError:
# Unhashable type: List - could be a collection
if isinstance(obj_k, list):
print("Found a list to synchronize: %s" % str(k))
logger.debug("Found a list to synchronize: %s" %
(k, )) # noqa: E501
ipython.__dict__["user_ns"][k] = compss_wait_on(
obj_k) # noqa: E501
else:
print("Warning: some of the variables used with PyCOMPSs may")
print(" have not been brought to the master.")
# Stop streaming
if STREAMING:
stop_streaming()
# Stop persistent storage
if PERSISTENT_STORAGE:
master_stop_storage(logger)
# Emit the 0 for the APPLICATION_RUNNING_EVENT emitted on start function.
emit_manual_event(0)
# Stop runtime
compss_stop(_hard_stop=_hard_stop)
# Cleanup events and files
release_event_manager(ipython)
__clean_temp_files__()
# Stop watching stdout and stderr
STDW.stop_watching(clean=True)
# Retrieve the remaining messages that could have been captured.
last_messages = STDW.get_messages()
if last_messages:
for message in last_messages:
print(message)
# Let the Python binding know we are not at master anymore
context.set_pycompss_context(context.OUT_OF_SCOPE)
print(LINE_SEPARATOR)
logger.debug("--- END ---")
def _synchronize(obj, mode):
# type: (typing.Any, int) -> typing.Any
""" Synchronization function.
This method retrieves the value of a future object.
Calls the runtime in order to wait for the value and returns it when
received.
:param obj: Object to synchronize.
:param mode: Direction of the object to synchronize.
:return: The value of the object requested.
"""
# TODO: Add a boolean to differentiate between files and object on the
# COMPSs.open_file call. This change pretends to obtain better traces.
# Must be implemented first in the Runtime, then in the bindings common
# C API and finally add the boolean here
app_id = 0
obj_id = "" # noqa
if is_psco(obj):
obj_id = str(get_id(obj))
if not OT.is_pending_to_synchronize(obj_id):
return obj
else:
# file_path is of the form storage://pscoId or
# file://sys_path_to_file
file_path = COMPSs.open_file(app_id, "".join(
("storage://", str(obj_id))), mode)
# TODO: Add switch on protocol (first parameter returned currently ignored)
_, file_name = file_path.split("://")
new_obj = get_by_id(file_name)
OT.stop_tracking(obj)
return new_obj
obj_id = OT.is_tracked(obj)
obj_name = OT.get_obj_name(obj_id)
if obj_id is None: # Not being tracked
return obj
if not OT.is_pending_to_synchronize(obj_id):
return obj
if __debug__:
logger.debug("Synchronizing object %s with mode %s" % (obj_id, mode))
file_name = OT.get_file_name(obj_id)
compss_file = COMPSs.open_file(app_id, file_name, mode)
# Runtime can return a path or a PSCOId
if compss_file.startswith('/'):
# If the real filename is null, then return None. The task that
# produces the output file may have been ignored or cancelled, so its
# result does not exist.
real_file_name = compss_file.split('/')[-1]
if real_file_name == "null":
print(
"WARNING: Could not retrieve the object " + str(file_name) +
" since the task that produces it may have been IGNORED or CANCELLED. Please, check the logs. Returning None."
) # noqa: E501
return None
new_obj = deserialize_from_file(compss_file)
COMPSs.close_file(app_id, file_name, mode)
else:
new_obj = get_by_id(compss_file)
if context.is_nesting_enabled() and context.in_worker():
# If nesting and in worker, the user wants to synchronize an object.
# So it is necessary to update the parameter with the new object.
update_worker_argument_parameter_content(obj_name, new_obj)
if mode == 'r':
OT.update_mapping(obj_id, new_obj)
if mode != 'r':
COMPSs.delete_file(app_id, OT.get_file_name(obj_id), False)
OT.stop_tracking(obj)
return new_obj
def sync_if_needed(obj):
# type: (typing.Any) -> None
if OT.is_obj_pending_to_synchronize(obj):
new_val = compss_wait_on(obj)
replace(obj, new_val)