def start_runtime(log_level="off", tracing=0, interactive=False): # type: (str, int, bool) -> None """ Starts the COMPSs runtime. Starts the runtime by calling the external python library that calls the bindings-common. :param log_level: Log level [ "trace" | "debug" | "info" | "api" | "off" ]. :param tracing: Tracing level [0 (deactivated) | 1 (basic) | 2 (advanced)]. :param interactive: Boolean if interactive (ipython or jupyter). :return: None """ if __debug__: logger.info("Starting COMPSs...") if tracing > 0 and not interactive: # Enabled only if not interactive - extrae issues within jupyter. enable_trace_master() with event_master(START_RUNTIME_EVENT): if interactive and context.in_master(): COMPSs.load_runtime(external_process=True) else: COMPSs.load_runtime(external_process=False) if log_level == "trace": # Could also be "debug" or True, but we only show the C extension # debug in the maximum tracing level. COMPSs.set_debug(True) OT.enable_report() COMPSs.start_runtime() if __debug__: logger.info("COMPSs started")
def delete_object(obj): # type: (object) -> bool """ Remove object. Removes a used object from the internal structures and calls the external python library (that calls the bindings-common) in order to request a its corresponding file removal. :param obj: Object to remove. :return: True if success. False otherwise. """ with event_master(DELETE_OBJECT_EVENT): app_id = 0 obj_id = OT.is_tracked(obj) if obj_id is None: # Not being tracked return False else: try: file_name = OT.get_file_name(obj_id) COMPSs.delete_file(app_id, file_name, False) OT.stop_tracking(obj) except KeyError: pass return True
def testDeleteObject1(self): obj_1 = [0] obj_2 = increment_object(obj_1) obj_2 = compss_wait_on(obj_2) obj_1_id = OT.get_object_id(obj_1) deletion_result = compss_delete_object(obj_1) self.assertTrue(deletion_result) self.assertFalse(obj_1_id in OT.pending_to_synchronize) self.assertTrue(OT.get_object_id(obj_1) is "")
def testInDeleteObject4(): obj_1 = MyClass2() obj_2 = modify_obj(obj_1) obj_2 = compss_wait_on(obj_2) obj_1_id = OT.get_object_id(obj_1) if not (obj_1_id in OT.pending_to_synchronize) and OT.get_object_id(obj_1) == "": print("- Test 4 IN_DELETE object: OK") else: print("- Test 4 IN_DELETE object: ERROR")
def testInDeleteObject(): obj_1 = [0] obj_2 = increment_object_delete(obj_1) obj_2 = compss_wait_on(obj_2) obj_1_id = OT.get_object_id(obj_1) if not (obj_1_id in OT.pending_to_synchronize) and OT.get_object_id(obj_1) == "": print("- Test 1 IN_DELETE object: OK") else: print("- Test 1 IN_DELETE object: ERROR")
def stop_runtime(code=0, hard_stop=False): # type: (int, bool) -> None """ Stops the COMPSs runtime. Stops the runtime by calling the external python library that calls the bindings-common. Also cleans objects and temporary files created during runtime. If the code is different from 0, all running or waiting tasks will be cancelled. :parameter code: Stop code (if code != 0 ==> cancel application tasks). :param hard_stop: Stop compss when runtime has died. :return: None """ with event_master(STOP_RUNTIME_EVENT): app_id = 0 if __debug__: logger.info("Stopping runtime...") # Stopping a possible wall clock limit signal.alarm(0) if code != 0: if __debug__: logger.info("Canceling all application tasks...") COMPSs.cancel_application_tasks(app_id, 0) if __debug__: logger.info("Cleaning objects...") _clean_objects(hard_stop=hard_stop) if __debug__: reporting = OT.is_report_enabled() if reporting: logger.info("Generating Object tracker report...") target_path = get_log_path() OT.generate_report(target_path) OT.clean_report() if __debug__: logger.info("Stopping COMPSs...") COMPSs.stop_runtime(code) if __debug__: logger.info("Cleaning temps...") _clean_temps() context.set_pycompss_context(context.OUT_OF_SCOPE) if __debug__: logger.info("COMPSs stopped")
def testInDeleteObject2(): obj_1 = [0] count = 0 for i in range(10): obj_1[0] = i - 1 obj_2 = increment_object_delete(obj_1) obj_2 = compss_wait_on(obj_2) obj_1_id = OT.get_object_id(obj_1) if not (obj_1_id in OT.pending_to_synchronize) and OT.get_object_id( obj_1) == "" and i == obj_2[0]: count += 1 if count == 10: print("- Test 2 IN_DELETE object: OK") else: print("- Test 2 IN_DELETE object: ERROR")
def _clean_objects(hard_stop=False): # type: (bool) -> None """ Clean all objects. Clean the objects stored in the global dictionaries: - pending_to_synchronize dict. - _addr2id2obj dict. - obj_id_to_filename dict. - _objs_written_by_mp dict. :param hard_stop: avoid call to delete_file when the runtime has died. :return: None """ app_id = 0 if not hard_stop: for filename in OT.get_all_file_names(): COMPSs.delete_file(app_id, filename, False) OT.clean_object_tracker()
def __hard_stop__(debug, sync, logger, ipython): # type: (bool, bool, typing.Any, typing.Any) -> None """ The runtime has been stopped by any error and this method stops the remaining things in the binding. :param debug: If debugging. :param sync: Scope variables synchronization [ True | False ]. :param logger: Logger where to put the logging messages. :param ipython: Ipython instance. :return: None """ print("The runtime is not running.") # Check that everything is stopped as well: # Stop streaming if STREAMING: stop_streaming() # Stop persistent storage if PERSISTENT_STORAGE: master_stop_storage(logger) # Clean any left object in the object tracker OT.clean_object_tracker() # Cleanup events and files release_event_manager(ipython) __clean_temp_files__() # Stop watching stdout and stderr STDW.stop_watching(clean=not debug) # Retrieve the remaining messages that could have been captured. last_messages = STDW.get_messages() if last_messages: for message in last_messages: print(message) if sync: print("* Can not synchronize any future object.") return None
def _wait_on_iterable(iter_obj, compss_mode): # type: (typing.Any, int) -> typing.Any """ Wait on an iterable object (Recursive). Currently supports lists and dictionaries (syncs the values). :param iter_obj: Iterable object. :return: Synchronized object. """ # check if the object is in our pending_to_synchronize dictionary obj_id = OT.is_tracked(iter_obj) if OT.is_pending_to_synchronize(obj_id): return _synchronize(iter_obj, compss_mode) else: if type(iter_obj) == list: return [_wait_on_iterable(x, compss_mode) for x in iter_obj] elif type(iter_obj) == dict: return { _wait_on_iterable(k, compss_mode): _wait_on_iterable(v, compss_mode) for k, v in iter_obj.items() } else: return _synchronize(iter_obj, compss_mode)
def stop(sync=False, _hard_stop=False): # type: (bool, bool) -> None """ Runtime stop. :param sync: Scope variables synchronization [ True | False ] (default: False) :param _hard_stop: Stop compss when runtime has died [ True | False ]. (default: False) :return: None """ logger = logging.getLogger(__name__) ipython = globals()["__builtins__"]["get_ipython"]() if not context.in_pycompss(): return __hard_stop__(interactive_helpers.DEBUG, sync, logger, ipython) from pycompss.api.api import compss_stop print(LINE_SEPARATOR) print("*************** STOPPING PyCOMPSs ******************") print(LINE_SEPARATOR) # Wait 5 seconds to give some time to process the remaining messages # of the STDW and check if there is some error that could have stopped # the runtime before continuing. print("Checking if any issue happened.") time.sleep(5) messages = STDW.get_messages() if messages: for message in messages: sys.stderr.write("".join((message, "\n"))) # Uncomment the following lines to see the ipython dictionary # in a structured way: # import pprint # pprint.pprint(ipython.__dict__, width=1) if sync and not _hard_stop: sync_msg = "Synchronizing all future objects left on the user scope." print(sync_msg) logger.debug(sync_msg) from pycompss.api.api import compss_wait_on reserved_names = ("quit", "exit", "get_ipython", "APP_PATH", "ipycompss", "In", "Out") raw_code = ipython.__dict__["user_ns"] for k in raw_code: obj_k = raw_code[k] if not k.startswith('_'): # not internal objects if type(obj_k) == Future: print("Found a future object: %s" % str(k)) logger.debug("Found a future object: %s" % str(k)) new_obj_k = compss_wait_on(obj_k) if new_obj_k == obj_k: print("\t - Could not retrieve object: %s" % str(k)) logger.debug("\t - Could not retrieve object: %s" % str(k)) else: ipython.__dict__["user_ns"][k] = new_obj_k elif k not in reserved_names: try: if OT.is_pending_to_synchronize(obj_k): print("Found an object to synchronize: %s" % str(k)) # noqa: E501 logger.debug("Found an object to synchronize: %s" % (k, )) # noqa: E501 ipython.__dict__["user_ns"][k] = compss_wait_on( obj_k) # noqa: E501 except TypeError: # Unhashable type: List - could be a collection if isinstance(obj_k, list): print("Found a list to synchronize: %s" % str(k)) logger.debug("Found a list to synchronize: %s" % (k, )) # noqa: E501 ipython.__dict__["user_ns"][k] = compss_wait_on( obj_k) # noqa: E501 else: print("Warning: some of the variables used with PyCOMPSs may") print(" have not been brought to the master.") # Stop streaming if STREAMING: stop_streaming() # Stop persistent storage if PERSISTENT_STORAGE: master_stop_storage(logger) # Emit the 0 for the APPLICATION_RUNNING_EVENT emitted on start function. emit_manual_event(0) # Stop runtime compss_stop(_hard_stop=_hard_stop) # Cleanup events and files release_event_manager(ipython) __clean_temp_files__() # Stop watching stdout and stderr STDW.stop_watching(clean=True) # Retrieve the remaining messages that could have been captured. last_messages = STDW.get_messages() if last_messages: for message in last_messages: print(message) # Let the Python binding know we are not at master anymore context.set_pycompss_context(context.OUT_OF_SCOPE) print(LINE_SEPARATOR) logger.debug("--- END ---")
def _synchronize(obj, mode): # type: (typing.Any, int) -> typing.Any """ Synchronization function. This method retrieves the value of a future object. Calls the runtime in order to wait for the value and returns it when received. :param obj: Object to synchronize. :param mode: Direction of the object to synchronize. :return: The value of the object requested. """ # TODO: Add a boolean to differentiate between files and object on the # COMPSs.open_file call. This change pretends to obtain better traces. # Must be implemented first in the Runtime, then in the bindings common # C API and finally add the boolean here app_id = 0 obj_id = "" # noqa if is_psco(obj): obj_id = str(get_id(obj)) if not OT.is_pending_to_synchronize(obj_id): return obj else: # file_path is of the form storage://pscoId or # file://sys_path_to_file file_path = COMPSs.open_file(app_id, "".join( ("storage://", str(obj_id))), mode) # TODO: Add switch on protocol (first parameter returned currently ignored) _, file_name = file_path.split("://") new_obj = get_by_id(file_name) OT.stop_tracking(obj) return new_obj obj_id = OT.is_tracked(obj) obj_name = OT.get_obj_name(obj_id) if obj_id is None: # Not being tracked return obj if not OT.is_pending_to_synchronize(obj_id): return obj if __debug__: logger.debug("Synchronizing object %s with mode %s" % (obj_id, mode)) file_name = OT.get_file_name(obj_id) compss_file = COMPSs.open_file(app_id, file_name, mode) # Runtime can return a path or a PSCOId if compss_file.startswith('/'): # If the real filename is null, then return None. The task that # produces the output file may have been ignored or cancelled, so its # result does not exist. real_file_name = compss_file.split('/')[-1] if real_file_name == "null": print( "WARNING: Could not retrieve the object " + str(file_name) + " since the task that produces it may have been IGNORED or CANCELLED. Please, check the logs. Returning None." ) # noqa: E501 return None new_obj = deserialize_from_file(compss_file) COMPSs.close_file(app_id, file_name, mode) else: new_obj = get_by_id(compss_file) if context.is_nesting_enabled() and context.in_worker(): # If nesting and in worker, the user wants to synchronize an object. # So it is necessary to update the parameter with the new object. update_worker_argument_parameter_content(obj_name, new_obj) if mode == 'r': OT.update_mapping(obj_id, new_obj) if mode != 'r': COMPSs.delete_file(app_id, OT.get_file_name(obj_id), False) OT.stop_tracking(obj) return new_obj
def sync_if_needed(obj): # type: (typing.Any) -> None if OT.is_obj_pending_to_synchronize(obj): new_val = compss_wait_on(obj) replace(obj, new_val)