def test_enable_nesting(): from pycompss.util.context import enable_nesting from pycompss.util.context import disable_nesting from pycompss.util.context import is_nesting_enabled not_enabled = is_nesting_enabled() enable_nesting() is_enabled = is_nesting_enabled() disable_nesting() is_disabled = is_nesting_enabled() assert not_enabled is False, "ERROR: Nesting must not be enabled by default." # noqa: E501 assert is_enabled is True, "ERROR: Nesting has not been enabled." assert is_disabled is False, "ERROR: Nesting has not been disabled."
def constrained_f(*args, **kwargs): # type: (*typing.Any, **typing.Any) -> typing.Any if not self.scope: from pycompss.api.dummy.on_failure import on_failure \ as dummy_on_failure d_c = dummy_on_failure(self.args, self.kwargs) return d_c.__call__(user_function)(*args, **kwargs) if __debug__: logger.debug("Executing on_failure_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs) # Set the on failure management action and default variables in # kwargs for its usage in @task decorator kwargs["on_failure"] = self.on_failure_action kwargs["defaults"] = self.defaults with keep_arguments(args, kwargs, prepend_strings=True): # Call the method ret = user_function(*args, **kwargs) return ret
def multinode_f(*args, **kwargs): if not self.scope: raise NotInPyCOMPSsException(not_in_pycompss("MultiNode")) if __debug__: logger.debug("Executing multinode_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs, user_function) if context.in_worker(): old_slurm_env = set_slurm_environment() # Set the computing_nodes variable in kwargs for its usage # in @task decorator kwargs['computing_nodes'] = self.kwargs['computing_nodes'] with keep_arguments(args, kwargs, prepend_strings=True): # Call the method ret = user_function(*args, **kwargs) if context.in_worker(): reset_slurm_environment(old_slurm_env) return ret
def __decorator_body__(self, user_function, args, kwargs): # type: (typing.Callable, tuple, dict) -> typing.Any if not self.scope: # Execute the mpi as with PyCOMPSs so that sequential # execution performs as parallel. # To disable: raise Exception(not_in_pycompss("mpi")) # TODO: Intercept @task parameters to get stream redirection if "binary" in self.kwargs: return self.__run_mpi__(args, kwargs) else: print( "WARN: Python MPI as dummy is not fully supported. Executing decorated funtion." ) return user_function(*args, **kwargs) if __debug__: logger.debug("Executing mpi_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs) # The processes parameter will have to go down until the execution # is invoked. To this end, set the computing_nodes variable in kwargs # for its usage in @task decorator # WARNING: processes can be an int, a env string, a str with # dynamic variable name. if "processes" in self.kwargs: kwargs["computing_nodes"] = self.kwargs["processes"] else: # If processes not defined, check computing_units or set default process_computing_nodes(self.decorator_name, self.kwargs) kwargs["computing_nodes"] = self.kwargs["computing_nodes"] if "processes_per_node" in self.kwargs: kwargs["processes_per_node"] = self.kwargs["processes_per_node"] else: kwargs["processes_per_node"] = 1 if __debug__: logger.debug("This MPI task will have " + str(kwargs["computing_nodes"]) + " processes and " + str(kwargs["processes_per_node"]) + " processes per node.") if self.task_type == IMPL_PYTHON_MPI: prepend_strings = True else: prepend_strings = False with keep_arguments(args, kwargs, prepend_strings=prepend_strings): # Call the method ret = user_function(*args, **kwargs) return ret
def container_f(*args, **kwargs): if not self.scope: raise NotInPyCOMPSsException(not_in_pycompss("container")) if __debug__: logger.debug("Executing container_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs, user_function) with keep_arguments(args, kwargs, prepend_strings=False): # Call the method ret = user_function(*args, **kwargs) return ret
def implement_f(*args, **kwargs): # type: (*typing.Any, **typing.Any) -> typing.Any # This is executed only when called. if not self.scope: raise NotInPyCOMPSsException(not_in_pycompss("implement")) if __debug__: logger.debug("Executing implement_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs) with keep_arguments(args, kwargs, prepend_strings=True): # Call the method ret = user_function(*args, **kwargs) return ret
def load_loggers(debug, persistent_storage): # type: (bool, str) -> (..., str, ...) """ Load all loggers. :param debug: is Debug enabled. :param persistent_storage: is persistent storage enabled. :return: main logger of the application, the log config file (json) and a list of loggers for the persistent data framework. """ # Load log level configuration file worker_path = os.path.dirname(os.path.realpath(__file__)) log_cfg_path = "".join((worker_path, '/../../../../log')) if not os.path.isdir(log_cfg_path): # If not exists, then we are using the source for unit testing log_cfg_path = "".join((worker_path, '/../../../../../log')) if debug: # Debug log_json = "/".join((log_cfg_path, 'logging_worker_debug.json')) else: # Default log_json = "/".join((log_cfg_path, 'logging_worker_off.json')) log_dir = get_temporary_directory() # log_dir is of the form: # With agents or worker in master: /path/to/working_directory/tmpFiles/pycompssID/ # Normal master-worker execution : /path/to/working_directory/machine_name/pycompssID/ # With normal master-worker execution, it transfers the err and out files in the # expected folder to the master. # With agents or worker in master it does not, so keep it in previous two folders: if context.is_nesting_enabled() or "tmpFiles" in log_dir: log_dir = os.path.join(log_dir, "..", "..") else: log_dir = os.path.join(log_dir, "..") init_logging_worker_piper(log_json, log_dir) # Define logger facilities logger = logging.getLogger('pycompss.worker.piper.piper_worker') storage_loggers = [] if persistent_storage: storage_loggers.append(logging.getLogger('dataclay')) storage_loggers.append(logging.getLogger('hecuba')) storage_loggers.append(logging.getLogger('redis')) storage_loggers.append(logging.getLogger('storage')) return logger, log_json, storage_loggers
def constrained_f(*args, **kwargs): if not self.scope: from pycompss.api.dummy.constraint import constraint \ as dummy_constraint d_c = dummy_constraint(self.args, self.kwargs) return d_c.__call__(user_function)(*args, **kwargs) if __debug__: logger.debug("Executing constrained_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs, user_function) with keep_arguments(args, kwargs, prepend_strings=True): # Call the method ret = user_function(*args, **kwargs) return ret
def __decorator_body__(self, user_function, args, kwargs): # type: (typing.Callable, tuple, dict) -> typing.Any if not self.scope: raise NotImplementedError if __debug__: logger.debug("Executing mpmd_mpi_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs) kwargs[PROCESSES_PER_NODE] = self.kwargs.get(PROCESSES_PER_NODE, 1) kwargs[COMPUTING_NODES] = self.processes with keep_arguments(args, kwargs, prepend_strings=False): # Call the method ret = user_function(*args, **kwargs) return ret
def __decorator_body__(self, user_function, args, kwargs): # type: (typing.Callable, tuple, dict) -> typing.Any # force to serialize with JSON serializer.FORCED_SERIALIZER = 4 if not self.scope: # run http self.__run_http__(args, kwargs) if __debug__: logger.debug("Executing http_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs) with keep_arguments(args, kwargs): # Call the method ret = user_function(*args, **kwargs) return ret
def binary_f(*args, **kwargs): if not self.scope: # Execute the binary as with PyCOMPSs so that sequential # execution performs as parallel. # To disable: raise Exception(not_in_pycompss("binary")) # TODO: Intercept @task parameters to get stream redirection return self.__run_binary__(args, kwargs) if __debug__: logger.debug("Executing binary_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs, user_function) with keep_arguments(args, kwargs, prepend_strings=False): # Call the method ret = user_function(*args, **kwargs) return ret
def decaf_f(*args, **kwargs): # type: (*typing.Any, **typing.Any) -> typing.Any if not self.scope: raise NotInPyCOMPSsException(not_in_pycompss("decaf")) if __debug__: logger.debug("Executing decaf_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs) # Set the computing_nodes variable in kwargs for its usage # in @task decorator kwargs[COMPUTING_NODES] = self.kwargs[COMPUTING_NODES] with keep_arguments(args, kwargs, prepend_strings=False): # Call the method ret = user_function(*args, **kwargs) return ret
def __decorator_body__(self, user_function, args, kwargs): # Determine the context and decide what to do if context.in_master(): # @task being executed in the master # Each task will have a TaskMaster, so its content will # not be shared. self.__check_core_element__(kwargs, user_function) with event(TASK_INSTANTIATION, master=True): master = TaskMaster(self.decorator_arguments, self.user_function, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints, self.on_failure, self.defaults) result = master.call(*args, **kwargs) fo, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints = result # noqa: E501 del master return fo elif context.in_worker(): if "compss_key" in kwargs.keys(): if context.is_nesting_enabled(): # Update the whole logger since it will be in job out/err update_logger_handlers(kwargs["compss_log_cfg"], kwargs["compss_log_files"][0], kwargs["compss_log_files"][1]) # @task being executed in the worker with event(WORKER_TASK_INSTANTIATION, master=False, inside=True): worker = TaskWorker(self.decorator_arguments, self.user_function, self.on_failure, self.defaults) result = worker.call(*args, **kwargs) # Force flush stdout and stderr sys.stdout.flush() sys.stderr.flush() # Remove worker del worker if context.is_nesting_enabled(): # Wait for all nested tasks to finish from pycompss.runtime.binding import nested_barrier nested_barrier() # Reestablish logger handlers update_logger_handlers(kwargs["compss_log_cfg"]) return result else: if context.is_nesting_enabled(): # Each task will have a TaskMaster, so its content will # not be shared. with event(TASK_INSTANTIATION, master=True): master = TaskMaster( self.decorator_arguments, self.user_function, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints, self.on_failure, self.defaults) result = master.call(*args, **kwargs) fo, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints = result # noqa: E501 del master return fo else: # Called from another task within the worker # Ignore the @task decorator and run it sequentially message = "".join(( "WARNING: Calling task: ", str(user_function.__name__), " from this task.\n", " It will be executed sequentially ", # noqa: E501 "within the caller task.")) print(message, file=sys.stderr) return self._sequential_call(*args, **kwargs) # We are neither in master nor in the worker, or the user has # stopped the interactive session. # Therefore, the user code is being executed with no # launch_compss/enqueue_compss/runcompss/interactive session return self._sequential_call(*args, **kwargs)
def _synchronize(obj, mode): # type: (typing.Any, int) -> typing.Any """ Synchronization function. This method retrieves the value of a future object. Calls the runtime in order to wait for the value and returns it when received. :param obj: Object to synchronize. :param mode: Direction of the object to synchronize. :return: The value of the object requested. """ # TODO: Add a boolean to differentiate between files and object on the # COMPSs.open_file call. This change pretends to obtain better traces. # Must be implemented first in the Runtime, then in the bindings common # C API and finally add the boolean here app_id = 0 obj_id = "" # noqa if is_psco(obj): obj_id = str(get_id(obj)) if not OT.is_pending_to_synchronize(obj_id): return obj else: # file_path is of the form storage://pscoId or # file://sys_path_to_file file_path = COMPSs.open_file(app_id, "".join( ("storage://", str(obj_id))), mode) # TODO: Add switch on protocol (first parameter returned currently ignored) _, file_name = file_path.split("://") new_obj = get_by_id(file_name) OT.stop_tracking(obj) return new_obj obj_id = OT.is_tracked(obj) obj_name = OT.get_obj_name(obj_id) if obj_id is None: # Not being tracked return obj if not OT.is_pending_to_synchronize(obj_id): return obj if __debug__: logger.debug("Synchronizing object %s with mode %s" % (obj_id, mode)) file_name = OT.get_file_name(obj_id) compss_file = COMPSs.open_file(app_id, file_name, mode) # Runtime can return a path or a PSCOId if compss_file.startswith('/'): # If the real filename is null, then return None. The task that # produces the output file may have been ignored or cancelled, so its # result does not exist. real_file_name = compss_file.split('/')[-1] if real_file_name == "null": print( "WARNING: Could not retrieve the object " + str(file_name) + " since the task that produces it may have been IGNORED or CANCELLED. Please, check the logs. Returning None." ) # noqa: E501 return None new_obj = deserialize_from_file(compss_file) COMPSs.close_file(app_id, file_name, mode) else: new_obj = get_by_id(compss_file) if context.is_nesting_enabled() and context.in_worker(): # If nesting and in worker, the user wants to synchronize an object. # So it is necessary to update the parameter with the new object. update_worker_argument_parameter_content(obj_name, new_obj) if mode == 'r': OT.update_mapping(obj_id, new_obj) if mode != 'r': COMPSs.delete_file(app_id, OT.get_file_name(obj_id), False) OT.stop_tracking(obj) return new_obj
def task_decorator(*args, **kwargs): # Determine the context and decide what to do if context.in_master(): # @task being executed in the master # Each task will have a TaskMaster, so its content will # not be shared. self.__check_core_element__(kwargs, user_function) with event(TASK_INSTANTIATION, master=True): master = TaskMaster( self.decorator_arguments, self.user_function, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints) result = master.call(*args, **kwargs) fo, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints = result # noqa: E501 del master return fo elif context.in_worker(): if 'compss_key' in kwargs.keys(): # @task being executed in the worker with event(WORKER_TASK_INSTANTIATION, master=False, inside=True): worker = TaskWorker(self.decorator_arguments, self.user_function) result = worker.call(*args, **kwargs) del worker return result else: if context.is_nesting_enabled(): # nested @task executed in the worker # Each task will have a TaskMaster, so its content will # not be shared. with event(TASK_INSTANTIATION, master=True): master = TaskMaster( self.decorator_arguments, self.user_function, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints) result = master.call(*args, **kwargs) fo, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints = result # noqa: E501 del master return fo else: # Called from another task within the worker # Ignore the @task decorator and run it sequentially message = "".join(( "WARNING: Calling task: ", str(user_function.__name__), " from this task.\n", " It will be executed sequentially ", # noqa: E501 "within the caller task.")) print(message, file=sys.stderr) return self._sequential_call(*args, **kwargs) # We are neither in master nor in the worker, or the user has # stopped the interactive session. # Therefore, the user code is being executed with no # launch_compss/enqueue_compss/runcompss/interactive session return self._sequential_call(*args, **kwargs)