def start_runtime(log_level='off', tracing=0, interactive=False): # type: (str, int, bool) -> None """ Starts the COMPSs runtime. Starts the runtime by calling the external python library that calls the bindings-common. :param log_level: Log level [ 'trace' | 'debug' | 'info' | 'api' | 'off' ]. :param tracing: Tracing level [0 (deactivated) | 1 (basic) | 2 (advanced)]. :param interactive: Boolean if interactive (ipython or jupyter). :return: None """ if __debug__: logger.info("Starting COMPSs...") if tracing > 0 and not interactive: # Enabled only if not interactive - extrae issues within jupyter. enable_trace_master() with event(START_RUNTIME_EVENT, master=True): if interactive and context.in_master(): COMPSs.load_runtime(external_process=True) else: COMPSs.load_runtime(external_process=False) if log_level == 'trace': # Could also be 'debug' or True, but we only show the C extension # debug in the maximum tracing level. COMPSs.set_debug(True) OT_enable_report() COMPSs.start_runtime() if __debug__: logger.info("COMPSs started")
def event(event_id, master=False, inside=False, cpu_affinity=False, gpu_affinity=False): # type: (int or str, bool, bool, bool, bool) -> None """ Emits an event wrapping the desired code. Does nothing if tracing is disabled. :param event_id: Event identifier to emit. :param master: If the event is emitted as master. :param inside: If the event is produced inside the worker. :param cpu_affinity: If the event is produced inside the worker for cpu affinity. :param gpu_affinity: If the event is produced inside the worker for gpu affinity. :return: None """ emit = False if TRACING and in_master() and master: emit = True if TRACING and in_worker() and not master: emit = True if emit: event_group, event_id = __get_proper_type_event__( event_id, master, inside, cpu_affinity, gpu_affinity) PYEXTRAE.eventandcounters(event_group, event_id) # noqa yield # here the code runs if emit: PYEXTRAE.eventandcounters(event_group, 0) # noqa
def multinode_f(*args, **kwargs): if not self.scope: raise Exception(not_in_pycompss("MultiNode")) if __debug__: logger.debug("Executing multinode_f wrapper.") if context.in_master(): # master code if not self.core_element_configured: self.__configure_core_element__(kwargs) else: # worker code set_slurm_environment() # Set the computing_nodes variable in kwargs for its usage # in @task decorator kwargs['computing_nodes'] = self.kwargs['computing_nodes'] with keep_arguments(args, kwargs, prepend_strings=True): # Call the method ret = func(*args, **kwargs) if context.in_worker(): reset_slurm_environment() return ret
def constrained_f(*args, **kwargs): # type: (*typing.Any, **typing.Any) -> typing.Any if not self.scope: from pycompss.api.dummy.on_failure import on_failure \ as dummy_on_failure d_c = dummy_on_failure(self.args, self.kwargs) return d_c.__call__(user_function)(*args, **kwargs) if __debug__: logger.debug("Executing on_failure_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs) # Set the on failure management action and default variables in # kwargs for its usage in @task decorator kwargs["on_failure"] = self.on_failure_action kwargs["defaults"] = self.defaults with keep_arguments(args, kwargs, prepend_strings=True): # Call the method ret = user_function(*args, **kwargs) return ret
def test_in_master_context(): from pycompss.util.context import in_master from pycompss.util.context import set_pycompss_context set_pycompss_context(MASTER) master_context = in_master() assert master_context is True, MASTER_CONTEXT_ERROR set_pycompss_context(OUT_OF_SCOPE)
def multinode_f(*args, **kwargs): if not self.scope: raise NotInPyCOMPSsException(not_in_pycompss("MultiNode")) if __debug__: logger.debug("Executing multinode_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs, user_function) if context.in_worker(): old_slurm_env = set_slurm_environment() # Set the computing_nodes variable in kwargs for its usage # in @task decorator kwargs['computing_nodes'] = self.kwargs['computing_nodes'] with keep_arguments(args, kwargs, prepend_strings=True): # Call the method ret = user_function(*args, **kwargs) if context.in_worker(): reset_slurm_environment(old_slurm_env) return ret
def __decorator_body__(self, user_function, args, kwargs): # type: (typing.Callable, tuple, dict) -> typing.Any if not self.scope: # Execute the mpi as with PyCOMPSs so that sequential # execution performs as parallel. # To disable: raise Exception(not_in_pycompss("mpi")) # TODO: Intercept @task parameters to get stream redirection if "binary" in self.kwargs: return self.__run_mpi__(args, kwargs) else: print( "WARN: Python MPI as dummy is not fully supported. Executing decorated funtion." ) return user_function(*args, **kwargs) if __debug__: logger.debug("Executing mpi_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs) # The processes parameter will have to go down until the execution # is invoked. To this end, set the computing_nodes variable in kwargs # for its usage in @task decorator # WARNING: processes can be an int, a env string, a str with # dynamic variable name. if "processes" in self.kwargs: kwargs["computing_nodes"] = self.kwargs["processes"] else: # If processes not defined, check computing_units or set default process_computing_nodes(self.decorator_name, self.kwargs) kwargs["computing_nodes"] = self.kwargs["computing_nodes"] if "processes_per_node" in self.kwargs: kwargs["processes_per_node"] = self.kwargs["processes_per_node"] else: kwargs["processes_per_node"] = 1 if __debug__: logger.debug("This MPI task will have " + str(kwargs["computing_nodes"]) + " processes and " + str(kwargs["processes_per_node"]) + " processes per node.") if self.task_type == IMPL_PYTHON_MPI: prepend_strings = True else: prepend_strings = False with keep_arguments(args, kwargs, prepend_strings=prepend_strings): # Call the method ret = user_function(*args, **kwargs) return ret
def binary_f(*args, **kwargs): if not self.scope: # Execute the binary as with PyCOMPSs so that sequential # execution performs as parallel. # To disable: raise Exception(not_in_pycompss("binary")) # TODO: Intercept the @task parameters to get stream redirection cmd = [self.kwargs['binary']] if args: args = [str(a) for a in args] cmd += args my_env = os.environ.copy() if "working_dir" in self.kwargs: my_env[ "PATH"] = self.kwargs["working_dir"] + my_env["PATH"] elif "workingDir" in self.kwargs: my_env["PATH"] = self.kwargs["workingDir"] + my_env["PATH"] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=my_env) # noqa: E501 out, err = proc.communicate() if sys.version_info[0] < 3: out_message = out.strip() err_message = err.strip() else: out_message = out.decode().strip() err_message = err.decode().strip() if out_message: print(out_message) if err_message: sys.stderr.write(err_message + '\n') return proc.returncode if __debug__: logger.debug("Executing binary_f wrapper.") if context.in_master(): # master code if not self.core_element_configured: self.__configure_core_element__(kwargs, user_function) else: # worker code pass with keep_arguments(args, kwargs, prepend_strings=False): # Call the method ret = user_function(*args, **kwargs) return ret
def container_f(*args, **kwargs): if not self.scope: raise NotInPyCOMPSsException(not_in_pycompss("container")) if __debug__: logger.debug("Executing container_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs, user_function) with keep_arguments(args, kwargs, prepend_strings=False): # Call the method ret = user_function(*args, **kwargs) return ret
def software_f(*args, **kwargs): # type: (*typing.Any, **typing.Any) -> typing.Any if not self.scope or not context.in_master(): # Execute the software as with PyCOMPSs so that sequential # execution performs as parallel. # To disable: raise Exception(not_in_pycompss(BINARY)) return user_function(*args, **kwargs) if __debug__: logger.debug("Executing software_f wrapper.") if self.constraints is not None: core_element = CE() core_element.set_impl_constraints(self.constraints) kwargs[CORE_ELEMENT_KEY] = core_element if self.container is not None: _func = str(user_function.__name__) impl_type = IMPL_CONTAINER impl_signature = '.'.join((impl_type, _func)) ce = kwargs.get(CORE_ELEMENT_KEY, CE()) impl_args = [self.container[ENGINE], # engine self.container[IMAGE], # image UNASSIGNED, # internal_type UNASSIGNED, # internal_binary UNASSIGNED, # internal_func UNASSIGNED, # working_dir UNASSIGNED] # fail_by_ev ce.set_impl_type(impl_type) ce.set_impl_signature(impl_signature) ce.set_impl_type_args(impl_args) kwargs[CORE_ELEMENT_KEY] = ce if self.decor: decorator = self.decor def decor_f(): def f(): ret = decorator(**self.config_args) return ret(user_function)(*args, **kwargs) return f() return decor_f() else: # It's a PyCOMPSs task with only @task and @software decorators return user_function(*args, **kwargs)
def implement_f(*args, **kwargs): # type: (*typing.Any, **typing.Any) -> typing.Any # This is executed only when called. if not self.scope: raise NotInPyCOMPSsException(not_in_pycompss("implement")) if __debug__: logger.debug("Executing implement_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs) with keep_arguments(args, kwargs, prepend_strings=True): # Call the method ret = user_function(*args, **kwargs) return ret
def constrained_f(*args, **kwargs): if not self.scope: from pycompss.api.dummy.constraint import constraint \ as dummy_constraint d_c = dummy_constraint(self.args, self.kwargs) return d_c.__call__(user_function)(*args, **kwargs) if __debug__: logger.debug("Executing constrained_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs, user_function) with keep_arguments(args, kwargs, prepend_strings=True): # Call the method ret = user_function(*args, **kwargs) return ret
def container_f(*args, **kwargs): if not self.scope: raise Exception(not_in_pycompss("container")) if __debug__: logger.debug("Executing container_f wrapper.") if context.in_master(): # master code if not self.core_element_configured: self.__configure_core_element__(kwargs, user_function) else: # worker code pass with keep_arguments(args, kwargs, prepend_strings=False): # Call the method ret = user_function(*args, **kwargs) return ret
def binary_f(*args, **kwargs): if not self.scope: # Execute the binary as with PyCOMPSs so that sequential # execution performs as parallel. # To disable: raise Exception(not_in_pycompss("binary")) # TODO: Intercept @task parameters to get stream redirection return self.__run_binary__(args, kwargs) if __debug__: logger.debug("Executing binary_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs, user_function) with keep_arguments(args, kwargs, prepend_strings=False): # Call the method ret = user_function(*args, **kwargs) return ret
def __decorator_body__(self, user_function, args, kwargs): # type: (typing.Callable, tuple, dict) -> typing.Any # force to serialize with JSON serializer.FORCED_SERIALIZER = 4 if not self.scope: # run http self.__run_http__(args, kwargs) if __debug__: logger.debug("Executing http_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs) with keep_arguments(args, kwargs): # Call the method ret = user_function(*args, **kwargs) return ret
def implement_f(*args, **kwargs): # This is executed only when called. if not self.scope: raise Exception(not_in_pycompss("implement")) if __debug__: logger.debug("Executing implement_f wrapper.") if context.in_master(): # master code if not self.core_element_configured: self.__configure_core_element__(kwargs) else: # worker code pass with keep_arguments(args, kwargs, prepend_strings=True): # Call the method ret = func(*args, **kwargs) return ret
def __decorator_body__(self, user_function, args, kwargs): # type: (typing.Callable, tuple, dict) -> typing.Any if not self.scope: raise NotImplementedError if __debug__: logger.debug("Executing mpmd_mpi_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs) kwargs[PROCESSES_PER_NODE] = self.kwargs.get(PROCESSES_PER_NODE, 1) kwargs[COMPUTING_NODES] = self.processes with keep_arguments(args, kwargs, prepend_strings=False): # Call the method ret = user_function(*args, **kwargs) return ret
def decaf_f(*args, **kwargs): # type: (*typing.Any, **typing.Any) -> typing.Any if not self.scope: raise NotInPyCOMPSsException(not_in_pycompss("decaf")) if __debug__: logger.debug("Executing decaf_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs) # Set the computing_nodes variable in kwargs for its usage # in @task decorator kwargs[COMPUTING_NODES] = self.kwargs[COMPUTING_NODES] with keep_arguments(args, kwargs, prepend_strings=False): # Call the method ret = user_function(*args, **kwargs) return ret
def __call__(self, func): """ Parse and set the implement parameters within the task core element. :param func: Function to decorate. :return: Decorated function. """ @wraps(func) def implement_f(*args, **kwargs): # This is executed only when called. if not self.scope: raise Exception(not_in_pycompss("implement")) if __debug__: logger.debug("Executing implement_f wrapper.") if context.in_master(): # master code if not self.core_element_configured: self.__configure_core_element__(kwargs) else: # worker code pass with keep_arguments(args, kwargs, prepend_strings=True): # Call the method ret = func(*args, **kwargs) return ret implement_f.__doc__ = func.__doc__ if context.in_master() and not self.first_register: import pycompss.api.task as t self.first_register = True t.REGISTER_ONLY = True self.__call__(func)(self) t.REGISTER_ONLY = False return implement_f
def __call__(self, user_function): # type: (typing.Callable) -> typing.Callable """ Parse and set the implement parameters within the task core element. :param user_function: Function to decorate. :return: Decorated function. """ @wraps(user_function) def implement_f(*args, **kwargs): # type: (*typing.Any, **typing.Any) -> typing.Any # This is executed only when called. if not self.scope: raise NotInPyCOMPSsException(not_in_pycompss("implement")) if __debug__: logger.debug("Executing implement_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs) with keep_arguments(args, kwargs, prepend_strings=True): # Call the method ret = user_function(*args, **kwargs) return ret implement_f.__doc__ = user_function.__doc__ if context.in_master() and not self.first_register: import pycompss.api.task as t self.first_register = True t.REGISTER_ONLY = True self.__call__(user_function)(self) t.REGISTER_ONLY = False return implement_f
def constrained_f(*args, **kwargs): if not self.scope: from pycompss.api.dummy.constraint import constraint \ as dummy_constraint d_c = dummy_constraint(self.args, self.kwargs) return d_c.__call__(func)(*args, **kwargs) if __debug__: logger.debug("Executing constrained_f wrapper.") if context.in_master(): # master code if not self.core_element_configured: self.__configure_core_element__(kwargs) else: # worker code pass with keep_arguments(args, kwargs, prepend_strings=True): # Call the method ret = func(*args, **kwargs) return ret
def __init__(self, *args, **kwargs): # type: (*typing.Any, **typing.Any) -> None """ Parse the config file and store the arguments that will be used later to wrap the "real" decorator. self = itself. args = not used. kwargs = dictionary with the given @software parameter (config_file). :param args: Arguments :param kwargs: Keyword arguments """ decorator_name = "".join(('@', Software.__name__.lower())) # super(Software, self).__init__(decorator_name, *args, **kwargs) self.task_type = None # type: typing.Any self.config_args = None # type: typing.Any self.decor = None # type: typing.Any self.constraints = None # type: typing.Any self.container = None # type: typing.Any self.decorator_name = decorator_name self.args = args self.kwargs = kwargs self.scope = context.in_pycompss() self.core_element = None # type: typing.Any self.core_element_configured = False if self.scope and context.in_master(): if __debug__: logger.debug("Init @software decorator..") # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), decorator_name) self.parse_config_file()
def ompss_f(*args, **kwargs): if not self.scope: # from pycompss.api.dummy.ompss import ompss as dummy_ompss # d_o = dummy_ompss(self.args, self.kwargs) # return d_o.__call__(func) raise Exception(not_in_pycompss("ompss")) if context.in_master(): # master code mod = inspect.getmodule(func) self.module = mod.__name__ # not func.__module__ if (self.module == '__main__' or self.module == 'pycompss.runtime.launch'): # The module where the function is defined was run as # __main__, so we need to find out the real module name. # path = mod.__file__ # dirs = mod.__file__.split(os.sep) # file_name = os.path.splitext( # os.path.basename(mod.__file__))[0] # Get the real module name from our launch.py variable path = getattr(mod, "APP_PATH") dirs = path.split(os.path.sep) file_name = os.path.splitext(os.path.basename(path))[0] mod_name = file_name i = len(dirs) - 1 while i > 0: new_l = len(path) - (len(dirs[i]) + 1) path = path[0:new_l] if "__init__.py" in os.listdir(path): # directory is a package i -= 1 mod_name = dirs[i] + '.' + mod_name else: break self.module = mod_name # Include the registering info related to @ompss # Retrieve the base core_element established at @task decorator from pycompss.api.task import current_core_element as cce if not self.registered: self.registered = True # Update the core element information with the @ompss # information cce.set_impl_type("OMPSS") binary = self.kwargs['binary'] if 'working_dir' in self.kwargs: working_dir = self.kwargs['working_dir'] elif 'workingDir' in self.kwargs: working_dir = self.kwargs['workingDir'] else: working_dir = '[unassigned]' # Empty or '[unassigned]' impl_signature = 'OMPSS.' + binary cce.set_impl_signature(impl_signature) impl_args = [binary, working_dir] cce.set_impl_type_args(impl_args) else: # worker code pass # This is executed only when called. if __debug__: logger.debug("Executing ompss_f wrapper.") # Set the computing_nodes variable in kwargs for its usage # in @task decorator kwargs['computing_nodes'] = self.kwargs['computing_nodes'] if len(args) > 0: # The 'self' for a method function is passed as args[0] slf = args[0] # Replace and store the attributes saved = {} for k, v in self.kwargs.items(): if hasattr(slf, k): saved[k] = getattr(slf, k) setattr(slf, k, v) # Call the method import pycompss.api.task as t t.prepend_strings = False ret = func(*args, **kwargs) t.prepend_strings = True if len(args) > 0: # Put things back for k, v in saved.items(): setattr(slf, k, v) return ret
def __decorator_body__(self, user_function, args, kwargs): # Determine the context and decide what to do if context.in_master(): # @task being executed in the master # Each task will have a TaskMaster, so its content will # not be shared. self.__check_core_element__(kwargs, user_function) with event(TASK_INSTANTIATION, master=True): master = TaskMaster(self.decorator_arguments, self.user_function, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints, self.on_failure, self.defaults) result = master.call(*args, **kwargs) fo, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints = result # noqa: E501 del master return fo elif context.in_worker(): if "compss_key" in kwargs.keys(): if context.is_nesting_enabled(): # Update the whole logger since it will be in job out/err update_logger_handlers(kwargs["compss_log_cfg"], kwargs["compss_log_files"][0], kwargs["compss_log_files"][1]) # @task being executed in the worker with event(WORKER_TASK_INSTANTIATION, master=False, inside=True): worker = TaskWorker(self.decorator_arguments, self.user_function, self.on_failure, self.defaults) result = worker.call(*args, **kwargs) # Force flush stdout and stderr sys.stdout.flush() sys.stderr.flush() # Remove worker del worker if context.is_nesting_enabled(): # Wait for all nested tasks to finish from pycompss.runtime.binding import nested_barrier nested_barrier() # Reestablish logger handlers update_logger_handlers(kwargs["compss_log_cfg"]) return result else: if context.is_nesting_enabled(): # Each task will have a TaskMaster, so its content will # not be shared. with event(TASK_INSTANTIATION, master=True): master = TaskMaster( self.decorator_arguments, self.user_function, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints, self.on_failure, self.defaults) result = master.call(*args, **kwargs) fo, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints = result # noqa: E501 del master return fo else: # Called from another task within the worker # Ignore the @task decorator and run it sequentially message = "".join(( "WARNING: Calling task: ", str(user_function.__name__), " from this task.\n", " It will be executed sequentially ", # noqa: E501 "within the caller task.")) print(message, file=sys.stderr) return self._sequential_call(*args, **kwargs) # We are neither in master nor in the worker, or the user has # stopped the interactive session. # Therefore, the user code is being executed with no # launch_compss/enqueue_compss/runcompss/interactive session return self._sequential_call(*args, **kwargs)
def mpi_f(*args, **kwargs): if not self.scope: # from pycompss.api.dummy.mpi import mpi as dummy_mpi # d_m = dummy_mpi(self.args, self.kwargs) # return d_m.__call__(func) raise Exception(not_in_pycompss("mpi")) if context.in_master(): # master code mod = inspect.getmodule(func) self.module = mod.__name__ # not func.__module__ if self.module == '__main__' or \ self.module == 'pycompss.runtime.launch': # The module where the function is defined was run as # __main__, so we need to find out the real module name. # path = mod.__file__ # dirs = mod.__file__.split(os.sep) # file_name = os.path.splitext( # os.path.basename(mod.__file__))[0] # Get the real module name from our launch.py variable path = getattr(mod, "APP_PATH") dirs = path.split(os.path.sep) file_name = os.path.splitext(os.path.basename(path))[0] mod_name = file_name i = len(dirs) - 1 while i > 0: new_l = len(path) - (len(dirs[i]) + 1) path = path[0:new_l] if "__init__.py" in os.listdir(path): # directory is a package i -= 1 mod_name = dirs[i] + '.' + mod_name else: break self.module = mod_name # Include the registering info related to @mpi # Retrieve the base core_element established at @task decorator from pycompss.api.task import current_core_element as cce if not self.registered: self.registered = True # Update the core element information with the @mpi # information if "binary" in self.kwargs: binary = self.kwargs['binary'] cce.set_impl_type("MPI") else: binary = "[unassigned]" cce.set_impl_type("PYTHON_MPI") self.task_type = "PYTHON_MPI" if 'working_dir' in self.kwargs: working_dir = self.kwargs['working_dir'] else: working_dir = '[unassigned]' # Empty or '[unassigned]' runner = self.kwargs['runner'] if 'flags' in self.kwargs: flags = self.kwargs['flags'] else: flags = '[unassigned]' # Empty or '[unassigned]' if 'scale_by_cu' in self.kwargs: scale_by_cu = self.kwargs['scale_by_cu'] if isinstance(scale_by_cu, bool): if scale_by_cu: scale_by_cu_str = 'true' else: scale_by_cu_str = 'false' elif isinstance(scale_by_cu, str): scale_by_cu_str = scale_by_cu else: raise Exception("Incorrect format for scale_by_cu property. " + # noqa: E501 "It should be boolean or an environment variable") # noqa: E501 else: scale_by_cu_str = 'false' if 'fail_by_exit_value' in self.kwargs: fail_by_ev = self.kwargs['fail_by_exit_value'] if isinstance(fail_by_ev, bool): if fail_by_ev: fail_by_ev_str = 'true' else: fail_by_ev_str = 'false' elif isinstance(fail_by_ev, str): fail_by_ev_str = fail_by_ev else: raise Exception("Incorrect format for fail_by_exit_value property. " + # noqa: E501 "It should be boolean or an environment variable") # noqa: E501 else: fail_by_ev_str = 'false' if binary == "[unassigned]": impl_signature = "MPI." else: impl_signature = 'MPI.' + \ str(self.kwargs['processes']) + \ "." + binary # Add information to CoreElement cce.set_impl_signature(impl_signature) impl_args = [binary, working_dir, runner, flags, scale_by_cu_str, fail_by_ev_str] cce.set_impl_type_args(impl_args) else: # worker code pass # This is executed only when called. if __debug__: logger.debug("Executing mpi_f wrapper.") # Set the computing_nodes variable in kwargs for its usage # in @task decorator kwargs['computing_nodes'] = self.kwargs['processes'] if len(args) > 0: # The 'self' for a method function is passed as args[0] slf = args[0] # Replace and store the attributes saved = {} for k, v in self.kwargs.items(): if hasattr(slf, k): saved[k] = getattr(slf, k) setattr(slf, k, v) # Call the method import pycompss.api.task as t if self.task_type == "PYTHON_MPI": t.prepend_strings = True else: t.prepend_strings = False ret = func(*args, **kwargs) t.prepend_strings = True if len(args) > 0: # Put things back for k, v in saved.items(): setattr(slf, k, v) return ret
def constrained_f(*args, **kwargs): if not self.scope: from pycompss.api.dummy.constraint import constraint as dummy_constraint # noqa d_c = dummy_constraint(self.args, self.kwargs) return d_c.__call__(func)(*args, **kwargs) if context.in_master(): # master code mod = inspect.getmodule(func) self.module = mod.__name__ # not func.__module__ if (self.module == '__main__' or self.module == 'pycompss.runtime.launch'): # The module where the function is defined was run as # __main__, so we need to find out the real module name. # path = mod.__file__ # dirs = mod.__file__.split(os.sep) # file_name = os.path.splitext( # os.path.basename(mod.__file__))[0] # Get the real module name from our launch.py variable path = getattr(mod, "APP_PATH") dirs = path.split(os.path.sep) file_name = os.path.splitext(os.path.basename(path))[0] mod_name = file_name i = len(dirs) - 1 while i > 0: new_l = len(path) - (len(dirs[i]) + 1) path = path[0:new_l] if "__init__.py" in os.listdir(path): # directory is a package i -= 1 mod_name = dirs[i] + '.' + mod_name else: break self.module = mod_name # Include the registering info related to @constraint if not self.registered: self.registered = True # Retrieve the base core_element established at @task # decorator from pycompss.api.task import current_core_element as cce # Update the core element information with the constraints cce.set_impl_constraints(self.kwargs) else: # worker code pass # This is executed only when called. if __debug__: logger.debug("Executing constrained_f wrapper.") if len(args) > 0: # The 'self' for a method function is passed as args[0] slf = args[0] # Replace and store the attributes saved = {} for k, v in self.kwargs.items(): if hasattr(slf, k): saved[k] = getattr(slf, k) setattr(slf, k, v) # Call the method ret = func(*args, **kwargs) if len(args) > 0: # Put things back for k, v in saved.items(): setattr(slf, k, v) return ret
def implement_f(*args, **kwargs): # This is executed only when called. if not self.scope: # from pycompss.api.dummy.implement import implement as dummy_implement # noqa # d_i = dummy_implement(self.args, self.kwargs) # return d_i.__call__(func) raise Exception(not_in_pycompss("implement")) if context.in_master(): # master code mod = inspect.getmodule(func) self.module = mod.__name__ # not func.__module__ if (self.module == '__main__' or self.module == 'pycompss.runtime.launch'): # The module where the function is defined was run as # __main__, so we need to find out the real module name. # path = mod.__file__ # dirs = mod.__file__.split(os.sep) # file_name = os.path.splitext( # os.path.basename(mod.__file__))[0] # Get the real module name from our launch.py variable path = getattr(mod, "APP_PATH") dirs = path.split(os.path.sep) file_name = os.path.splitext(os.path.basename(path))[0] mod_name = file_name i = len(dirs) - 1 while i > 0: new_l = len(path) - (len(dirs[i]) + 1) path = path[0:new_l] if "__init__.py" in os.listdir(path): # directory is a package i -= 1 mod_name = dirs[i] + '.' + mod_name else: break self.module = mod_name # Include the registering info related to @implement # Retrieve the base core_element established at @task decorator if not self.registered: self.registered = True from pycompss.api.task import current_core_element as cce # Update the core element information with the @implement # information if 'sourceClass' in self.kwargs: another_class = self.kwargs['sourceClass'] else: another_class = self.kwargs['source_class'] another_method = self.kwargs['method'] ce_signature = another_class + '.' + another_method cce.set_ce_signature(ce_signature) # This is not needed since the arguments are already set # by the task decorator. # implArgs = [another_class, another_method] # cce.set_implTypeArgs(implArgs) cce.set_impl_type("METHOD") else: # worker code pass if __debug__: logger.debug("Executing implement_f wrapper.") # The 'self' for a method function is passed as args[0] slf = args[0] # Replace and store the attributes saved = {} for k, v in self.kwargs.items(): if hasattr(slf, k): saved[k] = getattr(slf, k) setattr(slf, k, v) # Call the method ret = func(*args, **kwargs) # Put things back for k, v in saved.items(): setattr(slf, k, v) return ret
def __init__(self, event_id): # type: (int) -> None self.emitted = False if TRACING and in_master(): PYEXTRAE.eventandcounters(BINDING_MASTER_TYPE, event_id) self.emitted = True
def mpi_f(*args, **kwargs): if not self.scope: # Execute the mpi as with PyCOMPSs so that sequential # execution performs as parallel. # To disable: raise Exception(not_in_pycompss("mpi")) # TODO: Intercept the @task parameters to get stream redirection cmd = [self.kwargs['runner']] if 'processes' in self.kwargs: cmd += ['-np', self.kwargs['processes']] elif 'computing_nodes' in self.kwargs: cmd += ['-np', self.kwargs['computing_nodes']] elif 'computingNodes' in self.kwargs: cmd += ['-np', self.kwargs['computingNodes']] else: pass if 'flags' in self.kwargs: cmd += self.kwargs['flags'].split() cmd += [self.kwargs['binary']] if args: args = [str(a) for a in args] cmd += args my_env = os.environ.copy() if "working_dir" in self.kwargs: my_env[ "PATH"] = self.kwargs["working_dir"] + my_env["PATH"] elif "workingDir" in self.kwargs: my_env["PATH"] = self.kwargs["workingDir"] + my_env["PATH"] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=my_env) # noqa: E501 out, err = proc.communicate() if sys.version_info[0] < 3: out_message = out.strip() err_message = err.strip() else: out_message = out.decode().strip() err_message = err.decode().strip() if out_message: print(out_message) if err_message: sys.stderr.write(err_message + '\n') return proc.returncode if __debug__: logger.debug("Executing mpi_f wrapper.") if context.in_master(): # master code if not self.core_element_configured: self.__configure_core_element__(kwargs) else: # worker code pass # Set the computing_nodes variable in kwargs for its usage # in @task decorator kwargs['computing_nodes'] = self.kwargs['processes'] if self.task_type == "PYTHON_MPI": prepend_strings = True else: prepend_strings = False with keep_arguments(args, kwargs, prepend_strings=prepend_strings): # Call the method ret = func(*args, **kwargs) return ret
def mpi_f(*args, **kwargs): ret = self.__decorator_body__(user_function, args, kwargs) if context.in_master() and int(self.kwargs["processes"]) == 1: return [ret] else: return ret