def multinode_f(*args, **kwargs): if not self.scope: raise NotInPyCOMPSsException(not_in_pycompss("MultiNode")) if __debug__: logger.debug("Executing multinode_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs, user_function) if context.in_worker(): old_slurm_env = set_slurm_environment() # Set the computing_nodes variable in kwargs for its usage # in @task decorator kwargs['computing_nodes'] = self.kwargs['computing_nodes'] with keep_arguments(args, kwargs, prepend_strings=True): # Call the method ret = user_function(*args, **kwargs) if context.in_worker(): reset_slurm_environment(old_slurm_env) return ret
def multinode_f(*args, **kwargs): if not self.scope: raise Exception(not_in_pycompss("MultiNode")) if __debug__: logger.debug("Executing multinode_f wrapper.") if context.in_master(): # master code if not self.core_element_configured: self.__configure_core_element__(kwargs) else: # worker code set_slurm_environment() # Set the computing_nodes variable in kwargs for its usage # in @task decorator kwargs['computing_nodes'] = self.kwargs['computing_nodes'] with keep_arguments(args, kwargs, prepend_strings=True): # Call the method ret = func(*args, **kwargs) if context.in_worker(): reset_slurm_environment() return ret
def test_err_msgs_not_in_pycompss(): decorator_name = "@unittest" expected = "The %s decorator only works within PyCOMPSs framework." % \ decorator_name error = not_in_pycompss(decorator_name=decorator_name) assert error == expected,\ "Received wrong error message."
def container_f(*args, **kwargs): if not self.scope: raise NotInPyCOMPSsException(not_in_pycompss("container")) if __debug__: logger.debug("Executing container_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs, user_function) with keep_arguments(args, kwargs, prepend_strings=False): # Call the method ret = user_function(*args, **kwargs) return ret
def reduce_f(*args, **kwargs): if not self.scope: raise PyCOMPSsException(not_in_pycompss("reduction")) if __debug__: logger.debug("Executing reduce_f wrapper.") # Set the chunk size and is_reduce variables in kwargs for their # usage in @task decorator kwargs['chunk_size'] = self.kwargs['chunk_size'] kwargs['is_reduce'] = self.kwargs['is_reduce'] with keep_arguments(args, kwargs, prepend_strings=False): # Call the method ret = func(*args, **kwargs) return ret
def implement_f(*args, **kwargs): # type: (*typing.Any, **typing.Any) -> typing.Any # This is executed only when called. if not self.scope: raise NotInPyCOMPSsException(not_in_pycompss("implement")) if __debug__: logger.debug("Executing implement_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs) with keep_arguments(args, kwargs, prepend_strings=True): # Call the method ret = user_function(*args, **kwargs) return ret
def container_f(*args, **kwargs): if not self.scope: raise Exception(not_in_pycompss("container")) if __debug__: logger.debug("Executing container_f wrapper.") if context.in_master(): # master code if not self.core_element_configured: self.__configure_core_element__(kwargs, user_function) else: # worker code pass with keep_arguments(args, kwargs, prepend_strings=False): # Call the method ret = user_function(*args, **kwargs) return ret
def implement_f(*args, **kwargs): # This is executed only when called. if not self.scope: raise Exception(not_in_pycompss("implement")) if __debug__: logger.debug("Executing implement_f wrapper.") if context.in_master(): # master code if not self.core_element_configured: self.__configure_core_element__(kwargs) else: # worker code pass with keep_arguments(args, kwargs, prepend_strings=True): # Call the method ret = func(*args, **kwargs) return ret
def decaf_f(*args, **kwargs): # type: (*typing.Any, **typing.Any) -> typing.Any if not self.scope: raise NotInPyCOMPSsException(not_in_pycompss("decaf")) if __debug__: logger.debug("Executing decaf_f wrapper.") if (context.in_master() or context.is_nesting_enabled()) \ and not self.core_element_configured: # master code - or worker with nesting enabled self.__configure_core_element__(kwargs) # Set the computing_nodes variable in kwargs for its usage # in @task decorator kwargs[COMPUTING_NODES] = self.kwargs[COMPUTING_NODES] with keep_arguments(args, kwargs, prepend_strings=False): # Call the method ret = user_function(*args, **kwargs) return ret
def implement_f(*args, **kwargs): # This is executed only when called. if not self.scope: # from pycompss.api.dummy.implement import implement as dummy_implement # noqa # d_i = dummy_implement(self.args, self.kwargs) # return d_i.__call__(func) raise Exception(not_in_pycompss("implement")) if context.in_master(): # master code mod = inspect.getmodule(func) self.module = mod.__name__ # not func.__module__ if (self.module == '__main__' or self.module == 'pycompss.runtime.launch'): # The module where the function is defined was run as # __main__, so we need to find out the real module name. # path = mod.__file__ # dirs = mod.__file__.split(os.sep) # file_name = os.path.splitext( # os.path.basename(mod.__file__))[0] # Get the real module name from our launch.py variable path = getattr(mod, "APP_PATH") dirs = path.split(os.path.sep) file_name = os.path.splitext(os.path.basename(path))[0] mod_name = file_name i = len(dirs) - 1 while i > 0: new_l = len(path) - (len(dirs[i]) + 1) path = path[0:new_l] if "__init__.py" in os.listdir(path): # directory is a package i -= 1 mod_name = dirs[i] + '.' + mod_name else: break self.module = mod_name # Include the registering info related to @implement # Retrieve the base core_element established at @task decorator if not self.registered: self.registered = True from pycompss.api.task import current_core_element as cce # Update the core element information with the @implement # information if 'sourceClass' in self.kwargs: another_class = self.kwargs['sourceClass'] else: another_class = self.kwargs['source_class'] another_method = self.kwargs['method'] ce_signature = another_class + '.' + another_method cce.set_ce_signature(ce_signature) # This is not needed since the arguments are already set # by the task decorator. # implArgs = [another_class, another_method] # cce.set_implTypeArgs(implArgs) cce.set_impl_type("METHOD") else: # worker code pass if __debug__: logger.debug("Executing implement_f wrapper.") # The 'self' for a method function is passed as args[0] slf = args[0] # Replace and store the attributes saved = {} for k, v in self.kwargs.items(): if hasattr(slf, k): saved[k] = getattr(slf, k) setattr(slf, k, v) # Call the method ret = func(*args, **kwargs) # Put things back for k, v in saved.items(): setattr(slf, k, v) return ret
def test_err_msgs_not_in_pycompss(): decorator_name = DECORATOR_NAME expected = ("The %s decorator only works within PyCOMPSs framework." % decorator_name) error = not_in_pycompss(decorator_name=decorator_name) assert error == expected, "Received wrong NOT IN PYCOMPSS error message."
def __call__(self, func): """ Parse and set the opencl parameters within the task core element. :param func: Function to decorate :return: Decorated function. """ if not self.scope: # from pycompss.api.dummy.opencl import opencl as dummy_opencl # d_ocl = dummy_opencl(self.args, self.kwargs) # return d_ocl.__call__(func) raise Exception(not_in_pycompss("opencl")) if context.in_master(): # master code mod = inspect.getmodule(func) self.module = mod.__name__ # not func.__module__ if (self.module == '__main__' or self.module == 'pycompss.runtime.launch'): # The module where the function is defined was run as __main__, # we need to find out the real module name. # path = mod.__file__ # dirs = mod.__file__.split(os.sep) # file_name = os.path.splitext( # os.path.basename(mod.__file__))[0] # Get the real module name from our launch.py variable path = getattr(mod, "APP_PATH") dirs = path.split(os.path.sep) file_name = os.path.splitext(os.path.basename(path))[0] mod_name = file_name i = len(dirs) - 1 while i > 0: new_l = len(path) - (len(dirs[i]) + 1) path = path[0:new_l] if "__init__.py" in os.listdir(path): # directory is a package i -= 1 mod_name = dirs[i] + '.' + mod_name else: break self.module = mod_name # Include the registering info related to @opencl # Retrieve the base core_element established at @task decorator from pycompss.api.task import current_core_element as cce if not self.registered: self.registered = True # Update the core element information with the mpi information cce.set_impl_type("OPENCL") kernel = self.kwargs['kernel'] if 'working_dir' in self.kwargs: working_dir = self.kwargs['working_dir'] elif 'workingDir' in self.kwargs: working_dir = self.kwargs['workingDir'] else: working_dir = '[unassigned]' # Empty or '[unassigned]' impl_signature = 'OPENCL.' + kernel cce.set_impl_signature(impl_signature) impl_args = [kernel, working_dir] cce.set_impl_type_args(impl_args) func.__to_register__ = cce else: # worker code pass @wraps(func) def opencl_f(*args, **kwargs): # This is executed only when called. if __debug__: logger.debug("Executing opencl_f wrapper.") if len(args) > 0: # The 'self' for a method function is passed as args[0] slf = args[0] # Replace and store the attributes saved = {} for k, v in self.kwargs.items(): if hasattr(slf, k): saved[k] = getattr(slf, k) setattr(slf, k, v) # Call the method import pycompss.api.task as t t.prepend_strings = False ret = func(*args, **kwargs) t.prepend_strings = True if len(args) > 0: # Put things back for k, v in saved.items(): setattr(slf, k, v) return ret opencl_f.__doc__ = func.__doc__ return opencl_f
def decaf_f(*args, **kwargs): if not self.scope: # from pycompss.api.dummy.decaf import decaf as dummy_decaf # d_d = dummy_decaf(self.args, self.kwargs) # return d_d.__call__(func) raise Exception(not_in_pycompss("decaf")) if context.in_master(): # master code mod = inspect.getmodule(func) self.module = mod.__name__ # not func.__module__ if (self.module == '__main__' or self.module == 'pycompss.runtime.launch'): # The module where the function is defined was run as # __main__, so we need to find out the real module name. # pat = mod.__file__ # dirs = mod.__file__.split(os.sep) # file_name = os.path.splitext( # os.path.basename(mod.__file__))[0] # Get the real module name from our launch.py variable path = getattr(mod, "APP_PATH") dirs = path.split(os.path.sep) file_name = os.path.splitext(os.path.basename(path))[0] mod_name = file_name i = len(dirs) - 1 while i > 0: new_l = len(path) - (len(dirs[i]) + 1) path = path[0:new_l] if "__init__.py" in os.listdir(path): # directory is a package i -= 1 mod_name = dirs[i] + '.' + mod_name else: break self.module = mod_name # Include the registering info related to @decaf # Retrieve the base core_element established at @task decorator if not self.registered: from pycompss.api.task import current_core_element as cce self.registered = True # Update the core element information with the @decaf # information cce.set_impl_type("DECAF") if 'working_dir' in self.kwargs: working_dir = self.kwargs['working_dir'] elif 'workingDir' in self.kwargs: working_dir = self.kwargs['workingDir'] else: working_dir = '[unassigned]' # Empty or '[unassigned]' if 'runner' in self.kwargs: runner = self.kwargs['runner'] else: runner = 'mpirun' if 'dfScript' in self.kwargs: df_script = self.kwargs['dfScript'] else: df_script = self.kwargs['df_script'] if 'df_executor' in self.kwargs: df_executor = self.kwargs['df_executor'] elif 'dfExecutor' in self.kwargs: df_executor = self.kwargs['dfExecutor'] else: df_executor = '[unassigned]' # Empty or '[unassigned]' if 'df_lib' in self.kwargs: df_lib = self.kwargs['df_lib'] elif 'dfLib' in self.kwargs: df_lib = self.kwargs['dfLib'] else: df_lib = '[unassigned]' # Empty or '[unassigned]' if 'fail_by_exit_value' in self.kwargs: fail_by_ev = self.kwargs['fail_by_exit_value'] if isinstance(fail_by_ev, bool): if fail_by_ev: fail_by_ev_str = 'true' else: fail_by_ev_str = 'false' elif isinstance(fail_by_ev, str): fail_by_ev_str = fail_by_ev else: raise Exception( "Incorrect format for fail_by_exit_value property. " + # noqa: E501 "It should be boolean or an environment variable" ) # noqa: E501 else: fail_by_ev_str = 'false' impl_signature = 'DECAF.' + df_script cce.set_impl_signature(impl_signature) impl_args = [ df_script, df_executor, df_lib, working_dir, runner, fail_by_ev_str ] cce.set_impl_type_args(impl_args) else: # worker code pass # This is executed only when called. if __debug__: logger.debug("Executing decaf_f wrapper.") # Set the computing_nodes variable in kwargs for its usage in # @task decorator kwargs['computing_nodes'] = self.kwargs['computing_nodes'] if len(args) > 0: # The 'self' for a method function is passed as args[0] slf = args[0] # Replace and store the attributes saved = {} for k, v in self.kwargs.items(): if hasattr(slf, k): saved[k] = getattr(slf, k) setattr(slf, k, v) # Call the method import pycompss.api.task as t t.prepend_strings = False ret = func(*args, **kwargs) t.prepend_strings = True if len(args) > 0: # Put things back for k, v in saved.items(): setattr(slf, k, v) return ret
def multinode_f(*args, **kwargs): if not self.scope: # from pycompss.api.dummy.compss import COMPSs as dummy_compss # d_m = dummy_compss(self.args, self.kwargs) # return d_m.__call__(func) raise Exception(not_in_pycompss("MultiNode")) if context.in_master(): # master code mod = inspect.getmodule(func) self.module = mod.__name__ # not func.__module__ if self.module == '__main__' or \ self.module == 'pycompss.runtime.launch': # The module where the function is defined was run as # __main__, so we need to find out the real module name. # Get the real module name from our launch.py variable path = getattr(mod, "APP_PATH") dirs = path.split(os.path.sep) file_name = os.path.splitext(os.path.basename(path))[0] mod_name = file_name i = len(dirs) - 1 while i > 0: new_l = len(path) - (len(dirs[i]) + 1) path = path[0:new_l] if "__init__.py" in os.listdir(path): # directory is a package i -= 1 mod_name = dirs[i] + '.' + mod_name else: break self.module = mod_name # Include the registering info related to @compss # Retrieve the base core_element established at @task decorator from pycompss.api.task import current_core_element as cce if not self.registered: self.registered = True # Update the core element information with the # @MultiNode information cce.set_impl_type("MULTI_NODE") # Signature and implementation args are set by the # @task decorator else: # worker code pass # This is executed only when called. if __debug__: logger.debug("Executing multinode_f wrapper.") # Set the computing_nodes variable in kwargs for its usage # in @task decorator kwargs['computing_nodes'] = self.kwargs['computing_nodes'] if len(args) > 0: # The 'self' for a method function is passed as args[0] slf = args[0] # Replace and store the attributes saved = {} for k, v in self.kwargs.items(): if hasattr(slf, k): saved[k] = getattr(slf, k) setattr(slf, k, v) # Call the method ret = func(*args, **kwargs) if len(args) > 0: # Put things back for k, v in saved.items(): setattr(slf, k, v) return ret
def mpi_f(*args, **kwargs): if not self.scope: # from pycompss.api.dummy.mpi import mpi as dummy_mpi # d_m = dummy_mpi(self.args, self.kwargs) # return d_m.__call__(func) raise Exception(not_in_pycompss("mpi")) if context.in_master(): # master code mod = inspect.getmodule(func) self.module = mod.__name__ # not func.__module__ if self.module == '__main__' or \ self.module == 'pycompss.runtime.launch': # The module where the function is defined was run as # __main__, so we need to find out the real module name. # path = mod.__file__ # dirs = mod.__file__.split(os.sep) # file_name = os.path.splitext( # os.path.basename(mod.__file__))[0] # Get the real module name from our launch.py variable path = getattr(mod, "APP_PATH") dirs = path.split(os.path.sep) file_name = os.path.splitext(os.path.basename(path))[0] mod_name = file_name i = len(dirs) - 1 while i > 0: new_l = len(path) - (len(dirs[i]) + 1) path = path[0:new_l] if "__init__.py" in os.listdir(path): # directory is a package i -= 1 mod_name = dirs[i] + '.' + mod_name else: break self.module = mod_name # Include the registering info related to @mpi # Retrieve the base core_element established at @task decorator from pycompss.api.task import current_core_element as cce if not self.registered: self.registered = True # Update the core element information with the @mpi # information if "binary" in self.kwargs: binary = self.kwargs['binary'] cce.set_impl_type("MPI") else: binary = "[unassigned]" cce.set_impl_type("PYTHON_MPI") self.task_type = "PYTHON_MPI" if 'working_dir' in self.kwargs: working_dir = self.kwargs['working_dir'] else: working_dir = '[unassigned]' # Empty or '[unassigned]' runner = self.kwargs['runner'] if 'flags' in self.kwargs: flags = self.kwargs['flags'] else: flags = '[unassigned]' # Empty or '[unassigned]' if 'scale_by_cu' in self.kwargs: scale_by_cu = self.kwargs['scale_by_cu'] if isinstance(scale_by_cu, bool): if scale_by_cu: scale_by_cu_str = 'true' else: scale_by_cu_str = 'false' elif isinstance(scale_by_cu, str): scale_by_cu_str = scale_by_cu else: raise Exception("Incorrect format for scale_by_cu property. " + # noqa: E501 "It should be boolean or an environment variable") # noqa: E501 else: scale_by_cu_str = 'false' if 'fail_by_exit_value' in self.kwargs: fail_by_ev = self.kwargs['fail_by_exit_value'] if isinstance(fail_by_ev, bool): if fail_by_ev: fail_by_ev_str = 'true' else: fail_by_ev_str = 'false' elif isinstance(fail_by_ev, str): fail_by_ev_str = fail_by_ev else: raise Exception("Incorrect format for fail_by_exit_value property. " + # noqa: E501 "It should be boolean or an environment variable") # noqa: E501 else: fail_by_ev_str = 'false' if binary == "[unassigned]": impl_signature = "MPI." else: impl_signature = 'MPI.' + \ str(self.kwargs['processes']) + \ "." + binary # Add information to CoreElement cce.set_impl_signature(impl_signature) impl_args = [binary, working_dir, runner, flags, scale_by_cu_str, fail_by_ev_str] cce.set_impl_type_args(impl_args) else: # worker code pass # This is executed only when called. if __debug__: logger.debug("Executing mpi_f wrapper.") # Set the computing_nodes variable in kwargs for its usage # in @task decorator kwargs['computing_nodes'] = self.kwargs['processes'] if len(args) > 0: # The 'self' for a method function is passed as args[0] slf = args[0] # Replace and store the attributes saved = {} for k, v in self.kwargs.items(): if hasattr(slf, k): saved[k] = getattr(slf, k) setattr(slf, k, v) # Call the method import pycompss.api.task as t if self.task_type == "PYTHON_MPI": t.prepend_strings = True else: t.prepend_strings = False ret = func(*args, **kwargs) t.prepend_strings = True if len(args) > 0: # Put things back for k, v in saved.items(): setattr(slf, k, v) return ret
def ompss_f(*args, **kwargs): if not self.scope: # from pycompss.api.dummy.ompss import ompss as dummy_ompss # d_o = dummy_ompss(self.args, self.kwargs) # return d_o.__call__(func) raise Exception(not_in_pycompss("ompss")) if context.in_master(): # master code mod = inspect.getmodule(func) self.module = mod.__name__ # not func.__module__ if (self.module == '__main__' or self.module == 'pycompss.runtime.launch'): # The module where the function is defined was run as # __main__, so we need to find out the real module name. # path = mod.__file__ # dirs = mod.__file__.split(os.sep) # file_name = os.path.splitext( # os.path.basename(mod.__file__))[0] # Get the real module name from our launch.py variable path = getattr(mod, "APP_PATH") dirs = path.split(os.path.sep) file_name = os.path.splitext(os.path.basename(path))[0] mod_name = file_name i = len(dirs) - 1 while i > 0: new_l = len(path) - (len(dirs[i]) + 1) path = path[0:new_l] if "__init__.py" in os.listdir(path): # directory is a package i -= 1 mod_name = dirs[i] + '.' + mod_name else: break self.module = mod_name # Include the registering info related to @ompss # Retrieve the base core_element established at @task decorator from pycompss.api.task import current_core_element as cce if not self.registered: self.registered = True # Update the core element information with the @ompss # information cce.set_impl_type("OMPSS") binary = self.kwargs['binary'] if 'working_dir' in self.kwargs: working_dir = self.kwargs['working_dir'] elif 'workingDir' in self.kwargs: working_dir = self.kwargs['workingDir'] else: working_dir = '[unassigned]' # Empty or '[unassigned]' impl_signature = 'OMPSS.' + binary cce.set_impl_signature(impl_signature) impl_args = [binary, working_dir] cce.set_impl_type_args(impl_args) else: # worker code pass # This is executed only when called. if __debug__: logger.debug("Executing ompss_f wrapper.") # Set the computing_nodes variable in kwargs for its usage # in @task decorator kwargs['computing_nodes'] = self.kwargs['computing_nodes'] if len(args) > 0: # The 'self' for a method function is passed as args[0] slf = args[0] # Replace and store the attributes saved = {} for k, v in self.kwargs.items(): if hasattr(slf, k): saved[k] = getattr(slf, k) setattr(slf, k, v) # Call the method import pycompss.api.task as t t.prepend_strings = False ret = func(*args, **kwargs) t.prepend_strings = True if len(args) > 0: # Put things back for k, v in saved.items(): setattr(slf, k, v) return ret
def io_f(*args, **kwargs): if not self.scope: # from pycompss.api.dummy.binary import binary as dummy_binary # d_b = dummy_binary(self.args, self.kwargs) # return d_b.__call__(func) raise Exception(not_in_pycompss("IO")) if context.in_master(): # master code mod = inspect.getmodule(func) self.module = mod.__name__ # not func.__module__ if (self.module == '__main__' or self.module == 'pycompss.runtime.launch'): # The module where the function is defined was run as # __main__, so we need to find out the real module name. # path = mod.__file__ # dirs = mod.__file__.split(os.sep) # file_name = os.path.splitext( # os.path.basename(mod.__file__))[0] # Get the real module name from our launch.py variable path = getattr(mod, "APP_PATH") dirs = path.split(os.path.sep) file_name = os.path.splitext(os.path.basename(path))[0] mod_name = file_name i = len(dirs) - 1 while i > 0: new_l = len(path) - (len(dirs[i]) + 1) path = path[0:new_l] if "__init__.py" in os.listdir(path): # directory is a package i -= 1 mod_name = dirs[i] + '.' + mod_name else: break self.module = mod_name # Include the registering info related to @IO if not self.registered: # Set as registered self.registered = True # Retrieve the base core_element established at @task # decorator from pycompss.api.task import current_core_element as cce # Update the core element information with the IO # argument information cce.set_impl_io(True) else: # worker code pass # This is executed only when called. if __debug__: logger.debug("Executing IO_f wrapper.") if len(args) > 0: # The 'self' for a method function is passed as args[0] slf = args[0] # Replace and store the attributes saved = {} for k, v in self.kwargs.items(): if hasattr(slf, k): saved[k] = getattr(slf, k) setattr(slf, k, v) # Call the method import pycompss.api.task as t t.prepend_strings = False ret = func(*args, **kwargs) t.prepend_strings = True if len(args) > 0: # Put things back for k, v in saved.items(): setattr(slf, k, v) return ret