def __configure_core_element__(self, kwargs, user_function): # type: (dict, ...) -> None """ Include the registering info related to @multinode. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Keyword arguments received from call. :param user_function: Decorated function. :return: None """ if __debug__: logger.debug("Configuring @multinode core element.") # Resolve @multinode specific parameters impl_type = "MULTI_NODE" if CORE_ELEMENT_KEY in kwargs: # Core element has already been created in a higher level decorator # (e.g. @constraint) kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) else: # @binary is in the top of the decorators stack. # Instantiate a new core element object, update it and include # it into kwarg core_element = CE() core_element.set_impl_type(impl_type) kwargs[CORE_ELEMENT_KEY] = core_element # Set as configured self.core_element_configured = True
def __configure_core_element__(self, kwargs): # type: (dict) -> None """ Include the registering info related to @constraint. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Current keyword arguments to be updated with the core element information. :return: None """ if __debug__: logger.debug("Configuring @constraint core element.") if CORE_ELEMENT_KEY in kwargs: # Core element has already been created in a higher level decorator # (e.g. @implements and @compss) kwargs[CORE_ELEMENT_KEY].set_impl_constraints(self.kwargs) else: # @constraint is in the top of the decorators stack. # Instantiate a new core element object, update it and include # it into kwarg core_element = CE() core_element.set_impl_constraints(self.kwargs) kwargs[CORE_ELEMENT_KEY] = core_element # Set as configured self.core_element_configured = True
def __configure_core_element__(self, kwargs): # noqa # type: (dict) -> None """ Include the registering info related to @IO. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Keyword arguments received from call. :return: None """ if __debug__: logger.debug("Configuring @IO core element.") # Resolve @io specific parameters if CORE_ELEMENT_KEY in kwargs: # Core element has already been created in a higher level decorator # (e.g. @constraint) kwargs[CORE_ELEMENT_KEY].set_impl_io(True) else: # @binary is in the top of the decorators stack. # Instantiate a new core element object, update it and include # it into kwarg core_element = CE() core_element.set_impl_io(True) kwargs[CORE_ELEMENT_KEY] = core_element
def test_multinode_existing_core_element(): context.set_pycompss_context(context.MASTER) my_multinode = MultiNode() f = my_multinode(dummy_function) # a higher level decorator would place the compss core element as follows: _ = f(compss_core_element=CE()) assert CORE_ELEMENT_KEY not in my_multinode.kwargs, \ "Core Element is not defined in kwargs dictionary."
def test_decaf_existing_core_element(): context.set_pycompss_context(context.MASTER) my_decaf = Decaf(df_script="date") f = my_decaf(dummy_function) # a higher level decorator would place the compss core element as follows: _ = f(compss_core_element=CE()) assert CORE_ELEMENT_KEY not in my_decaf.kwargs, \ "Core Element is not defined in kwargs dictionary."
def software_f(*args, **kwargs): # type: (*typing.Any, **typing.Any) -> typing.Any if not self.scope or not context.in_master(): # Execute the software as with PyCOMPSs so that sequential # execution performs as parallel. # To disable: raise Exception(not_in_pycompss(BINARY)) return user_function(*args, **kwargs) if __debug__: logger.debug("Executing software_f wrapper.") if self.constraints is not None: core_element = CE() core_element.set_impl_constraints(self.constraints) kwargs[CORE_ELEMENT_KEY] = core_element if self.container is not None: _func = str(user_function.__name__) impl_type = IMPL_CONTAINER impl_signature = '.'.join((impl_type, _func)) ce = kwargs.get(CORE_ELEMENT_KEY, CE()) impl_args = [self.container[ENGINE], # engine self.container[IMAGE], # image UNASSIGNED, # internal_type UNASSIGNED, # internal_binary UNASSIGNED, # internal_func UNASSIGNED, # working_dir UNASSIGNED] # fail_by_ev ce.set_impl_type(impl_type) ce.set_impl_signature(impl_signature) ce.set_impl_type_args(impl_args) kwargs[CORE_ELEMENT_KEY] = ce if self.decor: decorator = self.decor def decor_f(): def f(): ret = decorator(**self.config_args) return ret(user_function)(*args, **kwargs) return f() return decor_f() else: # It's a PyCOMPSs task with only @task and @software decorators return user_function(*args, **kwargs)
def test_binary_existing_core_element(): context.set_pycompss_context(context.MASTER) my_bin = Binary(binary="date") f = my_bin(dummy_function) # a higher level decorator would place the compss core element as follows: _ = f(compss_core_element=CE()) assert (CORE_ELEMENT_KEY not in my_bin.kwargs ), "Core Element is not defined in kwargs dictionary."
def test_container_existing_core_element(): context.set_pycompss_context(context.MASTER) my_bin = Container(engine="docker", image="dummy") f = my_bin(dummy_function) # a higher level decorator would place the compss core element as follows: _ = f(compss_core_element=CE()) assert CORE_ELEMENT_KEY not in my_bin.kwargs, \ "Core Element is not defined in kwargs dictionary."
def test_constraint_existing_core_element(): context.set_pycompss_context(context.MASTER) my_constraint = Constraint() f = my_constraint(dummy_function) # a higher level decorator would place the compss core element as follows: _ = f(compss_core_element=CE()) context.set_pycompss_context(context.OUT_OF_SCOPE) assert (CORE_ELEMENT_KEY not in my_constraint.kwargs ), "Core Element is not defined in kwargs dictionary."
def __configure_core_element__(self, kwargs, user_function): # type: (dict, str) -> None """ Include the registering info related to @container. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Keyword arguments received from call. :param user_function: Decorated function. :return: None """ if __debug__: logger.debug("Configuring @container core element.") # Resolve @container (mandatory) specific parameters _engine = self.kwargs['engine'] _image = self.kwargs['image'] _func = str(user_function.__name__) # Type and signature impl_type = "CONTAINER" impl_signature = '.'.join((impl_type, _func)) unassigned = "[unassigned]" impl_args = [ _engine, # engine _image, # image unassigned, # internal_type unassigned, # internal_binary unassigned, # internal_func unassigned, # working_dir unassigned ] # fail_by_ev if CORE_ELEMENT_KEY in kwargs: # Core element has already been created in a higher level decorator # (e.g. @constraint) kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) else: # @container is in the top of the decorators stack. # Instantiate a new core element object, update it and include # it into kwarg core_element = CE() core_element.set_impl_type(impl_type) core_element.set_impl_signature(impl_signature) core_element.set_impl_type_args(impl_args) kwargs[CORE_ELEMENT_KEY] = core_element # Set as configured self.core_element_configured = True
def test_implement_existing_core_element(): context.set_pycompss_context(context.MASTER) my_implementation = Implement(source_class="s_class", method="s_method") # Hack to mimic registered my_implementation.first_register = True f = my_implementation(dummy_function) # a higher level decorator would place the compss core element as follows: _ = f(compss_core_element=CE()) context.set_pycompss_context(context.OUT_OF_SCOPE) assert (CORE_ELEMENT_KEY not in my_implementation.kwargs ), "Core Element is not defined in kwargs dictionary."
def __configure_core_element__(self, kwargs): # type: (dict) -> None """ Include the registering info related to @mpmd_mpi. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Keyword arguments received from call. :return: None """ if __debug__: logger.debug("Configuring @mpmd_mpi core element.") # Resolve @mpmd_mpi specific parameters impl_type = "MPMDMPI" runner = self.kwargs[RUNNER] # Resolve the working directory resolve_working_dir(self.kwargs) # Resolve the fail by exit value resolve_fail_by_exit_value(self.kwargs) ppn = str(self.kwargs.get(PROCESSES_PER_NODE, 1)) impl_signature = '.'.join((impl_type, str(ppn))) prog_params = self.__get_programs_params__() impl_args = [ runner, self.kwargs[WORKING_DIR], ppn, self.kwargs[FAIL_BY_EXIT_VALUE] ] impl_args.extend(prog_params) if CORE_ELEMENT_KEY in kwargs: kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) else: core_element = CE() core_element.set_impl_type(impl_type) core_element.set_impl_signature(impl_signature) core_element.set_impl_type_args(impl_args) kwargs[CORE_ELEMENT_KEY] = core_element # Set as configured self.core_element_configured = True
def __configure_core_element__(self, kwargs): # type: (dict) -> None """ Include the registering info related to @ompss. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Keyword arguments received from call. :return: None """ if __debug__: logger.debug("Configuring @ompss core element.") # Resolve @ompss specific parameters binary = self.kwargs['binary'] # Resolve the working directory self.__resolve_working_dir__() # Resolve the fail by exit value self.__resolve_fail_by_exit_value__() impl_type = "OMPSS" impl_signature = "".join(("OMPSS.", binary)) impl_args = [ binary, self.kwargs['working_dir'], self.kwargs['fail_by_exit_value'] ] if CORE_ELEMENT_KEY in kwargs: # Core element has already been created in a higher level decorator # (e.g. @constraint) kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) else: # @binary is in the top of the decorators stack. # Instantiate a new core element object, update it and include # it into kwarg core_element = CE() core_element.set_impl_type(impl_type) core_element.set_impl_signature(impl_signature) core_element.set_impl_type_args(impl_args) kwargs[CORE_ELEMENT_KEY] = core_element # Set as configured self.core_element_configured = True
def __configure_core_element__(self, kwargs, user_function): # type: (dict, ...) -> None """ Include the registering info related to @opencl. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Keyword arguments received from call. :param user_function: Decorated function. :return: None """ if __debug__: logger.debug("Configuring @opencl core element.") # Resolve @opencl specific parameters kernel = self.kwargs['kernel'] # Resolve the working directory self.__resolve_working_dir__() impl_type = 'OPENCL' impl_signature = '.'.join((impl_type, kernel)) impl_args = [kernel, self.kwargs['working_dir']] if CORE_ELEMENT_KEY in kwargs: # Core element has already been created in a higher level decorator # (e.g. @constraint) kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) else: # @binary is in the top of the decorators stack. # Instantiate a new core element object, update it and include # it into kwarg core_element = CE() core_element.set_impl_type(impl_type) core_element.set_impl_signature(impl_signature) core_element.set_impl_type_args(impl_args) kwargs[CORE_ELEMENT_KEY] = core_element # Set as configured self.core_element_configured = True
def __configure_core_element__(self, kwargs): # type: (dict) -> None """ Include the registering info related to @on_failure. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Current keyword arguments to be updated with the core element information. :return: None """ if __debug__: logger.debug("Configuring @on_failure core element.") if CORE_ELEMENT_KEY not in kwargs: # @on_failure is in the top of the decorators stack. # Instantiate a new core element object, update it and include # it into kwarg kwargs[CORE_ELEMENT_KEY] = CE() # Set as configured self.core_element_configured = True
def __configure_core_element__(self, kwargs, user_function): # type: (dict, ...) -> None """ Include the registering info related to @implement. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Keyword arguments received from call. :param user_function: Decorated function. :return: None """ if __debug__: logger.debug("Configuring @implement core element.") # Resolve @implement specific parameters if 'sourceClass' in self.kwargs: another_class = self.kwargs['sourceClass'] self.kwargs['source_class'] = self.kwargs.pop('sourceClass') else: another_class = self.kwargs['source_class'] another_method = self.kwargs['method'] ce_signature = '.'.join((another_class, another_method)) impl_type = "METHOD" # impl_args = [another_class, another_method] - set by @task if CORE_ELEMENT_KEY in kwargs: # Core element has already been created in a higher level decorator # (e.g. @constraint) kwargs[CORE_ELEMENT_KEY].set_ce_signature(ce_signature) kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) # @task sets the implementation type arguments # kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) else: # @implement is in the top of the decorators stack. # Instantiate a new core element object, update it and include # it into kwarg core_element = CE() core_element.set_ce_signature(ce_signature) core_element.set_impl_type(impl_type) # @task sets the implementation type arguments # core_element.set_impl_type_args(impl_args) kwargs[CORE_ELEMENT_KEY] = core_element # Set as configured self.core_element_configured = True
def __configure_core_element__(self, kwargs): # type: (dict) -> None """ Include the registering info related to @http. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Keyword arguments received from call. :return: None """ if __debug__: logger.debug("Configuring @http core element.") impl_type = "HTTP" impl_args = [ self.kwargs["service_name"], self.kwargs["resource"], self.kwargs["request"], self.kwargs.get("payload", '#'), self.kwargs.get("payload_type", "application/json"), self.kwargs.get("produces", '#'), self.kwargs.get("updates", '#') ] if CORE_ELEMENT_KEY in kwargs: # Core element has already been created in a higher level decorator # (e.g. @constraint) kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) else: # @binary is in the top of the decorators stack. # Instantiate a new core element object, update it and include # it into kwarg core_element = CE() core_element.set_impl_type(impl_type) core_element.set_impl_type_args(impl_args) kwargs[CORE_ELEMENT_KEY] = core_element # Set as configured self.core_element_configured = True
def __configure_core_element__(self, kwargs, user_function): # type: (dict, ...) -> None """ Include the registering info related to @binary. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Keyword arguments received from call. :param user_function: Decorated function. :return: None """ if __debug__: logger.debug("Configuring @binary core element.") # Resolve the working directory self.__resolve_working_dir__() _working_dir = self.kwargs['working_dir'] # Resolve the fail by exit value self.__resolve_fail_by_exit_value__() _fail_by_ev = self.kwargs['fail_by_exit_value'] # Resolve binary _binary = str(self.kwargs['binary']) if CORE_ELEMENT_KEY in kwargs and \ kwargs[CORE_ELEMENT_KEY].get_impl_type() == 'CONTAINER': # @container decorator sits on top of @binary decorator # Note: impl_type and impl_signature are NOT modified # ('CONTAINER' and 'CONTAINER.function_name' respectively) impl_args = kwargs[CORE_ELEMENT_KEY].get_impl_type_args() _engine = impl_args[0] _image = impl_args[1] impl_args = [ _engine, # engine _image, # image 'CET_BINARY', # internal_type _binary, # internal_binary '[unassigned]', # internal_func _working_dir, # working_dir _fail_by_ev ] # fail_by_ev kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) else: # @container decorator does NOT sit on top of @binary decorator _binary = str(self.kwargs['binary']) impl_type = 'BINARY' impl_signature = '.'.join((impl_type, _binary)) impl_args = [ _binary, # internal_binary _working_dir, # working_dir _fail_by_ev ] # fail_by_ev if CORE_ELEMENT_KEY in kwargs: # Core element has already been created in a higher level # decorator (e.g. @constraint) kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) else: # @binary is in the top of the decorators stack. # Instantiate a new core element object, update it and include # it into kwarg core_element = CE() core_element.set_impl_type(impl_type) core_element.set_impl_signature(impl_signature) core_element.set_impl_type_args(impl_args) kwargs[CORE_ELEMENT_KEY] = core_element # Set as configured self.core_element_configured = True
def __configure_core_element__(self, kwargs, user_function): # type: (dict, ...) -> None """ Include the registering info related to @decaf. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Keyword arguments received from call. :param user_function: Decorated function. :return: None """ if __debug__: logger.debug("Configuring @decaf core element.") # Resolve @decaf specific parameters if 'runner' in self.kwargs: runner = self.kwargs['runner'] else: runner = 'mpirun' if 'dfScript' in self.kwargs: df_script = self.kwargs['dfScript'] else: df_script = self.kwargs['df_script'] if 'df_executor' in self.kwargs: df_executor = self.kwargs['df_executor'] elif 'dfExecutor' in self.kwargs: df_executor = self.kwargs['dfExecutor'] else: df_executor = '[unassigned]' # Empty or '[unassigned]' if 'df_lib' in self.kwargs: df_lib = self.kwargs['df_lib'] elif 'dfLib' in self.kwargs: df_lib = self.kwargs['dfLib'] else: df_lib = '[unassigned]' # Empty or '[unassigned]' # Resolve the working directory self.__resolve_working_dir__() # Resolve the fail by exit value self.__resolve_fail_by_exit_value__() impl_type = 'DECAF' impl_signature = '.'.join((impl_type, df_script)) impl_args = [ df_script, df_executor, df_lib, self.kwargs['working_dir'], runner, self.kwargs['fail_by_exit_value'] ] if CORE_ELEMENT_KEY in kwargs: # Core element has already been created in a higher level decorator # (e.g. @constraint) kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) else: # @binary is in the top of the decorators stack. # Instantiate a new core element object, update it and include # it into kwarg core_element = CE() core_element.set_impl_type(impl_type) core_element.set_impl_signature(impl_signature) core_element.set_impl_type_args(impl_args) kwargs[CORE_ELEMENT_KEY] = core_element # Set as configured self.core_element_configured = True
def __configure_core_element__(self, kwargs, user_function): # type: (dict, ...) -> None """ Include the registering info related to @compss. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Keyword arguments received from call. :param user_function: Decorated function. :return: None """ if __debug__: logger.debug("Configuring @compss core element.") # Resolve @compss specific parameters if 'runcompss' in self.kwargs: runcompss = self.kwargs['runcompss'] else: runcompss = '[unassigned]' # Empty or '[unassigned]' if 'flags' in self.kwargs: flags = self.kwargs['flags'] else: flags = '[unassigned]' # Empty or '[unassigned]' if 'worker_in_master' in self.kwargs: worker_in_master = self.kwargs['worker_in_master'] elif 'workerInMaster' in self.kwargs: worker_in_master = self.kwargs['workerInMaster'] else: worker_in_master = 'true' # Empty or '[unassigned]' if 'appName' in self.kwargs: app_name = self.kwargs['appName'] else: app_name = self.kwargs['app_name'] # Resolve the working directory self.__resolve_working_dir__() # Resolve the fail by exit value self.__resolve_fail_by_exit_value__() impl_type = 'COMPSs' impl_signature = '.'.join((impl_type, app_name)) impl_args = [runcompss, flags, app_name, worker_in_master, self.kwargs['working_dir'], self.kwargs['fail_by_exit_value']] if CORE_ELEMENT_KEY in kwargs: # Core element has already been created in a higher level decorator # (e.g. @constraint) kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) else: # @binary is in the top of the decorators stack. # Instantiate a new core element object, update it and include # it into kwarg core_element = CE() core_element.set_impl_type(impl_type) core_element.set_impl_signature(impl_signature) core_element.set_impl_type_args(impl_args) kwargs[CORE_ELEMENT_KEY] = core_element # Set as configured self.core_element_configured = True
def __configure_core_element__(self, kwargs): # type: (dict) -> None """ Include the registering info related to @binary. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Keyword arguments received from call. :return: None """ if __debug__: logger.debug("Configuring @binary core element.") # Resolve the working directory resolve_working_dir(self.kwargs) _working_dir = self.kwargs[WORKING_DIR] # Resolve the fail by exit value resolve_fail_by_exit_value(self.kwargs) _fail_by_ev = self.kwargs[FAIL_BY_EXIT_VALUE] # Resolve binary _binary = str(self.kwargs[BINARY]) if CORE_ELEMENT_KEY in kwargs and \ kwargs[CORE_ELEMENT_KEY].get_impl_type() == IMPL_CONTAINER: # @container decorator sits on top of @binary decorator # Note: impl_type and impl_signature are NOT modified # (IMPL_CONTAINER and "CONTAINER.function_name" respectively) impl_args = kwargs[CORE_ELEMENT_KEY].get_impl_type_args() _engine = impl_args[0] _image = impl_args[1] impl_args = [ _engine, # engine _image, # image IMPL_CET_BINARY, # internal_type _binary, # internal_binary UNASSIGNED, # internal_func _working_dir, # working_dir _fail_by_ev ] # fail_by_ev kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) else: # @container decorator does NOT sit on top of @binary decorator _binary = str(self.kwargs[BINARY]) impl_type = IMPL_BINARY impl_signature = ".".join((impl_type, _binary)) impl_args = [ _binary, # internal_binary _working_dir, # working_dir self.kwargs.get('params', UNASSIGNED), # params _fail_by_ev ] # fail_by_ev if CORE_ELEMENT_KEY in kwargs: # Core element has already been created in a higher level # decorator (e.g. @constraint) kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) else: # @binary is in the top of the decorators stack. # Instantiate a new core element object, update it and include # it into kwarg core_element = CE() core_element.set_impl_type(impl_type) core_element.set_impl_signature(impl_signature) core_element.set_impl_type_args(impl_args) kwargs[CORE_ELEMENT_KEY] = core_element # Set as configured self.core_element_configured = True
def __configure_core_element__(self, kwargs): # type: (dict) -> None """ Include the registering info related to @compss. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Keyword arguments received from call. :return: None """ if __debug__: logger.debug("Configuring @compss core element.") # Resolve @compss specific parameters if RUNCOMPSS in self.kwargs: runcompss = self.kwargs[RUNCOMPSS] else: runcompss = UNASSIGNED # Empty or UNASSIGNED if FLAGS in self.kwargs: flags = self.kwargs[FLAGS] else: flags = UNASSIGNED # Empty or UNASSIGNED if WORKER_IN_MASTER in self.kwargs: worker_in_master = self.kwargs[WORKER_IN_MASTER] elif LEGACY_WORKER_IN_MASTER in self.kwargs: worker_in_master = self.kwargs[LEGACY_WORKER_IN_MASTER] else: worker_in_master = "true" # Empty or UNASSIGNED if LEGACY_APP_NAME in self.kwargs: app_name = self.kwargs[LEGACY_APP_NAME] else: app_name = self.kwargs[APP_NAME] # Resolve the working directory resolve_working_dir(self.kwargs) # Resolve the fail by exit value resolve_fail_by_exit_value(self.kwargs) impl_type = IMPL_COMPSs impl_signature = ".".join((impl_type, app_name)) impl_args = [ runcompss, flags, app_name, worker_in_master, self.kwargs[WORKING_DIR], self.kwargs[FAIL_BY_EXIT_VALUE] ] if CORE_ELEMENT_KEY in kwargs: # Core element has already been created in a higher level decorator # (e.g. @constraint) kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) else: # @binary is in the top of the decorators stack. # Instantiate a new core element object, update it and include # it into kwarg core_element = CE() core_element.set_impl_type(impl_type) core_element.set_impl_signature(impl_signature) core_element.set_impl_type_args(impl_args) kwargs[CORE_ELEMENT_KEY] = core_element # Set as configured self.core_element_configured = True
def __configure_core_element__(self, kwargs, user_function): # type: (dict, ...) -> None """ Include the registering info related to @mpi. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Keyword arguments received from call. :param user_function: Decorated function. :return: None """ if __debug__: logger.debug("Configuring @mpi core element.") # Resolve @mpi specific parameters if "binary" in self.kwargs: binary = self.kwargs['binary'] impl_type = "MPI" else: binary = "[unassigned]" impl_type = "PYTHON_MPI" self.task_type = impl_type runner = self.kwargs['runner'] if 'flags' in self.kwargs: flags = self.kwargs['flags'] else: flags = '[unassigned]' # Empty or '[unassigned]' # Check if scale by cu is defined scale_by_cu_str = self.__resolve_scale_by_cu__() # Resolve the working directory self.__resolve_working_dir__() # Resolve the fail by exit value self.__resolve_fail_by_exit_value__() # Resolve parameter collection layout collection_layout_params = self.__resolve_collection_layout_params__() if "processes" in self.kwargs: proc = self.kwargs["processes"] elif "computing_nodes" in self.kwargs: proc = self.kwargs["computing_nodes"] elif "computingNodes" in self.kwargs: proc = self.kwargs["computingNodes"] else: proc = "1" if binary == "[unassigned]": impl_signature = impl_type + '.' else: impl_signature = '.'.join((impl_type, str(proc), binary)) impl_args = [ binary, self.kwargs['working_dir'], runner, flags, scale_by_cu_str, self.kwargs['fail_by_exit_value'] ] if impl_type == "PYTHON_MPI": impl_args = impl_args + collection_layout_params if CORE_ELEMENT_KEY in kwargs: # Core element has already been created in a higher level decorator # (e.g. @constraint) kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) else: # @binary is in the top of the decorators stack. # Instantiate a new core element object, update it and include # it into kwarg core_element = CE() core_element.set_impl_type(impl_type) core_element.set_impl_signature(impl_signature) core_element.set_impl_type_args(impl_args) kwargs[CORE_ELEMENT_KEY] = core_element # Set as configured self.core_element_configured = True
def __init__(self, *args, **kwargs): # noqa # type: (*typing.Any, **typing.Any) -> None """ Task constructor. This part is called in the decoration process, not as an explicit function call. We do two things here: a) Assign default values to unspecified fields. b) Transform the parameters from user friendly types (i.e Parameter.IN, etc) to a more convenient internal representation. :param args: Decorator positional parameters (ignored). :param kwargs: Decorator parameters. A task decorator has no positional arguments. """ self.task_type = IMPL_METHOD self.decorator_name = "".join(("@", Task.__name__.lower())) self.args = args self.kwargs = kwargs self.scope = context.in_pycompss() self.core_element = CE() self.core_element_configured = False # Set missing values to their default ones (step a) self.decorator_arguments = kwargs default_decorator_values = { "target_direction": parameter.INOUT, "returns": False, "cache_returns": True, "priority": False, "defaults": {}, "time_out": 0, "is_replicated": False, "is_distributed": False, "computing_nodes": 1, "is_reduce": False, "chunk_size": 0, "tracing_hook": False, "numba": False, # numba mode (jit, vectorize, guvectorize) "numba_flags": {}, # user defined extra numba flags "numba_signature": None, # vectorize and guvectorize signature "numba_declaration": None, # guvectorize declaration "varargs_type": parameter.IN, # Here for legacy purposes } # type: typing.Dict[str, typing.Any] for (key, value) in default_decorator_values.items(): if key not in self.decorator_arguments: self.decorator_arguments[key] = value # Give all parameters a unique instance for them (step b) # Note that when a user defines a task like # @task(a = IN, b = IN, c = INOUT) # both a and b point to the same IN object (the one from parameter.py) # Giving them a unique instance makes life easier in further steps for (key, value) in self.decorator_arguments.items(): # Not all decorator arguments are necessarily parameters # (see default_decorator_values) if is_param(value): self.decorator_arguments[key] = get_new_parameter(value.key) # Specific case when value is a dictionary # Use case example: # @binary(binary="ls") # @task(hide={Type: FILE_IN, Prefix: "--hide="}, # sort={Type: IN, Prefix: "--sort="}) # def myLs(flag, hide, sort): # pass # Transform this dictionary to a Parameter object if isinstance(value, dict): if key not in [ "numba", "numba_flags", "numba_signature", "numba_declaration" ]: # Perform user -> instance substitution # param = self.decorator_arguments[key][parameter.Type] # Replace the whole dict by a single parameter object self.decorator_arguments[key] = \ get_parameter_from_dictionary( self.decorator_arguments[key] ) else: # It is a reserved word that we need to keep the user # defined value (not a Parameter object) self.decorator_arguments[key] = value self.user_function = dummy_function # type: typing.Callable # Global variables common for all tasks of this kind self.registered = False self.signature = "" # Saved from the initial task self.interactive = False self.module = None # type: typing.Any self.function_arguments = tuple() # type: tuple self.function_name = "" self.module_name = "" self.function_type = -1 self.class_name = "" self.hints = tuple() # type: tuple self.on_failure = "" self.defaults = dict() # type: dict
def __configure_core_element__(self, kwargs): # type: (dict) -> None """ Include the registering info related to @decaf. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Keyword arguments received from call. :return: None """ if __debug__: logger.debug("Configuring @decaf core element.") # Resolve @decaf specific parameters if RUNNER in self.kwargs: runner = self.kwargs[RUNNER] else: runner = "mpirun" if LEGACY_DF_SCRIPT in self.kwargs: df_script = self.kwargs[LEGACY_DF_SCRIPT] else: df_script = self.kwargs[DF_SCRIPT] if DF_EXECUTOR in self.kwargs: df_executor = self.kwargs[DF_EXECUTOR] elif LEGACY_DF_EXECUTOR in self.kwargs: df_executor = self.kwargs[LEGACY_DF_EXECUTOR] else: df_executor = UNASSIGNED # Empty or UNASSIGNED if DF_LIB in self.kwargs: df_lib = self.kwargs[DF_LIB] elif LEGACY_DF_LIB in self.kwargs: df_lib = self.kwargs[LEGACY_DF_LIB] else: df_lib = UNASSIGNED # Empty or UNASSIGNED # Resolve the working directory resolve_working_dir(self.kwargs) # Resolve the fail by exit value resolve_fail_by_exit_value(self.kwargs) impl_type = IMPL_DECAF impl_signature = ".".join((impl_type, df_script)) impl_args = [ df_script, df_executor, df_lib, self.kwargs[WORKING_DIR], runner, self.kwargs[FAIL_BY_EXIT_VALUE] ] if CORE_ELEMENT_KEY in kwargs: # Core element has already been created in a higher level decorator # (e.g. @constraint) kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) else: # @binary is in the top of the decorators stack. # Instantiate a new core element object, update it and include # it into kwarg core_element = CE() core_element.set_impl_type(impl_type) core_element.set_impl_signature(impl_signature) core_element.set_impl_type_args(impl_args) kwargs[CORE_ELEMENT_KEY] = core_element # Set as configured self.core_element_configured = True
def __configure_core_element__(self, kwargs): # type: (dict) -> None """ Include the registering info related to @mpi. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Keyword arguments received from call. :return: None """ if __debug__: logger.debug("Configuring @mpi core element.") # Resolve @mpi specific parameters if "binary" in self.kwargs: binary = self.kwargs['binary'] impl_type = "MPI" else: binary = "[unassigned]" impl_type = "PYTHON_MPI" self.task_type = impl_type runner = self.kwargs['runner'] if 'flags' in self.kwargs: flags = self.kwargs['flags'] else: flags = '[unassigned]' # Empty or '[unassigned]' if 'scale_by_cu' in self.kwargs: scale_by_cu = self.kwargs['scale_by_cu'] if isinstance(scale_by_cu, bool): if scale_by_cu: scale_by_cu_str = 'true' else: scale_by_cu_str = 'false' elif isinstance(scale_by_cu, str): scale_by_cu_str = scale_by_cu else: raise Exception( "Incorrect format for scale_by_cu property. " "It should be boolean or an environment variable" ) # noqa: E501 else: scale_by_cu_str = 'false' # Resolve the working directory self.__resolve_working_dir__() # Resolve the fail by exit value self.__resolve_fail_by_exit_value__() # Resolve parameter collection layout collection_layout_params = self.__resolve_collection_layout_params__() if binary == "[unassigned]": impl_signature = impl_type + '.' else: impl_signature = '.'.join( (impl_type, str(self.kwargs['processes']), binary)) impl_args = [ binary, self.kwargs['working_dir'], runner, flags, scale_by_cu_str, self.kwargs['fail_by_exit_value'] ] if impl_type == "PYTHON_MPI": impl_args = impl_args + collection_layout_params if CORE_ELEMENT_KEY in kwargs: # Core element has already been created in a higher level decorator # (e.g. @constraint) kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) else: # @binary is in the top of the decorators stack. # Instantiate a new core element object, update it and include # it into kwarg core_element = CE() core_element.set_impl_type(impl_type) core_element.set_impl_signature(impl_signature) core_element.set_impl_type_args(impl_args) kwargs[CORE_ELEMENT_KEY] = core_element # Set as configured self.core_element_configured = True
def test_core_element(): signature = "my_signature" impl_signature = "my_impl_signature" impl_constraints = "impl_constraints" impl_type = "impl_type" impl_io = "impl_io" impl_type_args = "impl_type_args" core_element = CE(signature, impl_signature, impl_constraints, impl_type, impl_io, impl_type_args) # Check signature result = core_element.get_ce_signature() assert result == signature, ERROR_SIGNATURE new_signature = "my_new_signature" core_element.set_ce_signature(new_signature) result = core_element.get_ce_signature() assert result == new_signature, ERROR_SIGNATURE # Check impl_signature result = core_element.get_impl_signature() assert result == impl_signature, ERROR_IMPL_SIGNATURE new_impl_signature = "my_new_impl_signature" core_element.set_impl_signature(new_impl_signature) result = core_element.get_impl_signature() assert result == new_impl_signature, ERROR_IMPL_SIGNATURE # Check impl_constraints result = core_element.get_impl_constraints() assert result == impl_constraints, ERROR_IMPL_CONSTRAINTS new_impl_constraints = {"my_new_impl_constraints": "value"} core_element.set_impl_constraints(new_impl_constraints) result = core_element.get_impl_constraints() assert result == new_impl_constraints, ERROR_IMPL_CONSTRAINTS # Check impl_type result = core_element.get_impl_type() assert result == impl_type, ERROR_IMPL_TYPE new_impl_type = "my_new_impl_type" core_element.set_impl_type(new_impl_type) result = core_element.get_impl_type() assert result == new_impl_type, ERROR_IMPL_TYPE # Check impl_io result = core_element.get_impl_io() assert result == impl_io, ERROR_IMPL_IO new_impl_io = "my_new_impl_io" core_element.set_impl_io(new_impl_io) result = core_element.get_impl_io() assert result == new_impl_io, ERROR_IMPL_IO # Check impl_type_args result = core_element.get_impl_type_args() assert result == impl_type_args, ERROR_IMPL_TYPE_ARGS new_impl_type_args = "my_new_impl_type_args" core_element.set_impl_type_args(new_impl_type_args) result = core_element.get_impl_type_args() assert result == new_impl_type_args, ERROR_IMPL_TYPE_ARGS # Check representation representation = core_element.__repr__() assert isinstance(representation, str), "ERROR: Received wrong representation type." # noqa: E501 expected = "CORE ELEMENT: \n" \ "\t - CE signature : my_new_signature\n" \ "\t - Impl. signature : my_new_impl_signature\n" \ "\t - Impl. constraints: my_new_impl_constraints:value;\n" \ "\t - Impl. type : my_new_impl_type\n" \ "\t - Impl. io : my_new_impl_io\n" \ "\t - Impl. type args : my_new_impl_type_args" assert representation == expected, "ERROR: Wrong representation." # Reset core_element.reset() # Check again representation representation = core_element.__repr__() assert isinstance(representation, str), "ERROR: Received wrong representation type." # noqa: E501 expected = "CORE ELEMENT: \n" \ "\t - CE signature : None\n" \ "\t - Impl. signature : None\n" \ "\t - Impl. constraints: None\n" \ "\t - Impl. type : None\n" \ "\t - Impl. io : None\n" \ "\t - Impl. type args : None" assert representation == expected, "ERROR: Wrong empty representation."
def __configure_core_element__(self, kwargs): # type: (dict) -> None """ Include the registering info related to @mpi. IMPORTANT! Updates self.kwargs[CORE_ELEMENT_KEY]. :param kwargs: Keyword arguments received from call. :return: None """ if __debug__: logger.debug("Configuring @mpi core element.") # Resolve @mpi specific parameters if BINARY in self.kwargs: binary = self.kwargs[BINARY] impl_type = IMPL_MPI else: binary = UNASSIGNED impl_type = IMPL_PYTHON_MPI self.task_type = impl_type runner = self.kwargs[RUNNER] if FLAGS in self.kwargs: flags = self.kwargs[FLAGS] else: flags = UNASSIGNED # Empty or UNASSIGNED # Check if scale by cu is defined scale_by_cu_str = self.__resolve_scale_by_cu__() # Resolve the working directory resolve_working_dir(self.kwargs) # Resolve the fail by exit value resolve_fail_by_exit_value(self.kwargs) # Resolve parameter collection layout collection_layout_params = self.__resolve_collection_layout_params__() if "processes" in self.kwargs: proc = self.kwargs["processes"] elif "computing_nodes" in self.kwargs: proc = self.kwargs["computing_nodes"] elif "computingNodes" in self.kwargs: proc = self.kwargs["computingNodes"] else: proc = "1" if "processes_per_node" in self.kwargs: ppn = str(self.kwargs["processes_per_node"]) else: ppn = "1" if binary == UNASSIGNED: impl_signature = impl_type + '.' else: impl_signature = '.'.join((impl_type, str(proc), binary)) impl_args = [ binary, self.kwargs[WORKING_DIR], runner, ppn, flags, scale_by_cu_str, self.kwargs.get("params", UNASSIGNED), self.kwargs[FAIL_BY_EXIT_VALUE] ] if impl_type == IMPL_PYTHON_MPI: impl_args = impl_args + collection_layout_params if CORE_ELEMENT_KEY in kwargs: # Core element has already been created in a higher level decorator # (e.g. @constraint) kwargs[CORE_ELEMENT_KEY].set_impl_type(impl_type) kwargs[CORE_ELEMENT_KEY].set_impl_signature(impl_signature) kwargs[CORE_ELEMENT_KEY].set_impl_type_args(impl_args) else: # @binary is in the top of the decorators stack. # Instantiate a new core element object, update it and include # it into kwarg core_element = CE() core_element.set_impl_type(impl_type) core_element.set_impl_signature(impl_signature) core_element.set_impl_type_args(impl_args) kwargs[CORE_ELEMENT_KEY] = core_element # Set as configured self.core_element_configured = True