def __init__(self, *args, **kwargs): """ Store arguments passed to the decorator. self = itself. args = not used. kwargs = dictionary with the given mpi parameters. :param args: Arguments :param kwargs: Keyword arguments """ self.task_type = "mpi" self.decorator_name = "".join(('@', MPI.__name__.lower())) super(MPI, self).__init__(self.decorator_name, *args, **kwargs) if self.scope: if __debug__: logger.debug("Init @mpi decorator...") # noqa TODO: Maybe add here the collection layout to avoid iterate twice per elements # Add <param_name>_layout params to SUPPORTED_ARGUMENTS for key in self.kwargs.keys(): if "_layout" in key: SUPPORTED_ARGUMENTS.add(key) # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), self.decorator_name)
def __init__(self, *args, **kwargs): # type: (*typing.Any, **typing.Any) -> None """ Store arguments passed to the decorator. self = itself. args = not used. kwargs = dictionary with the given mpi parameters. :param args: Arguments :param kwargs: Keyword arguments """ decorator_name = "".join(("@", MPMDMPI.__name__.lower())) # super(MPMDMPI, self).__init__(decorator_name, *args, **kwargs) self.decorator_name = decorator_name self.args = args self.kwargs = kwargs self.scope = context.in_pycompss() self.core_element = None # type: typing.Any self.core_element_configured = False # MPMD_MPI specific: self.task_type = "mpmd_mpi" self.processes = 0 if self.scope: if __debug__: logger.debug("Init @mpmd_mpi decorator...") # Add <param_name>_layout params to SUPPORTED_ARGUMENTS for key in self.kwargs.keys(): if "_layout" in key: SUPPORTED_ARGUMENTS.add(key) # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), self.decorator_name)
def __init__(self, *args, **kwargs): # type: (*typing.Any, **typing.Any) -> None """ Store arguments passed to the decorator # self = itself. # args = not used. # kwargs = dictionary with the given Reduce parameters :param args: Arguments :param kwargs: Keyword arguments """ decorator_name = "".join(("@", self.__class__.__name__.lower())) # super(self.__class__, self).__init__(decorator_name, *args, **kwargs) # Instantiate superclass explicitly to support mypy. pd = PyCOMPSsDecorator(decorator_name, *args, **kwargs) self.decorator_name = decorator_name self.args = args self.kwargs = kwargs self.scope = context.in_pycompss() self.core_element = None # type: typing.Any self.core_element_configured = False self.__configure_core_element__ = pd.__configure_core_element__ if self.scope: # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), decorator_name) # Get the computing nodes self.__process_reduction_params__()
def __init__(self, *args, **kwargs): # type: (*typing.Any, **typing.Any) -> None """ Store arguments passed to the decorator # self = itself. # args = not used. # kwargs = dictionary with the given constraints. :param args: Arguments :param kwargs: Keyword arguments """ decorator_name = "@" + Container.__name__.lower() # super(Container, self).__init__(decorator_name, *args, **kwargs) self.decorator_name = decorator_name self.args = args self.kwargs = kwargs self.scope = context.in_pycompss() self.core_element = None # type: typing.Any self.core_element_configured = False if self.scope: if __debug__: logger.debug("Init @container decorator...") # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), decorator_name)
def test_check_arguments_using_old_is_modifier(): from pycompss.util.arguments import check_arguments result = False mandatory_arguments = {"mandatory_argument_1", "mandatoryArgument2"} deprecated_arguments = {"deprecated_argument_1", "deprecated_argument_2"} supported_arguments = { "mandatory_argument_1", "mandatoryArgument2", "optional_argument" } argument_names = [ "mandatory_argument_1", "mandatoryArgument2", "isModifier" ] decorator = "Unittest" old_stderr = sys.stderr sys.stderr = my_stderr = StringIO() try: check_arguments(mandatory_arguments, deprecated_arguments, supported_arguments, argument_names, decorator) except PyCOMPSsException: sys.stderr = old_stderr assert "WARNING" not in my_stderr.getvalue(), ERROR_UNEXPECTED_WARNING assert "ERROR" in my_stderr.getvalue(), ERROR_MISSING_ERROR result = True assert result, ERROR_EXCEPTION
def __init__(self, *args, **kwargs): """ Store arguments passed to the decorator # self = itself. # args = not used. # kwargs = dictionary with the given implement parameters. :param args: Arguments :param kwargs: Keyword arguments """ self.args = args self.kwargs = kwargs self.registered = False self.first_register = False self.scope = context.in_pycompss() if self.scope: if __debug__: logger.debug("Init @implement decorator...") # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), "@implement")
def __init__(self, *args, **kwargs): # type: (*typing.Any, **typing.Any) -> None """ Store arguments passed to the decorator. self = itself. args = not used. kwargs = dictionary with the given http parameters. :param args: Arguments :param kwargs: Keyword arguments """ decorator_name = "".join(("@", HTTP.__name__.lower())) # super(HTTP, self).__init__(decorator_name, *args, **kwargs) self.decorator_name = decorator_name self.args = args self.kwargs = kwargs self.scope = context.in_pycompss() self.core_element = None # type: typing.Any self.core_element_configured = False self.task_type = "http" if self.scope: # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), decorator_name)
def __init__(self, *args, **kwargs): """ Store arguments passed to the decorator # self = itself. # args = not used. # kwargs = dictionary with the given constraints. :param args: Arguments :param kwargs: Keyword arguments """ self.args = args self.kwargs = kwargs self.scope = context.in_pycompss() self.registered = False # This enables the decorator to get info from the caller # (e.g. self.source_frame_info.filename or # self.source_frame_info.lineno) # self.source_frame_info = inspect.getframeinfo(inspect.stack()[1][0]) if self.scope: if __debug__: logger.debug("Init @binary decorator...") # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), "@binary")
def __init__(self, *args, **kwargs): # type: (*typing.Any, **typing.Any) -> None """ Store arguments passed to the decorator self = itself. args = not used. kwargs = dictionary with the given constraints. :param args: Arguments. :param kwargs: Keyword arguments. """ decorator_name = "".join(("@", Decaf.__name__.lower())) # super(Decaf, self).__init__(decorator_name, *args, **kwargs) self.decorator_name = decorator_name self.args = args self.kwargs = kwargs self.scope = context.in_pycompss() self.core_element = None # type: typing.Any self.core_element_configured = False if self.scope: # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), decorator_name) # Get the computing nodes process_computing_nodes(decorator_name, self.kwargs)
def __init__(self, *args, **kwargs): """ Store arguments passed to the decorator # self = itself. # args = not used. # kwargs = dictionary with the given ompss parameters :param args: Arguments :param kwargs: Keyword arguments """ self.args = args self.kwargs = kwargs self.registered = False self.scope = context.in_pycompss() if self.scope: if __debug__: logger.debug("Init @ompss decorator...") # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), "@ompss") # Get the computing nodes: This parameter will have to go down # until execution when invoked. if 'computing_nodes' not in self.kwargs and \ 'computingNodes' not in self.kwargs: self.kwargs['computing_nodes'] = 1 else: if 'computingNodes' in self.kwargs: self.kwargs['computing_nodes'] = \ self.kwargs.pop('computingNodes') computing_nodes = kwargs['computing_nodes'] if isinstance(computing_nodes, int): self.kwargs['computing_nodes'] = kwargs['computing_nodes'] elif isinstance(computing_nodes, str) and \ computing_nodes.strip().startswith('$'): env_var = computing_nodes.strip()[1:] # Remove $ if env_var.startswith('{'): env_var = env_var[1:-1] # remove brackets self.kwargs['computing_nodes'] = int(os.environ[env_var]) else: raise Exception("Wrong Computing Nodes value at" + " @ompss decorator.") if __debug__: logger.debug("This OMPSs task will have " + str(self.kwargs['computing_nodes']) + " computing nodes.") else: pass
def __init__(self, *args, **kwargs): """ Store arguments passed to the decorator. self = itself. args = not used. kwargs = dictionary with the given mpi parameters. :param args: Arguments :param kwargs: Keyword arguments """ self.task_type = "mpi" decorator_name = "".join(('@', self.__class__.__name__.lower())) super(self.__class__, self).__init__(decorator_name, *args, **kwargs) if self.scope: if __debug__: logger.debug("Init @mpi decorator...") layout_nums = 0 # TODO: Maybe add here the collection layout to avoid iterate twice per elements # Add <param_name>_layout params to SUPPORTED_ARGUMENTS for key in self.kwargs.keys(): if "_layout" in key: layout_nums += 1 SUPPORTED_ARGUMENTS.add(key) if layout_nums > 1: raise Exception( "More than one layout definition is not yet supported!") # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), decorator_name) # Get the computing nodes self.__process_computing_nodes__(decorator_name) # Set default value if it has not been defined if 'processes' not in self.kwargs: self.kwargs['processes'] = 1 # The processes parameter will have to go down until the execution is invoked. # WARNING: processes can be an int, a env string, a str with dynamic variable name. if __debug__: logger.debug("This MPI task will have " + str(self.kwargs['processes']) + " processes.") else: pass
def __init__(self, *args, **kwargs): """ Store arguments passed to the decorator. self = itself. args = not used. kwargs = dictionary with the given constraints. :param args: Arguments. :param kwargs: Keyword arguments. """ decorator_name = "".join(('@', self.__class__.__name__.lower())) super(self.__class__, self).__init__(decorator_name, *args, **kwargs) if self.scope: # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), decorator_name)
def __init__(self, *args, **kwargs): """ Store arguments passed to the decorator # self = itself. # args = not used. # kwargs = dictionary with the given mpi parameters :param args: Arguments :param kwargs: Keyword arguments """ self.args = args self.kwargs = kwargs self.registered = False self.scope = context.in_pycompss() self.task_type = "mpi" if self.scope: if __debug__: logger.debug("Init @mpi decorator...") # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), "@mpi") # Replace the legacy annotation if 'computingNodes' in self.kwargs: self.kwargs['processes'] = self.kwargs.pop('computingNodes') if 'computing_nodes' in self.kwargs: self.kwargs['processes'] = self.kwargs.pop('computing_nodes') # Set default value if it has not been defined if 'processes' not in self.kwargs: self.kwargs['processes'] = 1 # The processes parameter will have to go down until the execution # is invoked. # WARNING: processes can be an int, a env string, a str with # dynamic variable name. if __debug__: logger.debug("This MPI task will have " + str(self.kwargs['processes']) + " processes.") else: pass
def test_check_arguments_missing_mandatory_no_underscore(): from pycompss.util.arguments import check_arguments result = False mandatory_arguments = {"mandatory_argument_1", "mandatoryArgument2"} deprecated_arguments = {"deprecated_argument_1", "deprecated_argument_2"} supported_arguments = { "mandatory_argument_1", "mandatoryArgument2", "optional_argument" } argument_names = ["mandatory_argument_1"] decorator = "Unittest" try: check_arguments(mandatory_arguments, deprecated_arguments, supported_arguments, argument_names, decorator) except PyCOMPSsException: # This is ok result = True assert result, ERROR_EXCEPTION
def __init__(self, *args, **kwargs): """ Store arguments passed to the decorator # self = itself. # args = not used. # kwargs = dictionary with the given constraints. :param args: Arguments :param kwargs: Keyword arguments """ decorator_name = '@' + self.__class__.__name__.lower() super(self.__class__, self).__init__(decorator_name, *args, **kwargs) if self.scope: if __debug__: logger.debug("Init @container decorator...") # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), decorator_name)
def test_check_arguments_using_deprecated(): from pycompss.util.arguments import check_arguments mandatory_arguments = {"mandatory_argument_1", "mandatoryArgument2"} deprecated_arguments = {"deprecated_argument_1", "deprecated_argument_2"} supported_arguments = { "mandatory_argument_1", "mandatoryArgument2", "optional_argument" } argument_names = [ "mandatory_argument_1", "mandatoryArgument2", "deprecated_argument_1" ] decorator = "Unittest" old_stderr = sys.stderr sys.stderr = my_stderr = StringIO() check_arguments(mandatory_arguments, deprecated_arguments, supported_arguments, argument_names, decorator) sys.stderr = old_stderr assert "WARNING" in my_stderr.getvalue(), ERROR_MISSING_WARNING assert "ERROR" not in my_stderr.getvalue(), ERROR_UNEXPECTED_ERROR
def __init__(self, *args, **kwargs): """ Store arguments passed to the decorator # self = itself. # args = not used. # kwargs = dictionary with the given Reduce parameters :param args: Arguments :param kwargs: Keyword arguments """ decorator_name = "".join(('@', self.__class__.__name__.lower())) super(self.__class__, self).__init__(decorator_name, *args, **kwargs) if self.scope: # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), decorator_name) # Get the computing nodes self.__process_reduction_params__()
def __init__(self, *args, **kwargs): # type: (*typing.Any, **typing.Any) -> None """ Parse the config file and store the arguments that will be used later to wrap the "real" decorator. self = itself. args = not used. kwargs = dictionary with the given @software parameter (config_file). :param args: Arguments :param kwargs: Keyword arguments """ decorator_name = "".join(('@', Software.__name__.lower())) # super(Software, self).__init__(decorator_name, *args, **kwargs) self.task_type = None # type: typing.Any self.config_args = None # type: typing.Any self.decor = None # type: typing.Any self.constraints = None # type: typing.Any self.container = None # type: typing.Any self.decorator_name = decorator_name self.args = args self.kwargs = kwargs self.scope = context.in_pycompss() self.core_element = None # type: typing.Any self.core_element_configured = False if self.scope and context.in_master(): if __debug__: logger.debug("Init @software decorator..") # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), decorator_name) self.parse_config_file()
def __init__(self, *args, **kwargs): """ Store arguments passed to the decorator # self = itself. # args = not used. # kwargs = dictionary with the given COMPSs parameters :param args: Arguments :param kwargs: Keyword arguments """ self.args = args self.kwargs = kwargs self.registered = False self.scope = context.in_pycompss() if self.scope: if __debug__: logger.debug("Init @compss decorator...") # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), "@compss") # Get the computing nodes: This parameter will have to go down # until execution when invoked. if 'computing_nodes' not in self.kwargs and \ 'computingNodes' not in self.kwargs: self.kwargs['computing_nodes'] = 1 else: if 'computingNodes' in self.kwargs: self.kwargs['computing_nodes'] = \ self.kwargs.pop('computingNodes') computing_nodes = self.kwargs['computing_nodes'] if isinstance(computing_nodes, int): # Nothing to do pass elif isinstance(computing_nodes, str): # Check if it is an environment variable to be loaded if computing_nodes.strip().startswith('$'): # Computing nodes is an ENV variable, load it env_var = computing_nodes.strip()[1:] # Remove $ if env_var.startswith('{'): env_var = env_var[1:-1] # remove brackets try: self.kwargs['computing_nodes'] = \ int(os.environ[env_var]) except ValueError: raise Exception( cast_env_to_int_error('ComputingNodes')) else: # ComputingNodes is in string form, cast it try: self.kwargs['computing_nodes'] = \ int(computing_nodes) except ValueError: raise Exception( cast_string_to_int_error('ComputingNodes')) else: raise Exception("ERROR: Wrong Computing Nodes value at" + " @COMPSs decorator.") if __debug__: logger.debug("This COMPSs task will have " + str(self.kwargs['computing_nodes']) + " computing nodes.") else: pass