def test_in_pycompss_context(): from pycompss.util.context import in_pycompss from pycompss.util.context import set_pycompss_context set_pycompss_context(MASTER) master_context = in_pycompss() set_pycompss_context(WORKER) worker_context = in_pycompss() assert master_context is True, MASTER_CONTEXT_ERROR assert worker_context is True, WORKER_CONTEXT_ERROR set_pycompss_context(OUT_OF_SCOPE)
def local(input_function): """ Local decorator. :param input_function: Input function. :return: Wrapped function. """ if not context.in_pycompss(): # Return dummy local decorator def wrapped_function(*args, **kwargs): return input_function(*args, **kwargs) return wrapped_function else: def sync_if_needed(obj): # type: (object) -> None if OT_is_obj_pending_to_synchronize(obj): new_val = compss_wait_on(obj) replace(obj, new_val) def wrapped_function(*args, **kwargs): gc.collect() _args = [] _kwargs = {} for arg in args: sync_if_needed(arg) _args.append(arg) for (key, value) in kwargs.items(): sync_if_needed(value) _kwargs[key] = value return input_function(*_args, **_kwargs) return wrapped_function
def __init__(self, *args, **kwargs): # type: (*typing.Any, **typing.Any) -> None """ Store arguments passed to the decorator. self = itself. args = not used. kwargs = dictionary with the given mpi parameters. :param args: Arguments :param kwargs: Keyword arguments """ decorator_name = "".join(("@", MPMDMPI.__name__.lower())) # super(MPMDMPI, self).__init__(decorator_name, *args, **kwargs) self.decorator_name = decorator_name self.args = args self.kwargs = kwargs self.scope = context.in_pycompss() self.core_element = None # type: typing.Any self.core_element_configured = False # MPMD_MPI specific: self.task_type = "mpmd_mpi" self.processes = 0 if self.scope: if __debug__: logger.debug("Init @mpmd_mpi decorator...") # Add <param_name>_layout params to SUPPORTED_ARGUMENTS for key in self.kwargs.keys(): if "_layout" in key: SUPPORTED_ARGUMENTS.add(key) # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), self.decorator_name)
def __init__(self, *args, **kwargs): # type: (*typing.Any, **typing.Any) -> None """ Store arguments passed to the decorator # self = itself. # args = not used. # kwargs = dictionary with the given constraints. :param args: Arguments :param kwargs: Keyword arguments """ decorator_name = "@" + Container.__name__.lower() # super(Container, self).__init__(decorator_name, *args, **kwargs) self.decorator_name = decorator_name self.args = args self.kwargs = kwargs self.scope = context.in_pycompss() self.core_element = None # type: typing.Any self.core_element_configured = False if self.scope: if __debug__: logger.debug("Init @container decorator...") # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), decorator_name)
def __init__(self, *args, **kwargs): # type: (*typing.Any, **typing.Any) -> None """ Store arguments passed to the decorator # self = itself. # args = not used. # kwargs = dictionary with the given Reduce parameters :param args: Arguments :param kwargs: Keyword arguments """ decorator_name = "".join(("@", self.__class__.__name__.lower())) # super(self.__class__, self).__init__(decorator_name, *args, **kwargs) # Instantiate superclass explicitly to support mypy. pd = PyCOMPSsDecorator(decorator_name, *args, **kwargs) self.decorator_name = decorator_name self.args = args self.kwargs = kwargs self.scope = context.in_pycompss() self.core_element = None # type: typing.Any self.core_element_configured = False self.__configure_core_element__ = pd.__configure_core_element__ if self.scope: # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), decorator_name) # Get the computing nodes self.__process_reduction_params__()
def __init__(self, *args, **kwargs): # type: (*typing.Any, **typing.Any) -> None """ Store arguments passed to the decorator. self = itself. args = not used. kwargs = dictionary with the given on_failure. :param args: Arguments. :param kwargs: Keyword arguments. """ decorator_name = "".join(("@", OnFailure.__name__.lower())) # super(OnFailure, self).__init__(decorator_name, *args, **kwargs) self.decorator_name = decorator_name self.args = args self.kwargs = kwargs self.scope = context.in_pycompss() self.core_element = None # type: typing.Any self.core_element_configured = False if self.scope: # Check the arguments check_mandatory_arguments(MANDATORY_ARGUMENTS, list(kwargs.keys()), decorator_name) # Save the parameters into self so that they can be accessed when # the task fails and the action needs to be taken self.on_failure_action = kwargs.pop(MANAGEMENT) # Check supported management values if self.on_failure_action not in SUPPORTED_MANAGEMENT: raise PyCOMPSsException( "ERROR: Unsupported on failure action: %s" % self.on_failure_action) # Keep all defaults in a dictionary self.defaults = kwargs
def __init__(self, *args, **kwargs): """ Store arguments passed to the decorator # self = itself. # args = not used. # kwargs = dictionary with the given implement parameters. :param args: Arguments :param kwargs: Keyword arguments """ self.args = args self.kwargs = kwargs self.registered = False self.first_register = False self.scope = context.in_pycompss() if self.scope: if __debug__: logger.debug("Init @implement decorator...") # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), "@implement")
def __init__(self, *args, **kwargs): """ Store arguments passed to the decorator # self = itself. # args = not used. # kwargs = dictionary with the given constraints. :param args: Arguments :param kwargs: Keyword arguments """ self.args = args self.kwargs = kwargs self.scope = context.in_pycompss() self.registered = False # This enables the decorator to get info from the caller # (e.g. self.source_frame_info.filename or # self.source_frame_info.lineno) # self.source_frame_info = inspect.getframeinfo(inspect.stack()[1][0]) if self.scope: if __debug__: logger.debug("Init @binary decorator...") # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), "@binary")
def __init__(self, *args, **kwargs): # type: (*typing.Any, **typing.Any) -> None """ Store arguments passed to the decorator self = itself. args = not used. kwargs = dictionary with the given constraints. :param args: Arguments. :param kwargs: Keyword arguments. """ decorator_name = "".join(("@", Decaf.__name__.lower())) # super(Decaf, self).__init__(decorator_name, *args, **kwargs) self.decorator_name = decorator_name self.args = args self.kwargs = kwargs self.scope = context.in_pycompss() self.core_element = None # type: typing.Any self.core_element_configured = False if self.scope: # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), decorator_name) # Get the computing nodes process_computing_nodes(decorator_name, self.kwargs)
def __init__(self, *args, **kwargs): # type: (*typing.Any, **typing.Any) -> None """ Store arguments passed to the decorator. self = itself. args = not used. kwargs = dictionary with the given http parameters. :param args: Arguments :param kwargs: Keyword arguments """ decorator_name = "".join(("@", HTTP.__name__.lower())) # super(HTTP, self).__init__(decorator_name, *args, **kwargs) self.decorator_name = decorator_name self.args = args self.kwargs = kwargs self.scope = context.in_pycompss() self.core_element = None # type: typing.Any self.core_element_configured = False self.task_type = "http" if self.scope: # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), decorator_name)
def __enter__(self): # type: () -> None """ Group creation """ if context.in_pycompss(): __open_task_group__(self.group_name, self.implicit_barrier) else: pass
def __exit__(self, type, value, traceback): # type: (typing.Any, typing.Any, typing.Any) -> None """ Group closing """ if context.in_pycompss(): __close_task_group__(self.group_name) if self.implicit_barrier: compss_barrier_group(self.group_name) else: pass
def compss_get_number_of_resources(): # type: () -> int """ Request for the number of active resources. :return: The number of active resources. """ if context.in_pycompss(): return __get_number_of_resources__() else: return __dummy_compss_get_number_of_resources__()
def compss_set_wall_clock(wall_clock_limit): # type: (int) -> None """ Sets the application wall clock limit. :param wall_clock_limit: Wall clock limit in seconds. :return: None """ if context.in_pycompss(): __set_wall_clock__(wall_clock_limit) else: __dummy_compss_set_wall_clock__(wall_clock_limit)
def compss_free_resources(num_resources, group_name): # type: (int, str) -> None """ Requests the destruction of num_resources resources. :param num_resources: Number of resources to destroy. :param group_name: Task group to notify upon resource creation :return: None """ if context.in_pycompss(): __free_resources__(num_resources, group_name) else: __dummy_compss_free_resources__(num_resources, group_name)
def __init__(self, group_name, implicit_barrier=True): # type: (str, bool) -> None """ Define a new group of tasks. :param group_name: Group name. :param implicit_barrier: Perform implicit barrier. """ if context.in_pycompss(): self.group_name = group_name self.implicit_barrier = implicit_barrier else: pass
def compss_stop(code=0, _hard_stop=False): # type: (int, bool) -> None """ Stops the runtime. :param code: Stop code. :param _hard_stop: Stop compss when runtime has died. :return: None """ if context.in_pycompss(): __stop_runtime__(code, _hard_stop) else: __dummy_compss_stop__(code, _hard_stop)
def __init__(self, *args, **kwargs): """ Store arguments passed to the decorator # self = itself. # args = not used. # kwargs = dictionary with the given ompss parameters :param args: Arguments :param kwargs: Keyword arguments """ self.args = args self.kwargs = kwargs self.registered = False self.scope = context.in_pycompss() if self.scope: if __debug__: logger.debug("Init @ompss decorator...") # Check the arguments check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS, SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS, list(kwargs.keys()), "@ompss") # Get the computing nodes: This parameter will have to go down # until execution when invoked. if 'computing_nodes' not in self.kwargs and \ 'computingNodes' not in self.kwargs: self.kwargs['computing_nodes'] = 1 else: if 'computingNodes' in self.kwargs: self.kwargs['computing_nodes'] = \ self.kwargs.pop('computingNodes') computing_nodes = kwargs['computing_nodes'] if isinstance(computing_nodes, int): self.kwargs['computing_nodes'] = kwargs['computing_nodes'] elif isinstance(computing_nodes, str) and \ computing_nodes.strip().startswith('$'): env_var = computing_nodes.strip()[1:] # Remove $ if env_var.startswith('{'): env_var = env_var[1:-1] # remove brackets self.kwargs['computing_nodes'] = int(os.environ[env_var]) else: raise Exception("Wrong Computing Nodes value at" + " @ompss decorator.") if __debug__: logger.debug("This OMPSs task will have " + str(self.kwargs['computing_nodes']) + " computing nodes.") else: pass
def compss_request_resources(num_resources, group_name): # type: (int, str) -> None """ Requests the creation of num_resources resources. :param num_resources: Number of resources to create. :param group_name: Task group to notify upon resource creation. (it can be None) :return: None """ if context.in_pycompss(): __request_resources__(num_resources, group_name) else: __dummy_compss_request_resources__(num_resources, group_name)
def compss_barrier(no_more_tasks=False): # type: (bool) -> None """ Wait for all tasks. Perform a barrier waiting until all the submitted tasks have finished. :param no_more_tasks: No more tasks boolean. :return: None. """ if context.in_pycompss(): __barrier__(no_more_tasks) else: __dummy_compss_barrier__(no_more_tasks)
def compss_start(log_level="off", tracing=0, interactive=False): # type: (str, int, bool) -> None """ Starts the runtime. :param log_level: Log level ["trace"|"debug"|"info"|"api"|"off"]. :param tracing: Tracing level [0 (deactivated)|1 (basic)|2 (advanced)]. :param interactive: Boolean if interactive (ipython or jupyter). :return: None """ if context.in_pycompss(): __start_runtime__(log_level, tracing, interactive) else: __dummy_compss_start__(log_level, tracing, interactive)
def test_terasort_example(): result_path = tempfile.mkdtemp() current_path = os.path.dirname(os.path.abspath(__file__)) terasort_dataset_path = os.path.join(current_path, "dataset", "terasort") argv_backup = sys.argv sys.argv = [EXAMPLES_NAME, terasort_dataset_path, result_path] terasort() sys.argv = argv_backup if in_pycompss(): barrier() time.sleep(5) # TODO: Why is this sleep needed? # Clean the results directory shutil.rmtree(result_path)
def compss_barrier_group(group_name): # type: (str) -> None """ Perform a barrier to a group. Stop until all the tasks of a group have finished. :param group_name: Name of the group to wait. :return: None. """ if context.in_pycompss(): exception_message = __barrier_group__(group_name) if exception_message != "None": raise __COMPSsException__(exception_message) else: __dummy_compss_barrier_group__(group_name)
def __init__(self, decorator_name, *args, **kwargs): # noqa self.decorator_name = decorator_name self.args = args self.kwargs = kwargs self.scope = context.in_pycompss() self.core_element = None self.core_element_configured = False # This enables the decorator to get info from the caller # (e.g. self.source_frame_info.filename or # self.source_frame_info.lineno) # import inspect # self.source_frame_info = inspect.getframeinfo(inspect.stack()[1][0]) if __debug__ and self.scope: # Log only in the master logger.debug("Init " + decorator_name + " decorator...")
def __init__(self, *args, **kwargs): """ Store arguments passed to the decorator # self = itself. # args = not used. # kwargs = dictionary with the given constraints. :param args: Arguments :param kwargs: Keyword arguments """ self.args = args self.kwargs = kwargs self.registered = False self.scope = context.in_pycompss() if self.scope and __debug__: logger.debug("Init @constraint decorator...")
def compss_wait_on(*args, **kwargs): # type: (*typing.Any, **typing.Any) -> typing.Any """ Wait for objects. Waits on a set of objects defined in args with the options defined in kwargs. Kwargs options: - "mode" Write enable? [ 'r' | 'rw' ] Default = 'rw' :param args: Objects to wait on. :param kwargs: Options dictionary. :return: List with the final values (or a single element if only one). """ if context.in_pycompss(): return __wait_on__(*args, **kwargs) else: return __dummy_compss_wait_on__(*args, **kwargs)
def compss_wait_on_directory(*directory_name): # type: (*str) -> None """ Wait and get a directory. Calls the runtime to bring the directory to the master when possible and waits until produced. :param directory_name: Directory/ies name. :return: None """ if context.in_pycompss(): if len(directory_name) == 1: __get_directory__(directory_name[0]) else: for d_name in directory_name: __get_directory__(d_name) else: __dummy_compss_wait_on_directory__(*directory_name)
def compss_delete_file(*file_name): # type: (*str) -> typing.Union[bool, typing.List[bool]] """ Delete a file. Calls the runtime to delete the file everywhere in the infrastructure. The delete is asynchronous and will be performed when the file is not necessary anymore. :param file_name: File/s name. :return: True if success. False otherwise. """ if context.in_pycompss(): if len(file_name) == 1: return __delete_file__(file_name[0]) else: return [__delete_file__(f_name) for f_name in file_name] else: return __dummy_compss_delete_file__(*file_name)
def compss_wait_on_file(*file_name): # type: (*str) -> None """ Wait and get a file. Calls the runtime to bring the file to the master when possible and waits until produced. :param file_name: File/s name. :return: None """ if context.in_pycompss(): if len(file_name) == 1: __get_file__(file_name[0]) else: for f_name in file_name: __get_file__(f_name) else: __dummy_compss_wait_on_file__(*file_name)
def compss_delete_object(*obj): # type: (*typing.Any) -> typing.Union[bool, typing.List[bool]] """ Delete object. Removes a used object from the internal structures and calls the external python library (that calls the bindings-common) in order to request a its corresponding file removal. :param obj: Object/s to delete. :return: True if success. False otherwise. """ if context.in_pycompss(): if len(obj) == 1: return __delete_object__(obj[0]) else: return [__delete_object__(i_obj) for i_obj in obj] else: return __dummy_compss_delete_object__(*obj)