def test_in_pycompss_context():
from pycompss.util.context import in_pycompss
from pycompss.util.context import set_pycompss_context
set_pycompss_context(MASTER)
master_context = in_pycompss()
set_pycompss_context(WORKER)
worker_context = in_pycompss()
assert master_context is True, MASTER_CONTEXT_ERROR
assert worker_context is True, WORKER_CONTEXT_ERROR
set_pycompss_context(OUT_OF_SCOPE)
def local(input_function):
""" Local decorator.
:param input_function: Input function.
:return: Wrapped function.
"""
if not context.in_pycompss():
# Return dummy local decorator
def wrapped_function(*args, **kwargs):
return input_function(*args, **kwargs)
return wrapped_function
else:
def sync_if_needed(obj):
# type: (object) -> None
if OT_is_obj_pending_to_synchronize(obj):
new_val = compss_wait_on(obj)
replace(obj, new_val)
def wrapped_function(*args, **kwargs):
gc.collect()
_args = []
_kwargs = {}
for arg in args:
sync_if_needed(arg)
_args.append(arg)
for (key, value) in kwargs.items():
sync_if_needed(value)
_kwargs[key] = value
return input_function(*_args, **_kwargs)
return wrapped_function
def __init__(self, *args, **kwargs):
# type: (*typing.Any, **typing.Any) -> None
""" Store arguments passed to the decorator.
self = itself.
args = not used.
kwargs = dictionary with the given mpi parameters.
:param args: Arguments
:param kwargs: Keyword arguments
"""
decorator_name = "".join(("@", MPMDMPI.__name__.lower()))
# super(MPMDMPI, self).__init__(decorator_name, *args, **kwargs)
self.decorator_name = decorator_name
self.args = args
self.kwargs = kwargs
self.scope = context.in_pycompss()
self.core_element = None # type: typing.Any
self.core_element_configured = False
# MPMD_MPI specific:
self.task_type = "mpmd_mpi"
self.processes = 0
if self.scope:
if __debug__:
logger.debug("Init @mpmd_mpi decorator...")
# Add <param_name>_layout params to SUPPORTED_ARGUMENTS
for key in self.kwargs.keys():
if "_layout" in key:
SUPPORTED_ARGUMENTS.add(key)
# Check the arguments
check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS,
SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS,
list(kwargs.keys()), self.decorator_name)
def __init__(self, *args, **kwargs):
# type: (*typing.Any, **typing.Any) -> None
"""
Store arguments passed to the decorator
# self = itself.
# args = not used.
# kwargs = dictionary with the given constraints.
:param args: Arguments
:param kwargs: Keyword arguments
"""
decorator_name = "@" + Container.__name__.lower()
# super(Container, self).__init__(decorator_name, *args, **kwargs)
self.decorator_name = decorator_name
self.args = args
self.kwargs = kwargs
self.scope = context.in_pycompss()
self.core_element = None # type: typing.Any
self.core_element_configured = False
if self.scope:
if __debug__:
logger.debug("Init @container decorator...")
# Check the arguments
check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS,
SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS,
list(kwargs.keys()), decorator_name)
def __init__(self, *args, **kwargs):
# type: (*typing.Any, **typing.Any) -> None
"""
Store arguments passed to the decorator
# self = itself.
# args = not used.
# kwargs = dictionary with the given Reduce parameters
:param args: Arguments
:param kwargs: Keyword arguments
"""
decorator_name = "".join(("@", self.__class__.__name__.lower()))
# super(self.__class__, self).__init__(decorator_name, *args, **kwargs)
# Instantiate superclass explicitly to support mypy.
pd = PyCOMPSsDecorator(decorator_name, *args, **kwargs)
self.decorator_name = decorator_name
self.args = args
self.kwargs = kwargs
self.scope = context.in_pycompss()
self.core_element = None # type: typing.Any
self.core_element_configured = False
self.__configure_core_element__ = pd.__configure_core_element__
if self.scope:
# Check the arguments
check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS,
SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS,
list(kwargs.keys()), decorator_name)
# Get the computing nodes
self.__process_reduction_params__()
def __init__(self, *args, **kwargs):
# type: (*typing.Any, **typing.Any) -> None
""" Store arguments passed to the decorator.
self = itself.
args = not used.
kwargs = dictionary with the given on_failure.
:param args: Arguments.
:param kwargs: Keyword arguments.
"""
decorator_name = "".join(("@", OnFailure.__name__.lower()))
# super(OnFailure, self).__init__(decorator_name, *args, **kwargs)
self.decorator_name = decorator_name
self.args = args
self.kwargs = kwargs
self.scope = context.in_pycompss()
self.core_element = None # type: typing.Any
self.core_element_configured = False
if self.scope:
# Check the arguments
check_mandatory_arguments(MANDATORY_ARGUMENTS,
list(kwargs.keys()),
decorator_name)
# Save the parameters into self so that they can be accessed when
# the task fails and the action needs to be taken
self.on_failure_action = kwargs.pop(MANAGEMENT)
# Check supported management values
if self.on_failure_action not in SUPPORTED_MANAGEMENT:
raise PyCOMPSsException(
"ERROR: Unsupported on failure action: %s" %
self.on_failure_action)
# Keep all defaults in a dictionary
self.defaults = kwargs
def __init__(self, *args, **kwargs):
"""
Store arguments passed to the decorator
# self = itself.
# args = not used.
# kwargs = dictionary with the given implement parameters.
:param args: Arguments
:param kwargs: Keyword arguments
"""
self.args = args
self.kwargs = kwargs
self.registered = False
self.first_register = False
self.scope = context.in_pycompss()
if self.scope:
if __debug__:
logger.debug("Init @implement decorator...")
# Check the arguments
check_arguments(MANDATORY_ARGUMENTS,
DEPRECATED_ARGUMENTS,
SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS,
list(kwargs.keys()),
"@implement")
def __init__(self, *args, **kwargs):
"""
Store arguments passed to the decorator
# self = itself.
# args = not used.
# kwargs = dictionary with the given constraints.
:param args: Arguments
:param kwargs: Keyword arguments
"""
self.args = args
self.kwargs = kwargs
self.scope = context.in_pycompss()
self.registered = False
# This enables the decorator to get info from the caller
# (e.g. self.source_frame_info.filename or
# self.source_frame_info.lineno)
# self.source_frame_info = inspect.getframeinfo(inspect.stack()[1][0])
if self.scope:
if __debug__:
logger.debug("Init @binary decorator...")
# Check the arguments
check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS,
SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS,
list(kwargs.keys()), "@binary")
def __init__(self, *args, **kwargs):
# type: (*typing.Any, **typing.Any) -> None
""" Store arguments passed to the decorator
self = itself.
args = not used.
kwargs = dictionary with the given constraints.
:param args: Arguments.
:param kwargs: Keyword arguments.
"""
decorator_name = "".join(("@", Decaf.__name__.lower()))
# super(Decaf, self).__init__(decorator_name, *args, **kwargs)
self.decorator_name = decorator_name
self.args = args
self.kwargs = kwargs
self.scope = context.in_pycompss()
self.core_element = None # type: typing.Any
self.core_element_configured = False
if self.scope:
# Check the arguments
check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS,
SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS,
list(kwargs.keys()), decorator_name)
# Get the computing nodes
process_computing_nodes(decorator_name, self.kwargs)
def __init__(self, *args, **kwargs):
# type: (*typing.Any, **typing.Any) -> None
""" Store arguments passed to the decorator.
self = itself.
args = not used.
kwargs = dictionary with the given http parameters.
:param args: Arguments
:param kwargs: Keyword arguments
"""
decorator_name = "".join(("@", HTTP.__name__.lower()))
# super(HTTP, self).__init__(decorator_name, *args, **kwargs)
self.decorator_name = decorator_name
self.args = args
self.kwargs = kwargs
self.scope = context.in_pycompss()
self.core_element = None # type: typing.Any
self.core_element_configured = False
self.task_type = "http"
if self.scope:
# Check the arguments
check_arguments(MANDATORY_ARGUMENTS, DEPRECATED_ARGUMENTS,
SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS,
list(kwargs.keys()), decorator_name)
def __enter__(self):
# type: () -> None
""" Group creation """
if context.in_pycompss():
__open_task_group__(self.group_name, self.implicit_barrier)
else:
pass
def __exit__(self, type, value, traceback):
# type: (typing.Any, typing.Any, typing.Any) -> None
""" Group closing """
if context.in_pycompss():
__close_task_group__(self.group_name)
if self.implicit_barrier:
compss_barrier_group(self.group_name)
else:
pass
def compss_get_number_of_resources():
# type: () -> int
""" Request for the number of active resources.
:return: The number of active resources.
"""
if context.in_pycompss():
return __get_number_of_resources__()
else:
return __dummy_compss_get_number_of_resources__()
def compss_set_wall_clock(wall_clock_limit):
# type: (int) -> None
""" Sets the application wall clock limit.
:param wall_clock_limit: Wall clock limit in seconds.
:return: None
"""
if context.in_pycompss():
__set_wall_clock__(wall_clock_limit)
else:
__dummy_compss_set_wall_clock__(wall_clock_limit)
def compss_free_resources(num_resources, group_name):
# type: (int, str) -> None
""" Requests the destruction of num_resources resources.
:param num_resources: Number of resources to destroy.
:param group_name: Task group to notify upon resource creation
:return: None
"""
if context.in_pycompss():
__free_resources__(num_resources, group_name)
else:
__dummy_compss_free_resources__(num_resources, group_name)
def __init__(self, group_name, implicit_barrier=True):
# type: (str, bool) -> None
""" Define a new group of tasks.
:param group_name: Group name.
:param implicit_barrier: Perform implicit barrier.
"""
if context.in_pycompss():
self.group_name = group_name
self.implicit_barrier = implicit_barrier
else:
pass
def compss_stop(code=0, _hard_stop=False):
# type: (int, bool) -> None
""" Stops the runtime.
:param code: Stop code.
:param _hard_stop: Stop compss when runtime has died.
:return: None
"""
if context.in_pycompss():
__stop_runtime__(code, _hard_stop)
else:
__dummy_compss_stop__(code, _hard_stop)
def __init__(self, *args, **kwargs):
"""
Store arguments passed to the decorator
# self = itself.
# args = not used.
# kwargs = dictionary with the given ompss parameters
:param args: Arguments
:param kwargs: Keyword arguments
"""
self.args = args
self.kwargs = kwargs
self.registered = False
self.scope = context.in_pycompss()
if self.scope:
if __debug__:
logger.debug("Init @ompss decorator...")
# Check the arguments
check_arguments(MANDATORY_ARGUMENTS,
DEPRECATED_ARGUMENTS,
SUPPORTED_ARGUMENTS | DEPRECATED_ARGUMENTS,
list(kwargs.keys()),
"@ompss")
# Get the computing nodes: This parameter will have to go down
# until execution when invoked.
if 'computing_nodes' not in self.kwargs and \
'computingNodes' not in self.kwargs:
self.kwargs['computing_nodes'] = 1
else:
if 'computingNodes' in self.kwargs:
self.kwargs['computing_nodes'] = \
self.kwargs.pop('computingNodes')
computing_nodes = kwargs['computing_nodes']
if isinstance(computing_nodes, int):
self.kwargs['computing_nodes'] = kwargs['computing_nodes']
elif isinstance(computing_nodes, str) and \
computing_nodes.strip().startswith('$'):
env_var = computing_nodes.strip()[1:] # Remove $
if env_var.startswith('{'):
env_var = env_var[1:-1] # remove brackets
self.kwargs['computing_nodes'] = int(os.environ[env_var])
else:
raise Exception("Wrong Computing Nodes value at" +
" @ompss decorator.")
if __debug__:
logger.debug("This OMPSs task will have " +
str(self.kwargs['computing_nodes']) +
" computing nodes.")
else:
pass
def compss_request_resources(num_resources, group_name):
# type: (int, str) -> None
""" Requests the creation of num_resources resources.
:param num_resources: Number of resources to create.
:param group_name: Task group to notify upon resource creation.
(it can be None)
:return: None
"""
if context.in_pycompss():
__request_resources__(num_resources, group_name)
else:
__dummy_compss_request_resources__(num_resources, group_name)
def compss_barrier(no_more_tasks=False):
# type: (bool) -> None
""" Wait for all tasks.
Perform a barrier waiting until all the submitted tasks have finished.
:param no_more_tasks: No more tasks boolean.
:return: None.
"""
if context.in_pycompss():
__barrier__(no_more_tasks)
else:
__dummy_compss_barrier__(no_more_tasks)
def compss_start(log_level="off", tracing=0, interactive=False):
# type: (str, int, bool) -> None
""" Starts the runtime.
:param log_level: Log level ["trace"|"debug"|"info"|"api"|"off"].
:param tracing: Tracing level [0 (deactivated)|1 (basic)|2 (advanced)].
:param interactive: Boolean if interactive (ipython or jupyter).
:return: None
"""
if context.in_pycompss():
__start_runtime__(log_level, tracing, interactive)
else:
__dummy_compss_start__(log_level, tracing, interactive)
def test_terasort_example():
result_path = tempfile.mkdtemp()
current_path = os.path.dirname(os.path.abspath(__file__))
terasort_dataset_path = os.path.join(current_path, "dataset", "terasort")
argv_backup = sys.argv
sys.argv = [EXAMPLES_NAME, terasort_dataset_path, result_path]
terasort()
sys.argv = argv_backup
if in_pycompss():
barrier()
time.sleep(5) # TODO: Why is this sleep needed?
# Clean the results directory
shutil.rmtree(result_path)
def compss_barrier_group(group_name):
# type: (str) -> None
""" Perform a barrier to a group.
Stop until all the tasks of a group have finished.
:param group_name: Name of the group to wait.
:return: None.
"""
if context.in_pycompss():
exception_message = __barrier_group__(group_name)
if exception_message != "None":
raise __COMPSsException__(exception_message)
else:
__dummy_compss_barrier_group__(group_name)
def __init__(self, decorator_name, *args, **kwargs): # noqa
self.decorator_name = decorator_name
self.args = args
self.kwargs = kwargs
self.scope = context.in_pycompss()
self.core_element = None
self.core_element_configured = False
# This enables the decorator to get info from the caller
# (e.g. self.source_frame_info.filename or
# self.source_frame_info.lineno)
# import inspect
# self.source_frame_info = inspect.getframeinfo(inspect.stack()[1][0])
if __debug__ and self.scope:
# Log only in the master
logger.debug("Init " + decorator_name + " decorator...")
def __init__(self, *args, **kwargs):
"""
Store arguments passed to the decorator
# self = itself.
# args = not used.
# kwargs = dictionary with the given constraints.
:param args: Arguments
:param kwargs: Keyword arguments
"""
self.args = args
self.kwargs = kwargs
self.registered = False
self.scope = context.in_pycompss()
if self.scope and __debug__:
logger.debug("Init @constraint decorator...")
def compss_wait_on(*args, **kwargs):
# type: (*typing.Any, **typing.Any) -> typing.Any
""" Wait for objects.
Waits on a set of objects defined in args with the options defined in
kwargs.
Kwargs options:
- "mode" Write enable? [ 'r' | 'rw' ] Default = 'rw'
:param args: Objects to wait on.
:param kwargs: Options dictionary.
:return: List with the final values (or a single element if only one).
"""
if context.in_pycompss():
return __wait_on__(*args, **kwargs)
else:
return __dummy_compss_wait_on__(*args, **kwargs)
def compss_wait_on_directory(*directory_name):
# type: (*str) -> None
""" Wait and get a directory.
Calls the runtime to bring the directory to the master when possible
and waits until produced.
:param directory_name: Directory/ies name.
:return: None
"""
if context.in_pycompss():
if len(directory_name) == 1:
__get_directory__(directory_name[0])
else:
for d_name in directory_name:
__get_directory__(d_name)
else:
__dummy_compss_wait_on_directory__(*directory_name)
def compss_delete_file(*file_name):
# type: (*str) -> typing.Union[bool, typing.List[bool]]
""" Delete a file.
Calls the runtime to delete the file everywhere in the infrastructure.
The delete is asynchronous and will be performed when the file is not
necessary anymore.
:param file_name: File/s name.
:return: True if success. False otherwise.
"""
if context.in_pycompss():
if len(file_name) == 1:
return __delete_file__(file_name[0])
else:
return [__delete_file__(f_name) for f_name in file_name]
else:
return __dummy_compss_delete_file__(*file_name)
def compss_wait_on_file(*file_name):
# type: (*str) -> None
""" Wait and get a file.
Calls the runtime to bring the file to the master when possible
and waits until produced.
:param file_name: File/s name.
:return: None
"""
if context.in_pycompss():
if len(file_name) == 1:
__get_file__(file_name[0])
else:
for f_name in file_name:
__get_file__(f_name)
else:
__dummy_compss_wait_on_file__(*file_name)
def compss_delete_object(*obj):
# type: (*typing.Any) -> typing.Union[bool, typing.List[bool]]
""" Delete object.
Removes a used object from the internal structures and calls the
external python library (that calls the bindings-common)
in order to request a its corresponding file removal.
:param obj: Object/s to delete.
:return: True if success. False otherwise.
"""
if context.in_pycompss():
if len(obj) == 1:
return __delete_object__(obj[0])
else:
return [__delete_object__(i_obj) for i_obj in obj]
else:
return __dummy_compss_delete_object__(*obj)