def test_enable_nesting():
from pycompss.util.context import enable_nesting
from pycompss.util.context import disable_nesting
from pycompss.util.context import is_nesting_enabled
not_enabled = is_nesting_enabled()
enable_nesting()
is_enabled = is_nesting_enabled()
disable_nesting()
is_disabled = is_nesting_enabled()
assert not_enabled is False, "ERROR: Nesting must not be enabled by default." # noqa: E501
assert is_enabled is True, "ERROR: Nesting has not been enabled."
assert is_disabled is False, "ERROR: Nesting has not been disabled."
def constrained_f(*args, **kwargs):
# type: (*typing.Any, **typing.Any) -> typing.Any
if not self.scope:
from pycompss.api.dummy.on_failure import on_failure \
as dummy_on_failure
d_c = dummy_on_failure(self.args, self.kwargs)
return d_c.__call__(user_function)(*args, **kwargs)
if __debug__:
logger.debug("Executing on_failure_f wrapper.")
if (context.in_master() or context.is_nesting_enabled()) \
and not self.core_element_configured:
# master code - or worker with nesting enabled
self.__configure_core_element__(kwargs)
# Set the on failure management action and default variables in
# kwargs for its usage in @task decorator
kwargs["on_failure"] = self.on_failure_action
kwargs["defaults"] = self.defaults
with keep_arguments(args, kwargs, prepend_strings=True):
# Call the method
ret = user_function(*args, **kwargs)
return ret
def multinode_f(*args, **kwargs):
if not self.scope:
raise NotInPyCOMPSsException(not_in_pycompss("MultiNode"))
if __debug__:
logger.debug("Executing multinode_f wrapper.")
if (context.in_master() or context.is_nesting_enabled()) \
and not self.core_element_configured:
# master code - or worker with nesting enabled
self.__configure_core_element__(kwargs, user_function)
if context.in_worker():
old_slurm_env = set_slurm_environment()
# Set the computing_nodes variable in kwargs for its usage
# in @task decorator
kwargs['computing_nodes'] = self.kwargs['computing_nodes']
with keep_arguments(args, kwargs, prepend_strings=True):
# Call the method
ret = user_function(*args, **kwargs)
if context.in_worker():
reset_slurm_environment(old_slurm_env)
return ret
def __decorator_body__(self, user_function, args, kwargs):
# type: (typing.Callable, tuple, dict) -> typing.Any
if not self.scope:
# Execute the mpi as with PyCOMPSs so that sequential
# execution performs as parallel.
# To disable: raise Exception(not_in_pycompss("mpi"))
# TODO: Intercept @task parameters to get stream redirection
if "binary" in self.kwargs:
return self.__run_mpi__(args, kwargs)
else:
print(
"WARN: Python MPI as dummy is not fully supported. Executing decorated funtion."
)
return user_function(*args, **kwargs)
if __debug__:
logger.debug("Executing mpi_f wrapper.")
if (context.in_master() or context.is_nesting_enabled()) \
and not self.core_element_configured:
# master code - or worker with nesting enabled
self.__configure_core_element__(kwargs)
# The processes parameter will have to go down until the execution
# is invoked. To this end, set the computing_nodes variable in kwargs
# for its usage in @task decorator
# WARNING: processes can be an int, a env string, a str with
# dynamic variable name.
if "processes" in self.kwargs:
kwargs["computing_nodes"] = self.kwargs["processes"]
else:
# If processes not defined, check computing_units or set default
process_computing_nodes(self.decorator_name, self.kwargs)
kwargs["computing_nodes"] = self.kwargs["computing_nodes"]
if "processes_per_node" in self.kwargs:
kwargs["processes_per_node"] = self.kwargs["processes_per_node"]
else:
kwargs["processes_per_node"] = 1
if __debug__:
logger.debug("This MPI task will have " +
str(kwargs["computing_nodes"]) + " processes and " +
str(kwargs["processes_per_node"]) +
" processes per node.")
if self.task_type == IMPL_PYTHON_MPI:
prepend_strings = True
else:
prepend_strings = False
with keep_arguments(args, kwargs, prepend_strings=prepend_strings):
# Call the method
ret = user_function(*args, **kwargs)
return ret
def container_f(*args, **kwargs):
if not self.scope:
raise NotInPyCOMPSsException(not_in_pycompss("container"))
if __debug__:
logger.debug("Executing container_f wrapper.")
if (context.in_master() or context.is_nesting_enabled()) \
and not self.core_element_configured:
# master code - or worker with nesting enabled
self.__configure_core_element__(kwargs, user_function)
with keep_arguments(args, kwargs, prepend_strings=False):
# Call the method
ret = user_function(*args, **kwargs)
return ret
def implement_f(*args, **kwargs):
# type: (*typing.Any, **typing.Any) -> typing.Any
# This is executed only when called.
if not self.scope:
raise NotInPyCOMPSsException(not_in_pycompss("implement"))
if __debug__:
logger.debug("Executing implement_f wrapper.")
if (context.in_master() or context.is_nesting_enabled()) \
and not self.core_element_configured:
# master code - or worker with nesting enabled
self.__configure_core_element__(kwargs)
with keep_arguments(args, kwargs, prepend_strings=True):
# Call the method
ret = user_function(*args, **kwargs)
return ret
def load_loggers(debug, persistent_storage):
# type: (bool, str) -> (..., str, ...)
""" Load all loggers.
:param debug: is Debug enabled.
:param persistent_storage: is persistent storage enabled.
:return: main logger of the application, the log config file (json) and
a list of loggers for the persistent data framework.
"""
# Load log level configuration file
worker_path = os.path.dirname(os.path.realpath(__file__))
log_cfg_path = "".join((worker_path, '/../../../../log'))
if not os.path.isdir(log_cfg_path):
# If not exists, then we are using the source for unit testing
log_cfg_path = "".join((worker_path, '/../../../../../log'))
if debug:
# Debug
log_json = "/".join((log_cfg_path, 'logging_worker_debug.json'))
else:
# Default
log_json = "/".join((log_cfg_path, 'logging_worker_off.json'))
log_dir = get_temporary_directory()
# log_dir is of the form:
# With agents or worker in master: /path/to/working_directory/tmpFiles/pycompssID/
# Normal master-worker execution : /path/to/working_directory/machine_name/pycompssID/
# With normal master-worker execution, it transfers the err and out files in the
# expected folder to the master.
# With agents or worker in master it does not, so keep it in previous two folders:
if context.is_nesting_enabled() or "tmpFiles" in log_dir:
log_dir = os.path.join(log_dir, "..", "..")
else:
log_dir = os.path.join(log_dir, "..")
init_logging_worker_piper(log_json, log_dir)
# Define logger facilities
logger = logging.getLogger('pycompss.worker.piper.piper_worker')
storage_loggers = []
if persistent_storage:
storage_loggers.append(logging.getLogger('dataclay'))
storage_loggers.append(logging.getLogger('hecuba'))
storage_loggers.append(logging.getLogger('redis'))
storage_loggers.append(logging.getLogger('storage'))
return logger, log_json, storage_loggers
def constrained_f(*args, **kwargs):
if not self.scope:
from pycompss.api.dummy.constraint import constraint \
as dummy_constraint
d_c = dummy_constraint(self.args, self.kwargs)
return d_c.__call__(user_function)(*args, **kwargs)
if __debug__:
logger.debug("Executing constrained_f wrapper.")
if (context.in_master() or context.is_nesting_enabled()) \
and not self.core_element_configured:
# master code - or worker with nesting enabled
self.__configure_core_element__(kwargs, user_function)
with keep_arguments(args, kwargs, prepend_strings=True):
# Call the method
ret = user_function(*args, **kwargs)
return ret
def __decorator_body__(self, user_function, args, kwargs):
# type: (typing.Callable, tuple, dict) -> typing.Any
if not self.scope:
raise NotImplementedError
if __debug__:
logger.debug("Executing mpmd_mpi_f wrapper.")
if (context.in_master() or context.is_nesting_enabled()) \
and not self.core_element_configured:
# master code - or worker with nesting enabled
self.__configure_core_element__(kwargs)
kwargs[PROCESSES_PER_NODE] = self.kwargs.get(PROCESSES_PER_NODE, 1)
kwargs[COMPUTING_NODES] = self.processes
with keep_arguments(args, kwargs, prepend_strings=False):
# Call the method
ret = user_function(*args, **kwargs)
return ret
def __decorator_body__(self, user_function, args, kwargs):
# type: (typing.Callable, tuple, dict) -> typing.Any
# force to serialize with JSON
serializer.FORCED_SERIALIZER = 4
if not self.scope:
# run http
self.__run_http__(args, kwargs)
if __debug__:
logger.debug("Executing http_f wrapper.")
if (context.in_master() or context.is_nesting_enabled()) \
and not self.core_element_configured:
# master code - or worker with nesting enabled
self.__configure_core_element__(kwargs)
with keep_arguments(args, kwargs):
# Call the method
ret = user_function(*args, **kwargs)
return ret
def binary_f(*args, **kwargs):
if not self.scope:
# Execute the binary as with PyCOMPSs so that sequential
# execution performs as parallel.
# To disable: raise Exception(not_in_pycompss("binary"))
# TODO: Intercept @task parameters to get stream redirection
return self.__run_binary__(args, kwargs)
if __debug__:
logger.debug("Executing binary_f wrapper.")
if (context.in_master() or context.is_nesting_enabled()) \
and not self.core_element_configured:
# master code - or worker with nesting enabled
self.__configure_core_element__(kwargs, user_function)
with keep_arguments(args, kwargs, prepend_strings=False):
# Call the method
ret = user_function(*args, **kwargs)
return ret
def decaf_f(*args, **kwargs):
# type: (*typing.Any, **typing.Any) -> typing.Any
if not self.scope:
raise NotInPyCOMPSsException(not_in_pycompss("decaf"))
if __debug__:
logger.debug("Executing decaf_f wrapper.")
if (context.in_master() or context.is_nesting_enabled()) \
and not self.core_element_configured:
# master code - or worker with nesting enabled
self.__configure_core_element__(kwargs)
# Set the computing_nodes variable in kwargs for its usage
# in @task decorator
kwargs[COMPUTING_NODES] = self.kwargs[COMPUTING_NODES]
with keep_arguments(args, kwargs, prepend_strings=False):
# Call the method
ret = user_function(*args, **kwargs)
return ret
def __decorator_body__(self, user_function, args, kwargs):
# Determine the context and decide what to do
if context.in_master():
# @task being executed in the master
# Each task will have a TaskMaster, so its content will
# not be shared.
self.__check_core_element__(kwargs, user_function)
with event(TASK_INSTANTIATION, master=True):
master = TaskMaster(self.decorator_arguments,
self.user_function, self.core_element,
self.registered, self.signature,
self.interactive, self.module,
self.function_arguments,
self.function_name, self.module_name,
self.function_type, self.class_name,
self.hints, self.on_failure, self.defaults)
result = master.call(*args, **kwargs)
fo, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints = result # noqa: E501
del master
return fo
elif context.in_worker():
if "compss_key" in kwargs.keys():
if context.is_nesting_enabled():
# Update the whole logger since it will be in job out/err
update_logger_handlers(kwargs["compss_log_cfg"],
kwargs["compss_log_files"][0],
kwargs["compss_log_files"][1])
# @task being executed in the worker
with event(WORKER_TASK_INSTANTIATION,
master=False,
inside=True):
worker = TaskWorker(self.decorator_arguments,
self.user_function, self.on_failure,
self.defaults)
result = worker.call(*args, **kwargs)
# Force flush stdout and stderr
sys.stdout.flush()
sys.stderr.flush()
# Remove worker
del worker
if context.is_nesting_enabled():
# Wait for all nested tasks to finish
from pycompss.runtime.binding import nested_barrier
nested_barrier()
# Reestablish logger handlers
update_logger_handlers(kwargs["compss_log_cfg"])
return result
else:
if context.is_nesting_enabled():
# Each task will have a TaskMaster, so its content will
# not be shared.
with event(TASK_INSTANTIATION, master=True):
master = TaskMaster(
self.decorator_arguments, self.user_function,
self.core_element, self.registered, self.signature,
self.interactive, self.module,
self.function_arguments, self.function_name,
self.module_name, self.function_type,
self.class_name, self.hints, self.on_failure,
self.defaults)
result = master.call(*args, **kwargs)
fo, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints = result # noqa: E501
del master
return fo
else:
# Called from another task within the worker
# Ignore the @task decorator and run it sequentially
message = "".join((
"WARNING: Calling task: ",
str(user_function.__name__),
" from this task.\n",
" It will be executed sequentially ", # noqa: E501
"within the caller task."))
print(message, file=sys.stderr)
return self._sequential_call(*args, **kwargs)
# We are neither in master nor in the worker, or the user has
# stopped the interactive session.
# Therefore, the user code is being executed with no
# launch_compss/enqueue_compss/runcompss/interactive session
return self._sequential_call(*args, **kwargs)
def _synchronize(obj, mode):
# type: (typing.Any, int) -> typing.Any
""" Synchronization function.
This method retrieves the value of a future object.
Calls the runtime in order to wait for the value and returns it when
received.
:param obj: Object to synchronize.
:param mode: Direction of the object to synchronize.
:return: The value of the object requested.
"""
# TODO: Add a boolean to differentiate between files and object on the
# COMPSs.open_file call. This change pretends to obtain better traces.
# Must be implemented first in the Runtime, then in the bindings common
# C API and finally add the boolean here
app_id = 0
obj_id = "" # noqa
if is_psco(obj):
obj_id = str(get_id(obj))
if not OT.is_pending_to_synchronize(obj_id):
return obj
else:
# file_path is of the form storage://pscoId or
# file://sys_path_to_file
file_path = COMPSs.open_file(app_id, "".join(
("storage://", str(obj_id))), mode)
# TODO: Add switch on protocol (first parameter returned currently ignored)
_, file_name = file_path.split("://")
new_obj = get_by_id(file_name)
OT.stop_tracking(obj)
return new_obj
obj_id = OT.is_tracked(obj)
obj_name = OT.get_obj_name(obj_id)
if obj_id is None: # Not being tracked
return obj
if not OT.is_pending_to_synchronize(obj_id):
return obj
if __debug__:
logger.debug("Synchronizing object %s with mode %s" % (obj_id, mode))
file_name = OT.get_file_name(obj_id)
compss_file = COMPSs.open_file(app_id, file_name, mode)
# Runtime can return a path or a PSCOId
if compss_file.startswith('/'):
# If the real filename is null, then return None. The task that
# produces the output file may have been ignored or cancelled, so its
# result does not exist.
real_file_name = compss_file.split('/')[-1]
if real_file_name == "null":
print(
"WARNING: Could not retrieve the object " + str(file_name) +
" since the task that produces it may have been IGNORED or CANCELLED. Please, check the logs. Returning None."
) # noqa: E501
return None
new_obj = deserialize_from_file(compss_file)
COMPSs.close_file(app_id, file_name, mode)
else:
new_obj = get_by_id(compss_file)
if context.is_nesting_enabled() and context.in_worker():
# If nesting and in worker, the user wants to synchronize an object.
# So it is necessary to update the parameter with the new object.
update_worker_argument_parameter_content(obj_name, new_obj)
if mode == 'r':
OT.update_mapping(obj_id, new_obj)
if mode != 'r':
COMPSs.delete_file(app_id, OT.get_file_name(obj_id), False)
OT.stop_tracking(obj)
return new_obj
def task_decorator(*args, **kwargs):
# Determine the context and decide what to do
if context.in_master():
# @task being executed in the master
# Each task will have a TaskMaster, so its content will
# not be shared.
self.__check_core_element__(kwargs, user_function)
with event(TASK_INSTANTIATION, master=True):
master = TaskMaster(
self.decorator_arguments, self.user_function,
self.core_element, self.registered, self.signature,
self.interactive, self.module, self.function_arguments,
self.function_name, self.module_name,
self.function_type, self.class_name, self.hints)
result = master.call(*args, **kwargs)
fo, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints = result # noqa: E501
del master
return fo
elif context.in_worker():
if 'compss_key' in kwargs.keys():
# @task being executed in the worker
with event(WORKER_TASK_INSTANTIATION,
master=False,
inside=True):
worker = TaskWorker(self.decorator_arguments,
self.user_function)
result = worker.call(*args, **kwargs)
del worker
return result
else:
if context.is_nesting_enabled():
# nested @task executed in the worker
# Each task will have a TaskMaster, so its content will
# not be shared.
with event(TASK_INSTANTIATION, master=True):
master = TaskMaster(
self.decorator_arguments, self.user_function,
self.core_element, self.registered,
self.signature, self.interactive, self.module,
self.function_arguments, self.function_name,
self.module_name, self.function_type,
self.class_name, self.hints)
result = master.call(*args, **kwargs)
fo, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints = result # noqa: E501
del master
return fo
else:
# Called from another task within the worker
# Ignore the @task decorator and run it sequentially
message = "".join((
"WARNING: Calling task: ",
str(user_function.__name__),
" from this task.\n",
" It will be executed sequentially ", # noqa: E501
"within the caller task."))
print(message, file=sys.stderr)
return self._sequential_call(*args, **kwargs)
# We are neither in master nor in the worker, or the user has
# stopped the interactive session.
# Therefore, the user code is being executed with no
# launch_compss/enqueue_compss/runcompss/interactive session
return self._sequential_call(*args, **kwargs)
