def multinode_f(*args, **kwargs):
if not self.scope:
raise NotInPyCOMPSsException(not_in_pycompss("MultiNode"))
if __debug__:
logger.debug("Executing multinode_f wrapper.")
if (context.in_master() or context.is_nesting_enabled()) \
and not self.core_element_configured:
# master code - or worker with nesting enabled
self.__configure_core_element__(kwargs, user_function)
if context.in_worker():
old_slurm_env = set_slurm_environment()
# Set the computing_nodes variable in kwargs for its usage
# in @task decorator
kwargs['computing_nodes'] = self.kwargs['computing_nodes']
with keep_arguments(args, kwargs, prepend_strings=True):
# Call the method
ret = user_function(*args, **kwargs)
if context.in_worker():
reset_slurm_environment(old_slurm_env)
return ret
def test_in_worker_context():
from pycompss.util.context import in_worker
from pycompss.util.context import set_pycompss_context
set_pycompss_context(WORKER)
worker_context = in_worker()
assert worker_context is True, WORKER_CONTEXT_ERROR
set_pycompss_context(OUT_OF_SCOPE)
def event(event_id,
master=False,
inside=False,
cpu_affinity=False,
gpu_affinity=False):
# type: (int or str, bool, bool, bool, bool) -> None
""" Emits an event wrapping the desired code.
Does nothing if tracing is disabled.
:param event_id: Event identifier to emit.
:param master: If the event is emitted as master.
:param inside: If the event is produced inside the worker.
:param cpu_affinity: If the event is produced inside the worker for
cpu affinity.
:param gpu_affinity: If the event is produced inside the worker for
gpu affinity.
:return: None
"""
emit = False
if TRACING and in_master() and master:
emit = True
if TRACING and in_worker() and not master:
emit = True
if emit:
event_group, event_id = __get_proper_type_event__(
event_id, master, inside, cpu_affinity, gpu_affinity)
PYEXTRAE.eventandcounters(event_group, event_id) # noqa
yield # here the code runs
if emit:
PYEXTRAE.eventandcounters(event_group, 0) # noqa
def multinode_f(*args, **kwargs):
if not self.scope:
raise Exception(not_in_pycompss("MultiNode"))
if __debug__:
logger.debug("Executing multinode_f wrapper.")
if context.in_master():
# master code
if not self.core_element_configured:
self.__configure_core_element__(kwargs)
else:
# worker code
set_slurm_environment()
# Set the computing_nodes variable in kwargs for its usage
# in @task decorator
kwargs['computing_nodes'] = self.kwargs['computing_nodes']
with keep_arguments(args, kwargs, prepend_strings=True):
# Call the method
ret = func(*args, **kwargs)
if context.in_worker():
reset_slurm_environment()
return ret
def __decorator_body__(self, user_function, args, kwargs):
# Determine the context and decide what to do
if context.in_master():
# @task being executed in the master
# Each task will have a TaskMaster, so its content will
# not be shared.
self.__check_core_element__(kwargs, user_function)
with event(TASK_INSTANTIATION, master=True):
master = TaskMaster(self.decorator_arguments,
self.user_function, self.core_element,
self.registered, self.signature,
self.interactive, self.module,
self.function_arguments,
self.function_name, self.module_name,
self.function_type, self.class_name,
self.hints, self.on_failure, self.defaults)
result = master.call(*args, **kwargs)
fo, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints = result # noqa: E501
del master
return fo
elif context.in_worker():
if "compss_key" in kwargs.keys():
if context.is_nesting_enabled():
# Update the whole logger since it will be in job out/err
update_logger_handlers(kwargs["compss_log_cfg"],
kwargs["compss_log_files"][0],
kwargs["compss_log_files"][1])
# @task being executed in the worker
with event(WORKER_TASK_INSTANTIATION,
master=False,
inside=True):
worker = TaskWorker(self.decorator_arguments,
self.user_function, self.on_failure,
self.defaults)
result = worker.call(*args, **kwargs)
# Force flush stdout and stderr
sys.stdout.flush()
sys.stderr.flush()
# Remove worker
del worker
if context.is_nesting_enabled():
# Wait for all nested tasks to finish
from pycompss.runtime.binding import nested_barrier
nested_barrier()
# Reestablish logger handlers
update_logger_handlers(kwargs["compss_log_cfg"])
return result
else:
if context.is_nesting_enabled():
# Each task will have a TaskMaster, so its content will
# not be shared.
with event(TASK_INSTANTIATION, master=True):
master = TaskMaster(
self.decorator_arguments, self.user_function,
self.core_element, self.registered, self.signature,
self.interactive, self.module,
self.function_arguments, self.function_name,
self.module_name, self.function_type,
self.class_name, self.hints, self.on_failure,
self.defaults)
result = master.call(*args, **kwargs)
fo, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints = result # noqa: E501
del master
return fo
else:
# Called from another task within the worker
# Ignore the @task decorator and run it sequentially
message = "".join((
"WARNING: Calling task: ",
str(user_function.__name__),
" from this task.\n",
" It will be executed sequentially ", # noqa: E501
"within the caller task."))
print(message, file=sys.stderr)
return self._sequential_call(*args, **kwargs)
# We are neither in master nor in the worker, or the user has
# stopped the interactive session.
# Therefore, the user code is being executed with no
# launch_compss/enqueue_compss/runcompss/interactive session
return self._sequential_call(*args, **kwargs)
def __init__(self, event_id):
# type: (int) -> None
self.emitted = False
if TRACING and in_worker():
PYEXTRAE.eventandcounters(INSIDE_TASKS_TYPE, event_id) # noqa
self.emitted = True
def _synchronize(obj, mode):
# type: (typing.Any, int) -> typing.Any
""" Synchronization function.
This method retrieves the value of a future object.
Calls the runtime in order to wait for the value and returns it when
received.
:param obj: Object to synchronize.
:param mode: Direction of the object to synchronize.
:return: The value of the object requested.
"""
# TODO: Add a boolean to differentiate between files and object on the
# COMPSs.open_file call. This change pretends to obtain better traces.
# Must be implemented first in the Runtime, then in the bindings common
# C API and finally add the boolean here
app_id = 0
obj_id = "" # noqa
if is_psco(obj):
obj_id = str(get_id(obj))
if not OT.is_pending_to_synchronize(obj_id):
return obj
else:
# file_path is of the form storage://pscoId or
# file://sys_path_to_file
file_path = COMPSs.open_file(app_id, "".join(
("storage://", str(obj_id))), mode)
# TODO: Add switch on protocol (first parameter returned currently ignored)
_, file_name = file_path.split("://")
new_obj = get_by_id(file_name)
OT.stop_tracking(obj)
return new_obj
obj_id = OT.is_tracked(obj)
obj_name = OT.get_obj_name(obj_id)
if obj_id is None: # Not being tracked
return obj
if not OT.is_pending_to_synchronize(obj_id):
return obj
if __debug__:
logger.debug("Synchronizing object %s with mode %s" % (obj_id, mode))
file_name = OT.get_file_name(obj_id)
compss_file = COMPSs.open_file(app_id, file_name, mode)
# Runtime can return a path or a PSCOId
if compss_file.startswith('/'):
# If the real filename is null, then return None. The task that
# produces the output file may have been ignored or cancelled, so its
# result does not exist.
real_file_name = compss_file.split('/')[-1]
if real_file_name == "null":
print(
"WARNING: Could not retrieve the object " + str(file_name) +
" since the task that produces it may have been IGNORED or CANCELLED. Please, check the logs. Returning None."
) # noqa: E501
return None
new_obj = deserialize_from_file(compss_file)
COMPSs.close_file(app_id, file_name, mode)
else:
new_obj = get_by_id(compss_file)
if context.is_nesting_enabled() and context.in_worker():
# If nesting and in worker, the user wants to synchronize an object.
# So it is necessary to update the parameter with the new object.
update_worker_argument_parameter_content(obj_name, new_obj)
if mode == 'r':
OT.update_mapping(obj_id, new_obj)
if mode != 'r':
COMPSs.delete_file(app_id, OT.get_file_name(obj_id), False)
OT.stop_tracking(obj)
return new_obj
def task_decorator(*args, **kwargs):
# Determine the context and decide what to do
if context.in_master():
# @task being executed in the master
# Each task will have a TaskMaster, so its content will
# not be shared.
self.__check_core_element__(kwargs, user_function)
with event(TASK_INSTANTIATION, master=True):
master = TaskMaster(
self.decorator_arguments, self.user_function,
self.core_element, self.registered, self.signature,
self.interactive, self.module, self.function_arguments,
self.function_name, self.module_name,
self.function_type, self.class_name, self.hints)
result = master.call(*args, **kwargs)
fo, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints = result # noqa: E501
del master
return fo
elif context.in_worker():
if 'compss_key' in kwargs.keys():
# @task being executed in the worker
with event(WORKER_TASK_INSTANTIATION,
master=False,
inside=True):
worker = TaskWorker(self.decorator_arguments,
self.user_function)
result = worker.call(*args, **kwargs)
del worker
return result
else:
if context.is_nesting_enabled():
# nested @task executed in the worker
# Each task will have a TaskMaster, so its content will
# not be shared.
with event(TASK_INSTANTIATION, master=True):
master = TaskMaster(
self.decorator_arguments, self.user_function,
self.core_element, self.registered,
self.signature, self.interactive, self.module,
self.function_arguments, self.function_name,
self.module_name, self.function_type,
self.class_name, self.hints)
result = master.call(*args, **kwargs)
fo, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints = result # noqa: E501
del master
return fo
else:
# Called from another task within the worker
# Ignore the @task decorator and run it sequentially
message = "".join((
"WARNING: Calling task: ",
str(user_function.__name__),
" from this task.\n",
" It will be executed sequentially ", # noqa: E501
"within the caller task."))
print(message, file=sys.stderr)
return self._sequential_call(*args, **kwargs)
# We are neither in master nor in the worker, or the user has
# stopped the interactive session.
# Therefore, the user code is being executed with no
# launch_compss/enqueue_compss/runcompss/interactive session
return self._sequential_call(*args, **kwargs)