def delete_object(obj):
# type: (object) -> bool
""" Remove object.
Removes a used object from the internal structures and calls the
external python library (that calls the bindings-common)
in order to request a its corresponding file removal.
:param obj: Object to remove.
:return: True if success. False otherwise.
"""
with event_master(DELETE_OBJECT_EVENT):
app_id = 0
obj_id = OT.is_tracked(obj)
if obj_id is None:
# Not being tracked
return False
else:
try:
file_name = OT.get_file_name(obj_id)
COMPSs.delete_file(app_id, file_name, False)
OT.stop_tracking(obj)
except KeyError:
pass
return True
def close_task_group(group_name):
# type: (str) -> None
""" Close task group.
Calls the external python library (that calls the bindings-common)
in order to request a group closure.
:param group_name: Group name.
:return: None
"""
app_id = 0
COMPSs.close_task_group(group_name, app_id)
def open_task_group(group_name, implicit_barrier):
# type: (str, bool) -> None
""" Open task group.
Calls the external python library (that calls the bindings-common)
in order to request an opening of a group.
:param group_name: Group name.
:param implicit_barrier: Perform a wait on all group tasks before closing.
:return: None
"""
app_id = 0
COMPSs.open_task_group(group_name, implicit_barrier, app_id)
def stop_runtime(code=0, hard_stop=False):
# type: (int, bool) -> None
""" Stops the COMPSs runtime.
Stops the runtime by calling the external python library that calls
the bindings-common.
Also cleans objects and temporary files created during runtime.
If the code is different from 0, all running or waiting tasks will be
cancelled.
:parameter code: Stop code (if code != 0 ==> cancel application tasks).
:param hard_stop: Stop compss when runtime has died.
:return: None
"""
with event_master(STOP_RUNTIME_EVENT):
app_id = 0
if __debug__:
logger.info("Stopping runtime...")
# Stopping a possible wall clock limit
signal.alarm(0)
if code != 0:
if __debug__:
logger.info("Canceling all application tasks...")
COMPSs.cancel_application_tasks(app_id, 0)
if __debug__:
logger.info("Cleaning objects...")
_clean_objects(hard_stop=hard_stop)
if __debug__:
reporting = OT.is_report_enabled()
if reporting:
logger.info("Generating Object tracker report...")
target_path = get_log_path()
OT.generate_report(target_path)
OT.clean_report()
if __debug__:
logger.info("Stopping COMPSs...")
COMPSs.stop_runtime(code)
if __debug__:
logger.info("Cleaning temps...")
_clean_temps()
context.set_pycompss_context(context.OUT_OF_SCOPE)
if __debug__:
logger.info("COMPSs stopped")
def get_directory(dir_name):
# type: (str) -> None
""" Retrieve a directory.
Calls the external python library (that calls the bindings-common)
in order to request last version of file.
:param dir_name: dir name to retrieve.
:return: None
"""
app_id = 0
if __debug__:
logger.debug("Getting directory %s" % dir_name)
COMPSs.get_directory(app_id, dir_name)
def start_runtime(log_level='off', tracing=0, interactive=False):
# type: (str, int, bool) -> None
""" Starts the COMPSs runtime.
Starts the runtime by calling the external python library that calls
the bindings-common.
:param log_level: Log level [ 'trace' | 'debug' | 'info' | 'api' | 'off' ].
:param tracing: Tracing level [0 (deactivated) | 1 (basic) | 2 (advanced)].
:param interactive: Boolean if interactive (ipython or jupyter).
:return: None
"""
if __debug__:
logger.info("Starting COMPSs...")
if tracing > 0 and not interactive:
# Enabled only if not interactive - extrae issues within jupyter.
enable_trace_master()
with event(START_RUNTIME_EVENT, master=True):
if interactive and context.in_master():
COMPSs.load_runtime(external_process=True)
else:
COMPSs.load_runtime(external_process=False)
if log_level == 'trace':
# Could also be 'debug' or True, but we only show the C extension
# debug in the maximum tracing level.
COMPSs.set_debug(True)
OT_enable_report()
COMPSs.start_runtime()
if __debug__:
logger.info("COMPSs started")
def get_file(file_name):
# type: (str) -> None
""" Retrieve a file.
Calls the external python library (that calls the bindings-common)
in order to request last version of file.
:param file_name: File name to remove.
:return: None
"""
app_id = 0
if __debug__:
logger.debug("Getting file %s" % file_name)
COMPSs.get_file(app_id, file_name)
def _clean_objects():
# type: () -> None
""" Clean all objects.
Clean the objects stored in the global dictionaries:
- pending_to_synchronize dict.
- _addr2id2obj dict.
- obj_id_to_filename dict.
- _objs_written_by_mp dict.
:return: None
"""
app_id = 0
for filename in OT_get_all_file_names():
COMPSs.delete_file(app_id, filename, False)
OT_clean_object_tracker()
def set_wall_clock(wall_clock_limit):
# type: (int) -> None
""" Sets the application wall clock limit.
:param wall_clock_limit: Wall clock limit in seconds.
:return: None
"""
with event_master(WALL_CLOCK_LIMIT_EVENT):
app_id = 0
if __debug__:
logger.debug("Set a wall clock limit of " + str(wall_clock_limit))
# Activate wall clock limit alarm
signal.signal(signal.SIGALRM, _wall_clock_exceed)
signal.alarm(wall_clock_limit)
# Call the Runtime to set a timer in case wall clock is reached in a synch
COMPSs.set_wall_clock(app_id, wall_clock_limit)
def set_wall_clock(wall_clock_limit):
# type: (long) -> node
""" Sets the application wall clock limit.
:param wall_clock_limit: Wall clock limit in seconds.
:return: None
"""
app_id = 0
if __debug__:
logger.debug("Set a wall clock limit of " + str(wall_clock_limit))
# Activate wall clock limit alarm
signal.signal(signal.SIGALRM, _wall_clock_exceed)
signal.alarm(wall_clock_limit)
# Call the Runtime to set a timer in case wall clock is reached in a synch
COMPSs.set_wall_clock(app_id, wall_clock_limit)
def _clean_objects(hard_stop=False):
# type: (bool) -> None
""" Clean all objects.
Clean the objects stored in the global dictionaries:
- pending_to_synchronize dict.
- _addr2id2obj dict.
- obj_id_to_filename dict.
- _objs_written_by_mp dict.
:param hard_stop: avoid call to delete_file when the runtime has died.
:return: None
"""
app_id = 0
if not hard_stop:
for filename in OT_get_all_file_names():
COMPSs.delete_file(app_id, filename, False)
OT_clean_object_tracker()
def stop_runtime(code=0):
# type: (int) -> None
""" Stops the COMPSs runtime.
Stops the runtime by calling the external python library that calls
the bindings-common.
Also cleans objects and temporary files created during runtime.
If the code is different from 0, all running or waiting tasks will be
cancelled.
:parameter code: Stop code (if code != 0 ==> cancel application tasks).
:return: None
"""
app_id = 0
if __debug__:
logger.info("Stopping runtime...")
if code != 0:
if __debug__:
logger.info("Canceling all application tasks...")
COMPSs.cancel_application_tasks(app_id, 0)
if __debug__:
logger.info("Cleaning objects...")
_clean_objects()
if __debug__:
reporting = OT_is_report_enabled()
if reporting:
logger.info("Generating Object tracker report...")
target_path = get_log_path()
OT_generate_report(target_path)
OT_clean_report()
if __debug__:
logger.info("Stopping COMPSs...")
COMPSs.stop_runtime(code)
if __debug__:
logger.info("Cleaning temps...")
_clean_temps()
if __debug__:
logger.info("COMPSs stopped")
def barrier(no_more_tasks=False):
# type: (bool) -> None
""" Wait for all submitted tasks.
Calls the external python library (that calls the bindings-common)
in order to request a barrier.
:param no_more_tasks: If no more tasks are going to be submitted, remove
all objects.
:return: None
"""
if __debug__:
logger.debug("Barrier. No more tasks? %s" % str(no_more_tasks))
# If noMoreFlags is set, clean up the objects
if no_more_tasks:
_clean_objects()
app_id = 0
# Call the Runtime barrier (appId 0, not needed for the signature)
COMPSs.barrier(app_id, no_more_tasks)
def request_resources(num_resources, group_name):
# type: (int, str) -> None
""" Request new resources.
Calls the external python library (that calls the bindings-common)
in order to request for the creation of the given resources.
:param num_resources: Number of resources to create.
:param group_name: Task group to notify upon resource creation.
:return: None
"""
app_id = 0
if group_name is None:
group_name = "NULL"
if __debug__:
logger.debug("Request the creation of " + str(num_resources) +
" resources with notification to task group " +
str(group_name))
# Call the Runtime
COMPSs.request_resources(app_id, num_resources, group_name)
def barrier_group(group_name):
# type: (str) -> str
""" Wait for all tasks of the given group.
Calls the external python library (that calls the bindings-common)
in order to request a barrier of a group.
:param group_name: Group name.
:return: None or string with exception message.
"""
app_id = 0
# Call the Runtime group barrier
return COMPSs.barrier_group(app_id, group_name)
def nested_barrier():
# type: () -> None
""" Wait for all submitted tasks within nested task.
Calls the external python library (that calls the bindings-common)
in order to request a barrier.
CAUTION:
When using agents (nesting), we can not remove all object tracker objects
as with normal barrier (and no_more_tasks==True), nor leave all objects
with (no_more_tasks==False). In this case, it is necessary to perform a
smart object tracker cleanup (remove in, but not inout nor out).
:return: None
"""
if __debug__:
logger.debug("Nested Barrier.")
_clean_objects()
# Call the Runtime barrier (appId 0 -- not needed for the signature, and
# no_more_tasks == True)
COMPSs.barrier(0, True)
def free_resources(num_resources, group_name):
# type: (int, str) -> None
""" Liberate resources.
Calls the external python library (that calls the bindings-common)
in order to request for the destruction of the given resources.
:param num_resources: Number of resources to destroy.
:param group_name: Task group to notify upon resource creation.
:return: None
"""
with event_master(FREE_RESOURCES_EVENT):
app_id = 0
if group_name is None:
group_name = "NULL"
if __debug__:
logger.debug("Request the destruction of " + str(num_resources) +
" resources with notification to task group " +
str(group_name))
# Call the Runtime
COMPSs.free_resources(app_id, num_resources, group_name)
def accessed_file(file_name):
# type: (str) -> bool
""" Check if the file has been accessed.
Calls the external python library (that calls the bindings-common)
in order to check if a file has been accessed.
:param file_name: <String> File name.
:return: True if accessed, False otherwise.
"""
app_id = 0
if __debug__:
logger.debug("Checking if file %s has been accessed." % file_name)
return COMPSs.accessed_file(app_id, file_name)
def get_number_of_resources():
# type: () -> int
""" Get the number of resources.
Calls the external python library (that calls the bindings-common)
in order to request for the number of active resources.
:return: Number of active resources.
"""
app_id = 0
if __debug__:
logger.debug("Request the number of active resources")
# Call the Runtime
return COMPSs.get_number_of_resources(app_id)
def get_log_path():
# type: () -> str
""" Get logging path.
Requests the logging path to the external python library (that calls
the bindings-common).
:return: The path where to store the logs.
"""
if __debug__:
logger.debug("Requesting log path")
log_path = COMPSs.get_logging_path()
if __debug__:
logger.debug("Log path received: %s" % log_path)
return log_path
def accessed_file(file_name):
# type: (str) -> bool
""" Check if the file has been accessed.
Calls the external python library (that calls the bindings-common)
in order to check if a file has been accessed.
:param file_name: <String> File name.
:return: True if accessed, False otherwise.
"""
with event_master(ACCESSED_FILE_EVENT):
app_id = 0
if __debug__:
logger.debug("Checking if file %s has been accessed." % file_name)
if os.path.exists(file_name):
return True
else:
return COMPSs.accessed_file(app_id, file_name)
def open_file(file_name, mode):
# type: (str, str) -> str
""" Opens a file (retrieves if necessary).
Calls the external python library (that calls the bindings-common)
in order to request a file.
:param file_name: <String> File name.
:param mode: Open file mode ('r', 'rw', etc.).
:return: The current name of the file requested (that may have been
renamed during runtime).
"""
app_id = 0
compss_mode = get_compss_direction(mode)
if __debug__:
logger.debug("Getting file %s with mode %s" % (file_name, compss_mode))
compss_name = COMPSs.open_file(app_id, file_name, compss_mode)
if __debug__:
logger.debug("COMPSs file name is %s" % compss_name)
return compss_name
def delete_file(file_name):
# type: (str) -> bool
""" Remove a file.
Calls the external python library (that calls the bindings-common)
in order to request a file removal.
:param file_name: File name to remove.
:return: True if success. False otherwise.
"""
app_id = 0
if __debug__:
logger.debug("Deleting file %s" % file_name)
result = COMPSs.delete_file(app_id, file_name, True) == 'true'
if __debug__:
if result:
logger.debug("File %s successfully deleted." % file_name)
else:
logger.error("Failed to remove file %s." % file_name)
return result
def executor(queue, process_name, pipe, conf):
# type: (typing.Union[None, Queue], str, Pipe, typing.Any) -> None
"""Thread main body - Overrides Threading run method.
Iterates over the input pipe in order to receive tasks (with their
parameters) and process them.
Notifies the runtime when each task has finished with the
corresponding output value.
Finishes when the "quit" message is received.
:param queue: Queue where to put exception messages.
:param process_name: Process name (Thread-X, where X is the thread id).
:param pipe: Pipe to receive and send messages from/to the runtime.
:param conf: configuration of the executor.
:return: None
"""
try:
# Replace Python Worker's SIGTERM handler.
signal.signal(signal.SIGTERM, shutdown_handler)
if len(conf.logger.handlers) == 0:
# Logger has not been inherited correctly. Happens in MacOS.
set_temporary_directory(conf.tmp_dir, create_tmpdir=False)
# Reload logger
conf.logger, conf.logger_cfg, conf.storage_loggers, _ = \
load_loggers(conf.debug, conf.persistent_storage)
# Set the binding in worker mode too
context.set_pycompss_context(context.WORKER)
logger = conf.logger
tracing = conf.tracing
storage_conf = conf.storage_conf
storage_loggers = conf.storage_loggers
# Get a copy of the necessary information from the logger to
# re-establish after each task
logger_handlers = copy.copy(logger.handlers)
logger_level = logger.getEffectiveLevel()
logger_formatter = logging.Formatter(
logger_handlers[0].formatter._fmt) # noqa
storage_loggers_handlers = []
for storage_logger in storage_loggers:
storage_loggers_handlers.append(copy.copy(storage_logger.handlers))
# Establish link with the binding-commons to enable task nesting
if __debug__:
logger.debug(HEADER +
"Establishing link with runtime in process " +
str(process_name)) # noqa: E501
COMPSs.load_runtime(external_process=False, _logger=logger)
COMPSs.set_pipes(pipe.output_pipe, pipe.input_pipe)
if storage_conf != "null":
try:
from storage.api import initWorkerPostFork # noqa
with event_worker(INIT_WORKER_POSTFORK_EVENT):
initWorkerPostFork()
except (ImportError, AttributeError):
if __debug__:
logger.info(
HEADER +
"[%s] Could not find initWorkerPostFork storage call. Ignoring it."
% # noqa: E501
str(process_name))
# Start the streaming backend if necessary
streaming = False
if conf.stream_backend not in [None, "null", "NONE"]:
streaming = True
if streaming:
# Initialize streaming
logger.debug(HEADER + "Starting streaming for process " +
str(process_name))
try:
DistroStreamClientHandler.init_and_start(
master_ip=conf.stream_master_ip,
master_port=conf.stream_master_port)
except Exception as e:
logger.error(e)
raise e
# Connect to Shared memory manager
if conf.cache_queue:
load_shared_memory_manager()
# Process properties
alive = True
if __debug__:
logger.debug(HEADER + "[%s] Starting process" % str(process_name))
# MAIN EXECUTOR LOOP
while alive:
# Runtime -> pipe - Read command from pipe
command = COMPSs.read_pipes()
if command != "":
if __debug__:
logger.debug(HEADER + "[%s] Received command %s" %
(str(process_name), str(command)))
# Process the command
alive = process_message(
command, process_name, pipe, queue, tracing, logger,
conf.logger_cfg, logger_handlers, logger_level,
logger_formatter, storage_conf, storage_loggers,
storage_loggers_handlers, conf.cache_queue, conf.cache_ids,
conf.cache_profiler)
# Stop storage
if storage_conf != "null":
try:
from storage.api import finishWorkerPostFork # noqa
with event_worker(FINISH_WORKER_POSTFORK_EVENT):
finishWorkerPostFork()
except (ImportError, AttributeError):
if __debug__:
logger.info(
HEADER +
"[%s] Could not find finishWorkerPostFork storage call. Ignoring it."
% # noqa: E501
str(process_name))
# Stop streaming
if streaming:
logger.debug(HEADER + "Stopping streaming for process " +
str(process_name))
DistroStreamClientHandler.set_stop()
sys.stdout.flush()
sys.stderr.flush()
if __debug__:
logger.debug(HEADER + "[%s] Exiting process " % str(process_name))
pipe.write(QUIT_TAG)
pipe.close()
except Exception as e:
logger.error(e)
raise e
def process_task(
signature, # type: str
has_target, # type: bool
names, # type: list
values, # type: list
num_returns, # type: int
compss_types, # type: list
compss_directions, # type: list
compss_streams, # type: list
compss_prefixes, # type: list
content_types, # type: list
weights, # type: list
keep_renames, # type: list
has_priority, # type: bool
num_nodes, # type: int
reduction, # type: bool
chunk_size, # type: int
replicated, # type: bool
distributed, # type: bool
on_failure, # type: str
time_out, # type: int
): # NOSONAR
# type: (...) -> None
""" Submit a task to the runtime.
:param signature: Task signature
:param has_target: Boolean if the task has self
:param names: Task parameter names
:param values: Task parameter values
:param num_returns: Number of returns
:param compss_types: List of parameter types
:param compss_directions: List of parameter directions
:param compss_streams: List of parameter streams
:param compss_prefixes: List of parameter prefixes
:param content_types: Content types
:param weights: List of parameter weights
:param keep_renames: Boolean keep renaming
:param has_priority: Boolean has priority
:param num_nodes: Number of nodes that the task must use
:param reduction: Boolean indicating if the task is of type reduce
:param chunk_size: Size of chunks for executing the reduce operation
:param replicated: Boolean indicating if the task must be replicated
:param distributed: Boolean indicating if the task must be distributed
:param on_failure: Action on failure
:param time_out: Time for a task time out
:return: The future object related to the task return
"""
app_id = 0
if __debug__:
# Log the task submission values for debugging purposes.
values_str = ' '.join(str(v) for v in values)
types_str = ' '.join(str(t) for t in compss_types)
direct_str = ' '.join(str(d) for d in compss_directions)
streams_str = ' '.join(str(s) for s in compss_streams)
prefixes_str = ' '.join(str(p) for p in compss_prefixes)
names_str = ' '.join(x for x in names)
ct_str = ' '.join(str(x) for x in content_types)
weights_str = ' '.join(str(x) for x in weights)
keep_renames_str = ' '.join(str(x) for x in keep_renames)
logger.debug("Processing task:")
logger.debug("\t- App id: " + str(app_id))
logger.debug("\t- Signature: " + signature)
logger.debug("\t- Has target: " + str(has_target))
logger.debug("\t- Names: " + names_str)
logger.debug("\t- Values: " + values_str)
logger.debug("\t- COMPSs types: " + types_str)
logger.debug("\t- COMPSs directions: " + direct_str)
logger.debug("\t- COMPSs streams: " + streams_str)
logger.debug("\t- COMPSs prefixes: " + prefixes_str)
logger.debug("\t- Content Types: " + ct_str)
logger.debug("\t- Weights: " + weights_str)
logger.debug("\t- Keep_renames: " + keep_renames_str)
logger.debug("\t- Priority: " + str(has_priority))
logger.debug("\t- Num nodes: " + str(num_nodes))
logger.debug("\t- Reduce: " + str(reduction))
logger.debug("\t- Chunk Size: " + str(chunk_size))
logger.debug("\t- Replicated: " + str(replicated))
logger.debug("\t- Distributed: " + str(distributed))
logger.debug("\t- On failure behavior: " + on_failure)
logger.debug("\t- Task time out: " + str(time_out))
# Check that there is the same amount of values as their types, as well
# as their directions, streams and prefixes.
assert (len(values) == len(compss_types) == len(compss_directions) ==
len(compss_streams) == len(compss_prefixes) == len(content_types)
== len(weights) == len(keep_renames))
# Submit task to the runtime (call to the C extension):
# Parameters:
# 0 - <Integer> - application id (by default always 0 due to it is
# not currently needed for the signature)
# 1 - <String> - path of the module where the task is
#
# 2 - <String> - behavior if the task fails
#
# 3 - <String> - function name of the task (to be called from the
# worker)
# 4 - <String> - priority flag (true|false)
#
# 5 - <String> - has target (true|false). If the task is within an
# object or not.
# 6 - [<String>] - task parameters (basic types or file paths for
# objects)
# 7 - [<Integer>] - parameters types (number corresponding to the type
# of each parameter)
# 8 - [<Integer>] - parameters directions (number corresponding to the
# direction of each parameter)
# 9 - [<Integer>] - parameters streams (number corresponding to the
# stream of each parameter)
# 10 - [<String>] - parameters prefixes (string corresponding to the
# prefix of each parameter)
# 11 - [<String>] - parameters extra type (string corresponding to the
# extra type of each parameter)
# 12 - [<String>] - parameters weights (string corresponding to the
# weight of each parameter
# 13 - <String> - Keep renames flag (true|false)
#
COMPSs.process_task(app_id, signature, on_failure, time_out, has_priority,
num_nodes, reduction, chunk_size, replicated,
distributed, has_target, num_returns, values, names,
compss_types, compss_directions, compss_streams,
compss_prefixes, content_types, weights, keep_renames)
def register_ce(core_element): # noqa
# type: (CE) -> None
""" Register a core element.
Calls the external python library (that calls the bindings-common)
in order to notify the runtime about a core element that needs to be
registered.
Java Examples:
// METHOD
System.out.println('Registering METHOD implementation');
String core_elementSignature = 'methodClass.methodName';
String impl_signature = 'methodClass.methodName';
String impl_constraints = 'ComputingUnits:2';
String impl_type = 'METHOD';
String[] impl_type_args = new String[] { 'methodClass', 'methodName' };
rt.registerCoreElement(coreElementSignature, impl_signature, impl_constraints, impl_type, impl_type_args); # noqa: E501
// MPI
System.out.println('Registering MPI implementation');
core_elementSignature = 'methodClass1.methodName1';
impl_signature = 'mpi.MPI';
impl_constraints = 'StorageType:SSD';
impl_type = 'MPI';
impl_type_args = new String[] { 'mpiBinary', 'mpiWorkingDir', 'mpiRunner' }; # noqa: E501
rt.registerCoreElement(coreElementSignature, impl_signature, impl_constraints, impl_type, impl_type_args); # noqa: E501
// PYTHON MPI
System.out.println('Registering PYTHON MPI implementation');
core_elementSignature = 'methodClass1.methodName1';
impl_signature = 'MPI.methodClass1.methodName';
impl_constraints = 'ComputingUnits:2';
impl_type = 'PYTHON_MPI';
impl_type_args = new String[] { 'methodClass', 'methodName', 'mpiWorkingDir', 'mpiRunner' }; # noqa: E501
rt.registerCoreElement(coreElementSignature, impl_signature, impl_constraints, impl_type, impl_type_args); # noqa: E501
// BINARY
System.out.println('Registering BINARY implementation');
core_elementSignature = 'methodClass2.methodName2';
impl_signature = 'binary.BINARY';
impl_constraints = 'MemoryType:RAM';
impl_type = 'BINARY';
impl_type_args = new String[] { 'binary', 'binaryWorkingDir' };
rt.registerCoreElement(coreElementSignature, impl_signature, impl_constraints, impl_type, impl_type_args); # noqa: E501
// OMPSS
System.out.println('Registering OMPSS implementation');
core_elementSignature = 'methodClass3.methodName3';
impl_signature = 'ompss.OMPSS';
impl_constraints = 'ComputingUnits:3';
impl_type = 'OMPSS';
impl_type_args = new String[] { 'ompssBinary', 'ompssWorkingDir' };
rt.registerCoreElement(coreElementSignature, impl_signature, impl_constraints, impl_type, impl_type_args); # noqa: E501
// OPENCL
System.out.println('Registering OPENCL implementation');
core_elementSignature = 'methodClass4.methodName4';
impl_signature = 'opencl.OPENCL';
impl_constraints = 'ComputingUnits:4';
impl_type = 'OPENCL';
impl_type_args = new String[] { 'openclKernel', 'openclWorkingDir' };
rt.registerCoreElement(coreElementSignature, impl_signature, impl_constraints, impl_type, impl_type_args); # noqa: E501
// VERSIONING
System.out.println('Registering METHOD implementation');
core_elementSignature = 'methodClass.methodName';
impl_signature = 'anotherClass.anotherMethodName';
impl_constraints = 'ComputingUnits:1';
impl_type = 'METHOD';
impl_type_args = new String[] { 'anotherClass', 'anotherMethodName' };
rt.registerCoreElement(coreElementSignature, impl_signature, impl_constraints, impl_type, impl_type_args); # noqa: E501
---------------------
Core Element fields:
ce_signature: <String> Core Element signature (e.g.- 'methodClass.methodName') # noqa: E501
impl_signature: <String> Implementation signature (e.g.- 'methodClass.methodName') # noqa: E501
impl_constraints: <Dict> Implementation constraints (e.g.- '{ComputingUnits:2}') # noqa: E501
impl_type: <String> Implementation type ('METHOD' | 'MPI' | 'BINARY' | 'OMPSS' | 'OPENCL') # noqa: E501
impl_io: <String> IO Implementation
impl_type_args: <List(Strings)> Implementation arguments (e.g.- ['methodClass', 'methodName']) # noqa: E501
:param core_element: <CE> Core Element to register.
:return: None
"""
# Retrieve Core element fields
ce_signature = core_element.get_ce_signature()
impl_signature = core_element.get_impl_signature()
impl_constraints = core_element.get_impl_constraints()
impl_type = core_element.get_impl_type()
impl_io = str(core_element.get_impl_io())
impl_type_args = core_element.get_impl_type_args()
if __debug__:
logger.debug("Registering CE with signature: %s" % ce_signature)
logger.debug("\t - Implementation signature: %s" % impl_signature)
# Build constraints string from constraints dictionary
impl_constraints_lst = []
for key, value in impl_constraints.items():
val = value
if isinstance(value, list):
val = str(value).replace('\'', '')
kv_constraint = "".join((key, ':', str(val), ';'))
impl_constraints_lst.append(kv_constraint)
impl_constraints_str = "".join(impl_constraints_lst)
if __debug__:
logger.debug("\t - Implementation constraints: %s" %
impl_constraints_str)
logger.debug("\t - Implementation type: %s" % impl_type)
logger.debug("\t - Implementation type arguments: %s" %
' '.join(impl_type_args))
# Call runtime with the appropriate parameters
COMPSs.register_core_element(ce_signature, impl_signature,
impl_constraints_str, impl_type, impl_io,
impl_type_args)
if __debug__:
logger.debug("CE with signature %s registered." % ce_signature)
def executor(queue, process_name, pipe, conf):
# type: (..., str, Pipe, ...) -> None
"""Thread main body - Overrides Threading run method.
Iterates over the input pipe in order to receive tasks (with their
parameters) and process them.
Notifies the runtime when each task has finished with the
corresponding output value.
Finishes when the "quit" message is received.
:param queue: Queue where to put exception messages.
:param process_name: Process name (Thread-X, where X is the thread id).
:param pipe: Pipe to receive and send messages from/to the runtime.
:param conf: configuration of the executor.
:return: None
"""
logger = conf.logger
try:
# Replace Python Worker's SIGTERM handler.
signal.signal(signal.SIGTERM, shutdown_handler)
tracing = conf.tracing
storage_conf = conf.storage_conf
storage_loggers = conf.storage_loggers
# Get a copy of the necessary information from the logger to
# re-establish after each task
logger_handlers = copy.copy(logger.handlers)
logger_level = logger.getEffectiveLevel()
logger_formatter = logging.Formatter(
logger_handlers[0].formatter._fmt) # noqa
storage_loggers_handlers = []
for storage_logger in storage_loggers:
storage_loggers_handlers.append(copy.copy(storage_logger.handlers))
# Establish link with the binding-commons to enable task nesting
if __debug__:
logger.debug(HEADER +
"Establishing link with runtime in process " +
str(process_name)) # noqa: E501
COMPSs.load_runtime(external_process=False, _logger=logger)
COMPSs.set_pipes(pipe.output_pipe, pipe.input_pipe)
if storage_conf != 'null':
try:
from storage.api import initWorkerPostFork # noqa
with event(INIT_WORKER_POSTFORK_EVENT):
initWorkerPostFork()
except ImportError:
if __debug__:
logger.info(
HEADER +
"[%s] Could not find initWorkerPostFork storage call. Ignoring it."
% # noqa: E501
str(process_name))
# Start the streaming backend if necessary
streaming = False
if conf.stream_backend not in [None, 'null', 'NONE']:
streaming = True
if streaming:
# Initialize streaming
logger.debug(HEADER + "Starting streaming for process " +
str(process_name))
try:
DistroStreamClientHandler.init_and_start(
master_ip=conf.stream_master_ip,
master_port=int(conf.stream_master_port))
except Exception as e:
logger.error(e)
raise e
# Process properties
alive = True
if __debug__:
logger.debug(HEADER + "[%s] Starting process" % str(process_name))
# MAIN EXECUTOR LOOP
while alive:
# Runtime -> pipe - Read command from pipe
command = pipe.read_command(retry_period=0.5)
if command != "":
if __debug__:
logger.debug(HEADER + "Received %s" % command)
# Process the command
alive = process_message(command, process_name, pipe, queue,
tracing, logger, logger_handlers,
logger_level, logger_formatter,
storage_conf, storage_loggers,
storage_loggers_handlers)
# Stop storage
if storage_conf != 'null':
try:
from storage.api import finishWorkerPostFork # noqa
with event(FINISH_WORKER_POSTFORK_EVENT):
finishWorkerPostFork()
except ImportError:
if __debug__:
logger.info(
HEADER +
"[%s] Could not find finishWorkerPostFork storage call. Ignoring it."
% # noqa: E501
str(process_name))
# Stop streaming
if streaming:
logger.debug(HEADER + "Stopping streaming for process " +
str(process_name))
DistroStreamClientHandler.set_stop()
sys.stdout.flush()
sys.stderr.flush()
if __debug__:
logger.debug(HEADER + "[%s] Exiting process " % str(process_name))
pipe.write(QUIT_TAG)
pipe.close()
except BaseException as e:
logger.error(e)
raise e
def _synchronize(obj, mode):
# type: (object, int) -> object
""" Synchronization function.
This method retrieves the value of a future object.
Calls the runtime in order to wait for the value and returns it when
received.
:param obj: Object to synchronize.
:param mode: Direction of the object to synchronize.
:return: The value of the object requested.
"""
# TODO: Add a boolean to differentiate between files and object on the
# COMPSs.open_file call. This change pretends to obtain better traces.
# Must be implemented first in the Runtime, then in the bindings common
# C API and finally add the boolean here
app_id = 0
if is_psco(obj):
obj_id = get_id(obj)
if not OT_is_pending_to_synchronize(obj_id):
return obj
else:
# file_path is of the form storage://pscoId or
# file://sys_path_to_file
file_path = COMPSs.open_file(app_id,
"".join(("storage://", str(obj_id))),
mode)
# TODO: Add switch on protocol (first parameter returned currently ignored)
_, file_name = file_path.split("://")
new_obj = get_by_id(file_name)
OT_stop_tracking(obj)
return new_obj
obj_id = OT_is_tracked(obj)
if obj_id is None: # Not being tracked
return obj
if not OT_is_pending_to_synchronize(obj_id):
return obj
if __debug__:
logger.debug("Synchronizing object %s with mode %s" % (obj_id, mode))
file_name = OT_get_file_name(obj_id)
compss_file = COMPSs.open_file(app_id, file_name, mode)
# Runtime can return a path or a PSCOId
if compss_file.startswith('/'):
# If the real filename is null, then return None. The task that
# produces the output file may have been ignored or cancelled, so its
# result does not exist.
real_file_name = compss_file.split('/')[-1]
if real_file_name == "null":
print("WARNING: Could not retrieve the object " + str(file_name) +
" since the task that produces it may have been IGNORED or CANCELLED. Please, check the logs. Returning None.") # noqa: E501
return None
new_obj = deserialize_from_file(compss_file)
COMPSs.close_file(app_id, file_name, mode)
else:
new_obj = get_by_id(compss_file)
if mode == 'r':
OT_update_mapping(obj_id, new_obj)
if mode != 'r':
COMPSs.delete_file(app_id, OT_get_file_name(obj_id), False)
OT_stop_tracking(obj)
return new_obj