def bind_cpus(cpus, process_name, logger): # noqa
# type: (str, str, typing.Any) -> bool
""" Bind the given CPUs for core affinity to this process.
:param cpus: Target CPUs.
:param process_name: Process name for logger messages.
:param logger: Logger.
:return: True if success, False otherwise.
"""
with event_inside_worker(BIND_CPUS_EVENT):
if __debug__:
logger.debug(HEADER + "[%s] Assigning affinity %s" %
(str(process_name), str(cpus)))
cpus_list = cpus.split(",")
cpus_map = list(map(int, cpus_list))
try:
thread_affinity.setaffinity(cpus_map)
except Exception: # noqa
if __debug__:
logger.error(HEADER +
"[%s] WARNING: could not assign affinity %s" %
(str(process_name), str(cpus_map)))
return False
# Export only if success
os.environ["COMPSS_BINDED_CPUS"] = cpus
return True
def import_user_module(path, logger):
# type: (str, typing.Any) -> typing.Any
""" Import the user module.
:param path: Path to the user module.
:param logger: Logger.
:return: The loaded module.
"""
with event_inside_worker(IMPORT_USER_MODULE_EVENT):
py_version = sys.version_info
if py_version >= (2, 7):
import importlib
module = importlib.import_module(path) # Python 2.7
if path.startswith(INTERACTIVE_FILE_NAME):
# Force reload in interactive mode. The user may have
# overwritten a function or task.
if py_version < (3, 4):
import imp # noqa
imp.reload(module) # noqa
else:
importlib.reload(module)
if __debug__:
msg = "Module successfully loaded (Python version >= 2.7)"
logger.debug(msg)
else:
module = __import__(path, globals(), locals(), [path], -1)
if __debug__:
msg = "Module successfully loaded (Python version < 2.7"
logger.debug(msg)
return module
def retrieve_object_from_cache(logger, cache_ids, cache_queue, identifier,
parameter_name, user_function,
cache_profiler): # noqa
# type: (typing.Any, typing.Any, Queue, str, str, typing.Callable, bool) -> typing.Any
""" Retrieve an object from the given cache proxy dict.
:param logger: Logger where to push messages.
:param cache_ids: Cache proxy dictionary.
:param cache_queue: Cache notification queue.
:param identifier: Object identifier.
:param parameter_name: Parameter name.
:param user_function: Function name.
:param cache_profiler: If cache profiling is enabled.
:return: The object from cache.
"""
with event_inside_worker(RETRIEVE_OBJECT_FROM_CACHE_EVENT):
emit_manual_event_explicit(BINDING_DESERIALIZATION_CACHE_SIZE_TYPE, 0)
identifier = __get_file_name__(identifier)
if __debug__:
logger.debug(HEADER + "Retrieving: " + str(identifier))
obj_id, obj_shape, obj_d_type, _, obj_hits, shared_type = cache_ids[
identifier] # noqa: E501
output = None # type: typing.Any
existing_shm = None # type: typing.Any
object_size = 0
if shared_type == SHARED_MEMORY_TAG:
existing_shm = SharedMemory(name=obj_id)
output = np.ndarray(obj_shape,
dtype=obj_d_type,
buffer=existing_shm.buf) # noqa: E501
object_size = len(existing_shm.buf)
elif shared_type == SHAREABLE_LIST_TAG:
existing_shm = ShareableList(name=obj_id)
output = list(existing_shm)
object_size = len(existing_shm.shm.buf)
elif shared_type == SHAREABLE_TUPLE_TAG:
existing_shm = ShareableList(name=obj_id)
output = tuple(existing_shm)
object_size = len(existing_shm.shm.buf)
# Currently unsupported since conversion requires lists of lists.
# elif shared_type == SHAREABLE_DICT_TAG:
# existing_shm = ShareableList(name=obj_id)
# output = dict(existing_shm)
else:
raise PyCOMPSsException("Unknown cacheable type.")
if __debug__:
logger.debug(HEADER + "Retrieved: " + str(identifier))
emit_manual_event_explicit(BINDING_DESERIALIZATION_CACHE_SIZE_TYPE,
object_size)
# Profiling
filename = filename_cleaned(identifier)
function_name = function_cleaned(user_function)
if cache_profiler:
cache_queue.put(("GET", (filename, parameter_name, function_name)))
# Add hit
cache_ids[identifier][4] = obj_hits + 1
return output, existing_shm
def get_by_id(id):
# type: (str) -> typing.Any
""" Retrieve the object from the given identifier.
:param id: Persistent object identifier.
:return: object associated to the persistent object identifier.
"""
with event_inside_worker(GET_BY_ID_EVENT):
return GET_BY_ID(id)
def get_id(psco):
# type: (typing.Any) -> typing.Union[str, None]
""" Retrieve the persistent object identifier.
:param psco: Persistent object.
:return: Persistent object identifier.
"""
with event_inside_worker(GETID_EVENT):
return psco.getID()
def build_exception_message(job_id, exit_value):
# type: (str, int) -> str
""" Generate an exception message.
:param job_id: Job identifier.
:param exit_value: Exit value.
:return: Exception message.
"""
with event_inside_worker(BUILD_EXCEPTION_MESSAGE_EVENT):
message = " ".join((END_TASK_TAG, str(job_id), str(exit_value) + "\n"))
return message
def deserialize_from_file(file_name):
# type: (str) -> typing.Any
""" Deserialize the contents in a given file.
:param file_name: Name of the file with the contents to be deserialized
:return: A deserialized object
"""
with event_inside_worker(DESERIALIZE_FROM_FILE_EVENT):
handler = open(file_name, 'rb')
ret, close_handler = deserialize_from_handler(handler)
if close_handler:
handler.close()
return ret
def serialize_to_file(obj, file_name):
# type: (typing.Any, str) -> None
""" Serialize an object to a file.
:param obj: Object to be serialized.
:param file_name: File name where the object is going to be serialized.
:return: Nothing, it just serializes the object.
"""
with event_inside_worker(SERIALIZE_TO_FILE_EVENT):
# todo: can we make the binary mode optional?
handler = open(file_name, 'wb')
serialize_to_handler(obj, handler)
handler.close()
def build_compss_exception_message(except_msg, job_id):
# type: (str, str) -> typing.Tuple[str, str]
""" Generate a COMPSs exception message.
:param except_msg: Exception stacktrace.
:param job_id: Job identifier.
:return: Exception message and message.
"""
with event_inside_worker(BUILD_COMPSS_EXCEPTION_MESSAGE_EVENT):
except_msg = except_msg.replace(" ", "_")
message = " ".join(
(COMPSS_EXCEPTION_TAG, str(job_id), str(except_msg) + "\n"))
return except_msg, message
def setup_environment(cn, cn_names, cu):
# type: (int, str, str) -> None
""" Sets the environment (mainly environment variables).
:param cn: Number of COMPSs nodes.
:param cn_names: COMPSs hostnames.
:param cu: Number of COMPSs threads.
:return: None
"""
with event_inside_worker(SETUP_ENVIRONMENT_EVENT):
os.environ["COMPSS_NUM_NODES"] = str(cn)
os.environ["COMPSS_HOSTNAMES"] = cn_names
os.environ["COMPSS_NUM_THREADS"] = cu
os.environ["OMP_NUM_THREADS"] = cu
def get_task_params(num_params, logger, args): # noqa
# type: (int, typing.Any, list) -> list
""" Get and prepare the input parameters from string to lists.
:param num_params: Number of parameters
:param logger: Logger
:param args: Arguments (complete list of parameters with type, stream,
prefix and value)
:return: A list of TaskParameter objects
"""
with event_inside_worker(GET_TASK_PARAMS_EVENT):
pos = 0
ret = []
for i in range(0, num_params): # noqa
p_type = int(args[pos])
p_stream = int(args[pos + 1])
p_prefix = args[pos + 2]
p_name = args[pos + 3]
p_c_type = args[pos + 4]
p_value = args[pos + 5]
if __debug__:
logger.debug("Parameter : %s" % str(i))
logger.debug("\t * Type : %s" % str(p_type))
logger.debug("\t * Std IO Stream : %s" % str(p_stream))
logger.debug("\t * Prefix : %s" % str(p_prefix))
logger.debug("\t * Name : %s" % str(p_name))
logger.debug("\t * Content Type: %r" % p_c_type)
if p_type == parameter.TYPE.STRING:
logger.debug("\t * Number of substrings: %r" % p_value)
else:
logger.debug("\t * Value: %r" % p_value)
task_param, offset = build_task_parameter(p_type,
p_stream,
p_prefix,
p_name,
p_value,
p_c_type,
args,
pos, logger)
if __debug__:
logger.debug("\t * Updated type : %s" % str(task_param.content_type))
ret.append(task_param)
pos += offset + 6
return ret
def deserialize_from_bytes(serialized_content_bytes, show_exception=True):
# type: (bytes, bool) -> typing.Any
""" Deserialize the contents in a given byte array.
:param serialized_content_bytes: A byte array with serialized contents
:param show_exception: Show exception if happen (only with debug).
:return: A deserialized object
"""
with event_inside_worker(DESERIALIZE_FROM_BYTES_EVENT):
handler = BytesIO(serialized_content_bytes)
ret, close_handler = deserialize_from_handler(handler,
show_exception=show_exception) # noqa: E501
if close_handler:
handler.close()
return ret
def remove_object_from_cache(logger, cache_queue, f_name): # noqa
# type: (typing.Any, Queue, str) -> None
""" Removes an object from cache.
:param logger: Logger where to push messages.
:param cache_queue: Cache notification queue.
:param f_name: File name that corresponds to the object (used as id).
:return: None
"""
with event_inside_worker(REMOVE_OBJECT_FROM_CACHE_EVENT):
f_name = __get_file_name__(f_name)
if __debug__:
logger.debug(HEADER + "Removing from cache: " + str(f_name))
cache_queue.put(("REMOVE", f_name))
if __debug__:
logger.debug(HEADER + "Removed from cache: " + str(f_name))
def bind_gpus(gpus, process_name, logger): # noqa
# type: (str, str, typing.Any) -> None
""" Bind the given GPUs to this process.
:param gpus: Target GPUs.
:param process_name: Process name for logger messages.
:param logger: Logger.
:return: None
"""
with event_inside_worker(BIND_GPUS_EVENT):
os.environ["COMPSS_BINDED_GPUS"] = gpus
os.environ["CUDA_VISIBLE_DEVICES"] = gpus
os.environ["GPU_DEVICE_ORDINAL"] = gpus
if __debug__:
logger.debug(HEADER + "[%s] Assigning GPU %s" %
(str(process_name), str(gpus)))
def build_successful_message(new_types, new_values, job_id, exit_value):
# type: (list, list, str, int) -> str
""" Generate a successful message.
:param new_types: New types (can change if INOUT).
:param new_values: New values (can change if INOUT).
:param job_id: Job identifier.
:param exit_value: Exit value.
:return: Successful message.
"""
with event_inside_worker(BUILD_SUCCESSFUL_MESSAGE_EVENT):
# Task has finished without exceptions
# endTask jobId exitValue message
params = build_return_params_message(new_types, new_values)
message = " ".join(
(END_TASK_TAG, str(job_id), str(exit_value), str(params) + "\n"))
return message
def clean_environment(cpus, gpus):
# type: (bool, bool) -> None
""" Clean the environment
Mainly unset environment variables.
:param cpus: If binded cpus.
:param gpus: If binded gpus.
:return: None
"""
with event_inside_worker(CLEAN_ENVIRONMENT_EVENT):
if cpus:
del os.environ["COMPSS_BINDED_CPUS"]
if gpus:
del os.environ["COMPSS_BINDED_GPUS"]
del os.environ["CUDA_VISIBLE_DEVICES"]
del os.environ["GPU_DEVICE_ORDINAL"]
del os.environ["COMPSS_HOSTNAMES"]
def serialize_to_file_mpienv(obj, file_name, rank_zero_reduce):
# type: (typing.Any, str, bool) -> None
""" Serialize an object to a file for Python MPI Tasks.
:param obj: Object to be serialized.
:param file_name: File name where the object is going to be serialized.
:param rank_zero_reduce: A boolean to indicate whether objects should be
reduced to MPI rank zero.
False for INOUT objects and True otherwise.
:return: Nothing, it just serializes the object.
"""
with event_inside_worker(SERIALIZE_TO_FILE_MPIENV_EVENT):
from mpi4py import MPI
if rank_zero_reduce:
nprocs = MPI.COMM_WORLD.Get_size()
if nprocs > 1:
obj = MPI.COMM_WORLD.reduce([obj], root=0)
if MPI.COMM_WORLD.rank == 0:
serialize_to_file(obj, file_name)
else:
serialize_to_file(obj, file_name)
def wait_on(*args, **kwargs):
# type: (*typing.Any, **typing.Any) -> typing.Any
""" Wait on a set of objects.
Waits on a set of objects defined in args with the options defined in
kwargs.
:param args: Objects to wait on.
:param kwargs: Options: Write enable? [True | False] Default = True.
May include: master_event: Emit master event. [Default: True | False]
False will emit the event inside task
(for nested).
:return: Real value of the objects requested.
"""
master_event = True
if "master_event" in kwargs:
master_event = kwargs["master_event"]
if master_event:
with event_master(WAIT_ON_EVENT):
return __wait_on__(*args, **kwargs)
else:
with event_inside_worker(WAIT_ON_EVENT):
return __wait_on__(*args, **kwargs)
def insert_object_into_cache(logger, cache_queue, obj, f_name, parameter,
user_function): # noqa
# type: (typing.Any, Queue, typing.Any, str, str, typing.Callable) -> None
""" Put an object into cache.
:param logger: Logger where to push messages.
:param cache_queue: Cache notification queue.
:param obj: Object to store.
:param f_name: File name that corresponds to the object (used as id).
:param parameter: Parameter name.
:param user_function: Function.
:return: None
"""
with event_inside_worker(INSERT_OBJECT_INTO_CACHE_EVENT):
function = function_cleaned(user_function)
f_name = __get_file_name__(f_name)
if __debug__:
logger.debug(HEADER + "Inserting into cache (%s): %s" %
(str(type(obj)), str(f_name)))
try:
inserted = True
if isinstance(obj, np.ndarray):
emit_manual_event_explicit(
BINDING_SERIALIZATION_CACHE_SIZE_TYPE, 0)
shape = obj.shape
d_type = obj.dtype
size = obj.nbytes
shm = SHARED_MEMORY_MANAGER.SharedMemory(size=size) # noqa
within_cache = np.ndarray(shape, dtype=d_type, buffer=shm.buf)
within_cache[:] = obj[:] # Copy contents
new_cache_id = shm.name
cache_queue.put(("PUT", (f_name, new_cache_id, shape, d_type,
size, SHARED_MEMORY_TAG, parameter,
function))) # noqa: E501
elif isinstance(obj, list):
emit_manual_event_explicit(
BINDING_SERIALIZATION_CACHE_SIZE_TYPE, 0)
sl = SHARED_MEMORY_MANAGER.ShareableList(obj) # noqa
new_cache_id = sl.shm.name
size = total_sizeof(obj)
cache_queue.put(("PUT", (f_name, new_cache_id, 0, 0, size,
SHAREABLE_LIST_TAG, parameter,
function))) # noqa: E501
elif isinstance(obj, tuple):
emit_manual_event_explicit(
BINDING_SERIALIZATION_CACHE_SIZE_TYPE, 0)
sl = SHARED_MEMORY_MANAGER.ShareableList(obj) # noqa
new_cache_id = sl.shm.name
size = total_sizeof(obj)
cache_queue.put(("PUT", (f_name, new_cache_id, 0, 0, size,
SHAREABLE_TUPLE_TAG, parameter,
function))) # noqa: E501
# Unsupported dicts since they are lists of lists when converted.
# elif isinstance(obj, dict):
# # Convert dict to list of tuples
# list_tuples = list(zip(obj.keys(), obj.values()))
# sl = SHARED_MEMORY_MANAGER.ShareableList(list_tuples) # noqa
# new_cache_id = sl.shm.name
# size = total_sizeof(obj)
# cache_queue.put(("PUT", (f_name, new_cache_id, 0, 0, size, SHAREABLE_DICT_TAG, parameter, function))) # noqa: E501
else:
inserted = False
if __debug__:
logger.debug(
HEADER +
"Can not put into cache: Not a [np.ndarray | list | tuple ] object"
) # noqa: E501
if inserted:
emit_manual_event_explicit(
BINDING_SERIALIZATION_CACHE_SIZE_TYPE, size)
if __debug__ and inserted:
logger.debug(HEADER + "Inserted into cache: " + str(f_name) +
" as " + str(new_cache_id)) # noqa: E501
except KeyError as e: # noqa
if __debug__:
logger.debug(
HEADER +
"Can not put into cache. It may be a [np.ndarray | list | tuple ] object containing an unsupported type"
) # noqa: E501
logger.debug(str(e))
def __decorator_body__(self, user_function, args, kwargs):
# type: (typing.Callable, tuple, dict) -> typing.Any
# Determine the context and decide what to do
if context.in_master():
# @task being executed in the master
# Each task will have a TaskMaster, so its content will
# not be shared.
self.__check_core_element__(kwargs, user_function)
with event_master(TASK_INSTANTIATION):
master = TaskMaster(self.decorator_arguments,
self.user_function, self.core_element,
self.registered, self.signature,
self.interactive, self.module,
self.function_arguments,
self.function_name, self.module_name,
self.function_type, self.class_name,
self.hints, self.on_failure, self.defaults)
result = master.call(args, kwargs)
fo, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints = result # noqa: E501
del master
return fo
elif context.in_worker():
if "compss_key" in kwargs.keys():
if context.is_nesting_enabled():
# Update the whole logger since it will be in job out/err
update_logger_handlers(kwargs["compss_log_cfg"],
kwargs["compss_log_files"][0],
kwargs["compss_log_files"][1])
# @task being executed in the worker
with event_inside_worker(WORKER_TASK_INSTANTIATION):
worker = TaskWorker(self.decorator_arguments,
self.user_function, self.on_failure,
self.defaults)
result = worker.call(*args, **kwargs)
# Force flush stdout and stderr
sys.stdout.flush()
sys.stderr.flush()
# Remove worker
del worker
if context.is_nesting_enabled():
# Wait for all nested tasks to finish
from pycompss.runtime.binding import nested_barrier
nested_barrier()
# Reestablish logger handlers
update_logger_handlers(kwargs["compss_log_cfg"])
return result
else:
if context.is_nesting_enabled():
# Each task will have a TaskMaster, so its content will
# not be shared.
with event_master(TASK_INSTANTIATION):
master = TaskMaster(
self.decorator_arguments, self.user_function,
self.core_element, self.registered, self.signature,
self.interactive, self.module,
self.function_arguments, self.function_name,
self.module_name, self.function_type,
self.class_name, self.hints, self.on_failure,
self.defaults)
result = master.call(args, kwargs)
fo, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints = result # noqa: E501
del master
return fo
else:
# Called from another task within the worker
# Ignore the @task decorator and run it sequentially
message = "".join(
("WARNING: Calling task: ",
str(user_function.__name__), " from this task.\n",
" It will be executed ",
"sequentially within the caller task."))
print(message, file=sys.stderr)
return self._sequential_call(*args, **kwargs)
# We are neither in master nor in the worker, or the user has
# stopped the interactive session.
# Therefore, the user code is being executed with no
# launch_compss/enqueue_compss/runcompss/interactive session
return self._sequential_call(*args, **kwargs)