def bind_cpus(cpus, process_name, logger): # noqa # type: (str, str, typing.Any) -> bool """ Bind the given CPUs for core affinity to this process. :param cpus: Target CPUs. :param process_name: Process name for logger messages. :param logger: Logger. :return: True if success, False otherwise. """ with event_inside_worker(BIND_CPUS_EVENT): if __debug__: logger.debug(HEADER + "[%s] Assigning affinity %s" % (str(process_name), str(cpus))) cpus_list = cpus.split(",") cpus_map = list(map(int, cpus_list)) try: thread_affinity.setaffinity(cpus_map) except Exception: # noqa if __debug__: logger.error(HEADER + "[%s] WARNING: could not assign affinity %s" % (str(process_name), str(cpus_map))) return False # Export only if success os.environ["COMPSS_BINDED_CPUS"] = cpus return True
def import_user_module(path, logger): # type: (str, typing.Any) -> typing.Any """ Import the user module. :param path: Path to the user module. :param logger: Logger. :return: The loaded module. """ with event_inside_worker(IMPORT_USER_MODULE_EVENT): py_version = sys.version_info if py_version >= (2, 7): import importlib module = importlib.import_module(path) # Python 2.7 if path.startswith(INTERACTIVE_FILE_NAME): # Force reload in interactive mode. The user may have # overwritten a function or task. if py_version < (3, 4): import imp # noqa imp.reload(module) # noqa else: importlib.reload(module) if __debug__: msg = "Module successfully loaded (Python version >= 2.7)" logger.debug(msg) else: module = __import__(path, globals(), locals(), [path], -1) if __debug__: msg = "Module successfully loaded (Python version < 2.7" logger.debug(msg) return module
def retrieve_object_from_cache(logger, cache_ids, cache_queue, identifier, parameter_name, user_function, cache_profiler): # noqa # type: (typing.Any, typing.Any, Queue, str, str, typing.Callable, bool) -> typing.Any """ Retrieve an object from the given cache proxy dict. :param logger: Logger where to push messages. :param cache_ids: Cache proxy dictionary. :param cache_queue: Cache notification queue. :param identifier: Object identifier. :param parameter_name: Parameter name. :param user_function: Function name. :param cache_profiler: If cache profiling is enabled. :return: The object from cache. """ with event_inside_worker(RETRIEVE_OBJECT_FROM_CACHE_EVENT): emit_manual_event_explicit(BINDING_DESERIALIZATION_CACHE_SIZE_TYPE, 0) identifier = __get_file_name__(identifier) if __debug__: logger.debug(HEADER + "Retrieving: " + str(identifier)) obj_id, obj_shape, obj_d_type, _, obj_hits, shared_type = cache_ids[ identifier] # noqa: E501 output = None # type: typing.Any existing_shm = None # type: typing.Any object_size = 0 if shared_type == SHARED_MEMORY_TAG: existing_shm = SharedMemory(name=obj_id) output = np.ndarray(obj_shape, dtype=obj_d_type, buffer=existing_shm.buf) # noqa: E501 object_size = len(existing_shm.buf) elif shared_type == SHAREABLE_LIST_TAG: existing_shm = ShareableList(name=obj_id) output = list(existing_shm) object_size = len(existing_shm.shm.buf) elif shared_type == SHAREABLE_TUPLE_TAG: existing_shm = ShareableList(name=obj_id) output = tuple(existing_shm) object_size = len(existing_shm.shm.buf) # Currently unsupported since conversion requires lists of lists. # elif shared_type == SHAREABLE_DICT_TAG: # existing_shm = ShareableList(name=obj_id) # output = dict(existing_shm) else: raise PyCOMPSsException("Unknown cacheable type.") if __debug__: logger.debug(HEADER + "Retrieved: " + str(identifier)) emit_manual_event_explicit(BINDING_DESERIALIZATION_CACHE_SIZE_TYPE, object_size) # Profiling filename = filename_cleaned(identifier) function_name = function_cleaned(user_function) if cache_profiler: cache_queue.put(("GET", (filename, parameter_name, function_name))) # Add hit cache_ids[identifier][4] = obj_hits + 1 return output, existing_shm
def get_by_id(id): # type: (str) -> typing.Any """ Retrieve the object from the given identifier. :param id: Persistent object identifier. :return: object associated to the persistent object identifier. """ with event_inside_worker(GET_BY_ID_EVENT): return GET_BY_ID(id)
def get_id(psco): # type: (typing.Any) -> typing.Union[str, None] """ Retrieve the persistent object identifier. :param psco: Persistent object. :return: Persistent object identifier. """ with event_inside_worker(GETID_EVENT): return psco.getID()
def build_exception_message(job_id, exit_value): # type: (str, int) -> str """ Generate an exception message. :param job_id: Job identifier. :param exit_value: Exit value. :return: Exception message. """ with event_inside_worker(BUILD_EXCEPTION_MESSAGE_EVENT): message = " ".join((END_TASK_TAG, str(job_id), str(exit_value) + "\n")) return message
def deserialize_from_file(file_name): # type: (str) -> typing.Any """ Deserialize the contents in a given file. :param file_name: Name of the file with the contents to be deserialized :return: A deserialized object """ with event_inside_worker(DESERIALIZE_FROM_FILE_EVENT): handler = open(file_name, 'rb') ret, close_handler = deserialize_from_handler(handler) if close_handler: handler.close() return ret
def serialize_to_file(obj, file_name): # type: (typing.Any, str) -> None """ Serialize an object to a file. :param obj: Object to be serialized. :param file_name: File name where the object is going to be serialized. :return: Nothing, it just serializes the object. """ with event_inside_worker(SERIALIZE_TO_FILE_EVENT): # todo: can we make the binary mode optional? handler = open(file_name, 'wb') serialize_to_handler(obj, handler) handler.close()
def build_compss_exception_message(except_msg, job_id): # type: (str, str) -> typing.Tuple[str, str] """ Generate a COMPSs exception message. :param except_msg: Exception stacktrace. :param job_id: Job identifier. :return: Exception message and message. """ with event_inside_worker(BUILD_COMPSS_EXCEPTION_MESSAGE_EVENT): except_msg = except_msg.replace(" ", "_") message = " ".join( (COMPSS_EXCEPTION_TAG, str(job_id), str(except_msg) + "\n")) return except_msg, message
def setup_environment(cn, cn_names, cu): # type: (int, str, str) -> None """ Sets the environment (mainly environment variables). :param cn: Number of COMPSs nodes. :param cn_names: COMPSs hostnames. :param cu: Number of COMPSs threads. :return: None """ with event_inside_worker(SETUP_ENVIRONMENT_EVENT): os.environ["COMPSS_NUM_NODES"] = str(cn) os.environ["COMPSS_HOSTNAMES"] = cn_names os.environ["COMPSS_NUM_THREADS"] = cu os.environ["OMP_NUM_THREADS"] = cu
def get_task_params(num_params, logger, args): # noqa # type: (int, typing.Any, list) -> list """ Get and prepare the input parameters from string to lists. :param num_params: Number of parameters :param logger: Logger :param args: Arguments (complete list of parameters with type, stream, prefix and value) :return: A list of TaskParameter objects """ with event_inside_worker(GET_TASK_PARAMS_EVENT): pos = 0 ret = [] for i in range(0, num_params): # noqa p_type = int(args[pos]) p_stream = int(args[pos + 1]) p_prefix = args[pos + 2] p_name = args[pos + 3] p_c_type = args[pos + 4] p_value = args[pos + 5] if __debug__: logger.debug("Parameter : %s" % str(i)) logger.debug("\t * Type : %s" % str(p_type)) logger.debug("\t * Std IO Stream : %s" % str(p_stream)) logger.debug("\t * Prefix : %s" % str(p_prefix)) logger.debug("\t * Name : %s" % str(p_name)) logger.debug("\t * Content Type: %r" % p_c_type) if p_type == parameter.TYPE.STRING: logger.debug("\t * Number of substrings: %r" % p_value) else: logger.debug("\t * Value: %r" % p_value) task_param, offset = build_task_parameter(p_type, p_stream, p_prefix, p_name, p_value, p_c_type, args, pos, logger) if __debug__: logger.debug("\t * Updated type : %s" % str(task_param.content_type)) ret.append(task_param) pos += offset + 6 return ret
def deserialize_from_bytes(serialized_content_bytes, show_exception=True): # type: (bytes, bool) -> typing.Any """ Deserialize the contents in a given byte array. :param serialized_content_bytes: A byte array with serialized contents :param show_exception: Show exception if happen (only with debug). :return: A deserialized object """ with event_inside_worker(DESERIALIZE_FROM_BYTES_EVENT): handler = BytesIO(serialized_content_bytes) ret, close_handler = deserialize_from_handler(handler, show_exception=show_exception) # noqa: E501 if close_handler: handler.close() return ret
def remove_object_from_cache(logger, cache_queue, f_name): # noqa # type: (typing.Any, Queue, str) -> None """ Removes an object from cache. :param logger: Logger where to push messages. :param cache_queue: Cache notification queue. :param f_name: File name that corresponds to the object (used as id). :return: None """ with event_inside_worker(REMOVE_OBJECT_FROM_CACHE_EVENT): f_name = __get_file_name__(f_name) if __debug__: logger.debug(HEADER + "Removing from cache: " + str(f_name)) cache_queue.put(("REMOVE", f_name)) if __debug__: logger.debug(HEADER + "Removed from cache: " + str(f_name))
def bind_gpus(gpus, process_name, logger): # noqa # type: (str, str, typing.Any) -> None """ Bind the given GPUs to this process. :param gpus: Target GPUs. :param process_name: Process name for logger messages. :param logger: Logger. :return: None """ with event_inside_worker(BIND_GPUS_EVENT): os.environ["COMPSS_BINDED_GPUS"] = gpus os.environ["CUDA_VISIBLE_DEVICES"] = gpus os.environ["GPU_DEVICE_ORDINAL"] = gpus if __debug__: logger.debug(HEADER + "[%s] Assigning GPU %s" % (str(process_name), str(gpus)))
def build_successful_message(new_types, new_values, job_id, exit_value): # type: (list, list, str, int) -> str """ Generate a successful message. :param new_types: New types (can change if INOUT). :param new_values: New values (can change if INOUT). :param job_id: Job identifier. :param exit_value: Exit value. :return: Successful message. """ with event_inside_worker(BUILD_SUCCESSFUL_MESSAGE_EVENT): # Task has finished without exceptions # endTask jobId exitValue message params = build_return_params_message(new_types, new_values) message = " ".join( (END_TASK_TAG, str(job_id), str(exit_value), str(params) + "\n")) return message
def clean_environment(cpus, gpus): # type: (bool, bool) -> None """ Clean the environment Mainly unset environment variables. :param cpus: If binded cpus. :param gpus: If binded gpus. :return: None """ with event_inside_worker(CLEAN_ENVIRONMENT_EVENT): if cpus: del os.environ["COMPSS_BINDED_CPUS"] if gpus: del os.environ["COMPSS_BINDED_GPUS"] del os.environ["CUDA_VISIBLE_DEVICES"] del os.environ["GPU_DEVICE_ORDINAL"] del os.environ["COMPSS_HOSTNAMES"]
def serialize_to_file_mpienv(obj, file_name, rank_zero_reduce): # type: (typing.Any, str, bool) -> None """ Serialize an object to a file for Python MPI Tasks. :param obj: Object to be serialized. :param file_name: File name where the object is going to be serialized. :param rank_zero_reduce: A boolean to indicate whether objects should be reduced to MPI rank zero. False for INOUT objects and True otherwise. :return: Nothing, it just serializes the object. """ with event_inside_worker(SERIALIZE_TO_FILE_MPIENV_EVENT): from mpi4py import MPI if rank_zero_reduce: nprocs = MPI.COMM_WORLD.Get_size() if nprocs > 1: obj = MPI.COMM_WORLD.reduce([obj], root=0) if MPI.COMM_WORLD.rank == 0: serialize_to_file(obj, file_name) else: serialize_to_file(obj, file_name)
def wait_on(*args, **kwargs): # type: (*typing.Any, **typing.Any) -> typing.Any """ Wait on a set of objects. Waits on a set of objects defined in args with the options defined in kwargs. :param args: Objects to wait on. :param kwargs: Options: Write enable? [True | False] Default = True. May include: master_event: Emit master event. [Default: True | False] False will emit the event inside task (for nested). :return: Real value of the objects requested. """ master_event = True if "master_event" in kwargs: master_event = kwargs["master_event"] if master_event: with event_master(WAIT_ON_EVENT): return __wait_on__(*args, **kwargs) else: with event_inside_worker(WAIT_ON_EVENT): return __wait_on__(*args, **kwargs)
def insert_object_into_cache(logger, cache_queue, obj, f_name, parameter, user_function): # noqa # type: (typing.Any, Queue, typing.Any, str, str, typing.Callable) -> None """ Put an object into cache. :param logger: Logger where to push messages. :param cache_queue: Cache notification queue. :param obj: Object to store. :param f_name: File name that corresponds to the object (used as id). :param parameter: Parameter name. :param user_function: Function. :return: None """ with event_inside_worker(INSERT_OBJECT_INTO_CACHE_EVENT): function = function_cleaned(user_function) f_name = __get_file_name__(f_name) if __debug__: logger.debug(HEADER + "Inserting into cache (%s): %s" % (str(type(obj)), str(f_name))) try: inserted = True if isinstance(obj, np.ndarray): emit_manual_event_explicit( BINDING_SERIALIZATION_CACHE_SIZE_TYPE, 0) shape = obj.shape d_type = obj.dtype size = obj.nbytes shm = SHARED_MEMORY_MANAGER.SharedMemory(size=size) # noqa within_cache = np.ndarray(shape, dtype=d_type, buffer=shm.buf) within_cache[:] = obj[:] # Copy contents new_cache_id = shm.name cache_queue.put(("PUT", (f_name, new_cache_id, shape, d_type, size, SHARED_MEMORY_TAG, parameter, function))) # noqa: E501 elif isinstance(obj, list): emit_manual_event_explicit( BINDING_SERIALIZATION_CACHE_SIZE_TYPE, 0) sl = SHARED_MEMORY_MANAGER.ShareableList(obj) # noqa new_cache_id = sl.shm.name size = total_sizeof(obj) cache_queue.put(("PUT", (f_name, new_cache_id, 0, 0, size, SHAREABLE_LIST_TAG, parameter, function))) # noqa: E501 elif isinstance(obj, tuple): emit_manual_event_explicit( BINDING_SERIALIZATION_CACHE_SIZE_TYPE, 0) sl = SHARED_MEMORY_MANAGER.ShareableList(obj) # noqa new_cache_id = sl.shm.name size = total_sizeof(obj) cache_queue.put(("PUT", (f_name, new_cache_id, 0, 0, size, SHAREABLE_TUPLE_TAG, parameter, function))) # noqa: E501 # Unsupported dicts since they are lists of lists when converted. # elif isinstance(obj, dict): # # Convert dict to list of tuples # list_tuples = list(zip(obj.keys(), obj.values())) # sl = SHARED_MEMORY_MANAGER.ShareableList(list_tuples) # noqa # new_cache_id = sl.shm.name # size = total_sizeof(obj) # cache_queue.put(("PUT", (f_name, new_cache_id, 0, 0, size, SHAREABLE_DICT_TAG, parameter, function))) # noqa: E501 else: inserted = False if __debug__: logger.debug( HEADER + "Can not put into cache: Not a [np.ndarray | list | tuple ] object" ) # noqa: E501 if inserted: emit_manual_event_explicit( BINDING_SERIALIZATION_CACHE_SIZE_TYPE, size) if __debug__ and inserted: logger.debug(HEADER + "Inserted into cache: " + str(f_name) + " as " + str(new_cache_id)) # noqa: E501 except KeyError as e: # noqa if __debug__: logger.debug( HEADER + "Can not put into cache. It may be a [np.ndarray | list | tuple ] object containing an unsupported type" ) # noqa: E501 logger.debug(str(e))
def __decorator_body__(self, user_function, args, kwargs): # type: (typing.Callable, tuple, dict) -> typing.Any # Determine the context and decide what to do if context.in_master(): # @task being executed in the master # Each task will have a TaskMaster, so its content will # not be shared. self.__check_core_element__(kwargs, user_function) with event_master(TASK_INSTANTIATION): master = TaskMaster(self.decorator_arguments, self.user_function, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints, self.on_failure, self.defaults) result = master.call(args, kwargs) fo, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints = result # noqa: E501 del master return fo elif context.in_worker(): if "compss_key" in kwargs.keys(): if context.is_nesting_enabled(): # Update the whole logger since it will be in job out/err update_logger_handlers(kwargs["compss_log_cfg"], kwargs["compss_log_files"][0], kwargs["compss_log_files"][1]) # @task being executed in the worker with event_inside_worker(WORKER_TASK_INSTANTIATION): worker = TaskWorker(self.decorator_arguments, self.user_function, self.on_failure, self.defaults) result = worker.call(*args, **kwargs) # Force flush stdout and stderr sys.stdout.flush() sys.stderr.flush() # Remove worker del worker if context.is_nesting_enabled(): # Wait for all nested tasks to finish from pycompss.runtime.binding import nested_barrier nested_barrier() # Reestablish logger handlers update_logger_handlers(kwargs["compss_log_cfg"]) return result else: if context.is_nesting_enabled(): # Each task will have a TaskMaster, so its content will # not be shared. with event_master(TASK_INSTANTIATION): master = TaskMaster( self.decorator_arguments, self.user_function, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints, self.on_failure, self.defaults) result = master.call(args, kwargs) fo, self.core_element, self.registered, self.signature, self.interactive, self.module, self.function_arguments, self.function_name, self.module_name, self.function_type, self.class_name, self.hints = result # noqa: E501 del master return fo else: # Called from another task within the worker # Ignore the @task decorator and run it sequentially message = "".join( ("WARNING: Calling task: ", str(user_function.__name__), " from this task.\n", " It will be executed ", "sequentially within the caller task.")) print(message, file=sys.stderr) return self._sequential_call(*args, **kwargs) # We are neither in master nor in the worker, or the user has # stopped the interactive session. # Therefore, the user code is being executed with no # launch_compss/enqueue_compss/runcompss/interactive session return self._sequential_call(*args, **kwargs)