def update_params(self, argv): # type: (list) -> None """ Constructs a configuration description for the piper worker using the arguments. :param argv: arguments from the command line. :return: None """ set_temporary_directory(argv[1], create_tmpdir=False) if argv[2] == 'true': context.enable_nesting() self.nesting = True self.debug = argv[3] == 'true' self.tracing = argv[4] == '1' self.storage_conf = argv[5] self.stream_backend = argv[6] self.stream_master_name = argv[7] self.stream_master_port = argv[8] self.tasks_x_node = int(argv[9]) in_pipes = argv[10:10 + self.tasks_x_node] out_pipes = argv[10 + self.tasks_x_node:-2] if self.debug: assert self.tasks_x_node == len(in_pipes) assert self.tasks_x_node == len(out_pipes) self.pipes = [] for i in range(0, self.tasks_x_node): self.pipes.append(Pipe(in_pipes[i], out_pipes[i])) self.control_pipe = Pipe(argv[-2], argv[-1])
def generate_report(self, target_path): # type: (str) -> None """ Generates a plot reporting the behaviour of the object tracker. Uses the self.report_info internal variable contents. :param target_path: Path where to store the report. :return: None """ try: import matplotlib # noqa matplotlib.use('Agg') # avoid issues in MN import matplotlib.pyplot as plt # noqa except ImportError: print("WARNING: Could not generate the Object Tracker report") print("REASON : matplotlib not available.") return None if __debug__: logger.debug("Generating object tracker report...") x = [status[0] for status in self.reporting_info] y = [status[1] for status in self.reporting_info] plt.xlabel("Time (seconds)") plt.ylabel("# Elements") plt.title("Object tracker behaviour") labels = [ 'File names', 'Pending to synchronize', 'Updated mappings', 'IDs', 'Addresses' ] for i in range(len(y[0])): plt.plot(x, [pt[i] for pt in y], label='%s' % labels[i]) plt.legend() target = os.path.join(target_path, "object_tracker.png") plt.savefig(target) if __debug__: logger.debug("Object tracker report stored in " + target)
def get_task_params(num_params, logger, args): # noqa # type: (int, ..., list) -> list """ Get and prepare the input parameters from string to lists. :param num_params: Number of parameters :param logger: Logger :param args: Arguments (complete list of parameters with type, stream, prefix and value) :return: A list of TaskParameter objects """ pos = 0 ret = [] for i in range(0, num_params): p_type = int(args[pos]) p_stream = int(args[pos + 1]) p_prefix = args[pos + 2] p_name = args[pos + 3] p_c_type = args[pos + 4] p_value = args[pos + 5] if __debug__: logger.debug("Parameter : %s" % str(i)) logger.debug("\t * Type : %s" % str(p_type)) logger.debug("\t * Std IO Stream : %s" % str(p_stream)) logger.debug("\t * Prefix : %s" % str(p_prefix)) logger.debug("\t * Name : %s" % str(p_name)) logger.debug("\t * Content Type: %r" % p_c_type) logger.debug("\t * Value: %r" % p_value) task_param, offset = build_task_parameter(p_type, p_stream, p_prefix, p_name, p_value, p_c_type, args, pos, logger) if __debug__: logger.debug("\t * Type : %s" % str(task_param.content_type)) ret.append(task_param) pos += offset + 6 return ret
def _get_return_values_for_exception(types, values): # type: (list, list) -> list """ Builds the values list to retrieve on an exception. It takes the input types and returns a list of 'null' for each type unless it is a PSCO, where it puts the psco identifier. :param types: List of input types. :param values: List of input values. :return: List of values to return """ new_values = [] for i in range(len(types)): if types[i] == parameter.TYPE.EXTERNAL_PSCO: new_values.append(values[i]) else: new_values.append('null') return new_values
def compss_persistent_worker(config): # type: (PiperWorkerConfiguration) -> None """ Persistent worker main function. Retrieves the initial configuration and spawns the worker processes. :param config: Piper Worker Configuration description. :return: None """ pids = COMM.gather(str(os.getpid()), root=0) # Catch SIGTERM sent by bindings_piper signal.signal(signal.SIGTERM, shutdown_handler) # Catch SIGUSER2 to solve strange behaviour with mpi4py signal.signal(signal.SIGUSR2, user_signal_handler) # Set the binding in worker mode import pycompss.util.context as context context.set_pycompss_context(context.WORKER) persistent_storage = (config.storage_conf != 'null') logger, _, _ = load_loggers(config.debug, persistent_storage) if __debug__: logger.debug(HEADER + "mpi_piper_worker.py rank: " + str(RANK) + " wake up") config.print_on_logger(logger) # Start storage if persistent_storage: # Initialize storage if __debug__: logger.debug(HEADER + "Starting persistent storage") from storage.api import initWorker as initStorageAtWorker # noqa initStorageAtWorker(config_file_path=config.storage_conf) for i in range(0, config.tasks_x_node): child_in_pipe = config.pipes[i].input_pipe try: child_pid = pids[i + 1] except IndexError: child_pid = pids[i] PROCESSES[child_in_pipe] = child_pid if __debug__: logger.debug(HEADER + "Starting alive") logger.debug(HEADER + "Control pipe: " + str(config.control_pipe)) # Read command from control pipe alive = True control_pipe = config.control_pipe while alive: command = control_pipe.read_command() if command != "": line = command.split() if line[0] == ADD_EXECUTOR_TAG: in_pipe = line[1] out_pipe = line[2] control_pipe.write(" ".join( (ADD_EXECUTOR_FAILED_TAG, out_pipe, in_pipe, str(0)))) elif line[0] == REMOVE_EXECUTOR_TAG: in_pipe = line[1] out_pipe = line[2] PROCESSES.pop(in_pipe, None) control_pipe.write(" ".join( (REMOVED_EXECUTOR_TAG, out_pipe, in_pipe))) elif line[0] == QUERY_EXECUTOR_ID_TAG: in_pipe = line[1] out_pipe = line[2] pid = PROCESSES.get(in_pipe) control_pipe.write(" ".join( (REPLY_EXECUTOR_ID_TAG, out_pipe, in_pipe, str(pid)))) elif line[0] == CANCEL_TASK_TAG: in_pipe = line[1] pid = PROCESSES.get(in_pipe) logger.debug(HEADER + "Signaling process with PID " + pid + " to cancel a task") os.kill( int(pid), signal.SIGUSR2) # NOSONAR cancellation produced by COMPSs elif line[0] == PING_TAG: control_pipe.write(PONG_TAG) elif line[0] == QUIT_TAG: alive = False else: logger.debug(HEADER + "ERROR: UNKNOWN COMMAND: " + command) alive = False # Stop storage if persistent_storage: # Finish storage logger.debug(HEADER + "Stopping persistent storage") from storage.api import finishWorker as finishStorageAtWorker # noqa finishStorageAtWorker() if __debug__: logger.debug(HEADER + "Finished") control_pipe.write(QUIT_TAG) control_pipe.close()
def compss_persistent_worker(config): # type: (PiperWorkerConfiguration) -> None """ Persistent worker main function. Retrieves the initial configuration and spawns the worker processes. :param config: Piper Worker Configuration description. :return: None """ global CACHE global CACHE_PROCESS # Catch SIGTERM sent by bindings_piper signal.signal(signal.SIGTERM, shutdown_handler) # Set the binding in worker mode context.set_pycompss_context(context.WORKER) persistent_storage = (config.storage_conf != 'null') logger, logger_cfg, storage_loggers = load_loggers(config.debug, persistent_storage) if __debug__: logger.debug(HEADER + "piper_worker.py wake up") config.print_on_logger(logger) if persistent_storage: # Initialize storage logger.debug(HEADER + "Starting persistent storage") with event(INIT_STORAGE_AT_WORKER_EVENT): from storage.api import initWorker as initStorageAtWorker # noqa initStorageAtWorker(config_file_path=config.storage_conf) # Create new processes queues = [] # Setup cache if is_cache_enabled(config.cache): # Deploy the necessary processes CACHE = True cache_params = start_cache(logger, config.cache) else: # No cache CACHE = False cache_params = (None, None, None, None) smm, CACHE_PROCESS, cache_queue, cache_ids = cache_params # Create new executor processes conf = ExecutorConf(TRACING, config.storage_conf, logger, logger_cfg, storage_loggers, config.stream_backend, config.stream_master_name, config.stream_master_port, cache_ids, cache_queue) for i in range(0, config.tasks_x_node): if __debug__: logger.debug(HEADER + "Launching process " + str(i)) process_name = "".join(("Process-", str(i))) pid, queue = create_executor_process(process_name, conf, config.pipes[i]) queues.append(queue) # Read command from control pipe alive = True process_counter = config.tasks_x_node control_pipe = config.control_pipe while alive: command = control_pipe.read_command(retry_period=1) if command != "": line = command.split() if line[0] == ADD_EXECUTOR_TAG: process_name = "".join(("Process-", str(process_counter))) process_counter = process_counter + 1 in_pipe = line[1] out_pipe = line[2] pipe = Pipe(in_pipe, out_pipe) pid, queue = create_executor_process(process_name, conf, pipe) queues.append(queue) control_pipe.write(" ".join( (ADDED_EXECUTOR_TAG, out_pipe, in_pipe, str(pid)))) elif line[0] == QUERY_EXECUTOR_ID_TAG: in_pipe = line[1] out_pipe = line[2] proc = PROCESSES.get(in_pipe) pid = proc.pid control_pipe.write(" ".join( (REPLY_EXECUTOR_ID_TAG, out_pipe, in_pipe, str(pid)))) elif line[0] == CANCEL_TASK_TAG: in_pipe = line[1] proc = PROCESSES.get(in_pipe) pid = proc.pid if __debug__: logger.debug(HEADER + "Signaling process with PID " + str(pid) + " to cancel a task") os.kill( pid, signal.SIGUSR2) # NOSONAR cancellation produced by COMPSs elif line[0] == REMOVE_EXECUTOR_TAG: in_pipe = line[1] out_pipe = line[2] proc = PROCESSES.pop(in_pipe, None) if proc: if proc.is_alive(): logger.warn(HEADER + "Forcing terminate on : " + proc.name) proc.terminate() proc.join() control_pipe.write(" ".join( (REMOVED_EXECUTOR_TAG, out_pipe, in_pipe))) elif line[0] == PING_TAG: control_pipe.write(PONG_TAG) elif line[0] == QUIT_TAG: alive = False # Wait for all threads for proc in PROCESSES.values(): proc.join() # Check if there is any exception message from the threads for i in range(0, config.tasks_x_node): if not queues[i].empty: logger.error(HEADER + "Exception in threads queue: " + str(queues[i].get())) for queue in queues: queue.close() queue.join_thread() if CACHE: stop_cache(smm, cache_queue, CACHE_PROCESS) # noqa if persistent_storage: # Finish storage if __debug__: logger.debug(HEADER + "Stopping persistent storage") with event(FINISH_STORAGE_AT_WORKER_EVENT): from storage.api import finishWorker as finishStorageAtWorker # noqa finishStorageAtWorker() if __debug__: logger.debug(HEADER + "Finished") control_pipe.write(QUIT_TAG) control_pipe.close()
def build_task_parameter( p_type, # type: int p_stream, # type: int p_prefix, # type: str p_name, # type: str p_value, # type: object p_c_type, # type: str args=None, # type: list pos=None, # type: int logger=None # noqa # type: ... ): # type: (...) -> (Parameter, int) """ Build task parameter object from the given parameters. :param p_type: Parameter type. :param p_stream: Parameter stream. :param p_prefix: Parameter prefix. :param p_name: Parameter name. :param p_value: Parameter value. :param p_c_type: Parameter Python Type. :param args: Arguments (Default: None). :param pos: Position (Default: None). :param logger: Logger where to push the logging messages. :return: Parameter object and the number fo substrings. """ num_substrings = 0 if p_type in [ parameter.TYPE.FILE, parameter.TYPE.DIRECTORY, parameter.TYPE.COLLECTION, parameter.TYPE.DICT_COLLECTION ]: # Maybe the file is a object, we do not care about this here # We will decide whether to deserialize or to forward the value # when processing parameters in the task decorator _param = Parameter(name=p_name, content_type=p_type, stream=p_stream, prefix=p_prefix, file_name=COMPSsFile(p_value), extra_content_type=p_c_type) return _param, 0 elif p_type == parameter.TYPE.EXTERNAL_PSCO: # Next position contains R/W but we do not need it. Currently skipped. return Parameter(content=p_value, content_type=p_type, stream=p_stream, prefix=p_prefix, name=p_name, extra_content_type=p_c_type), 1 elif p_type == parameter.TYPE.EXTERNAL_STREAM: # Next position contains R/W but we do not need it. Currently skipped. return Parameter(content_type=p_type, stream=p_stream, prefix=p_prefix, name=p_name, file_name=COMPSsFile(p_value), extra_content_type=p_c_type), 1 elif p_type == parameter.TYPE.STRING: if args is not None: num_substrings = int(p_value) # noqa aux_str = [] for j in range(6, num_substrings + 6): aux_str.append(args[pos + j]) aux = " ".join(aux_str) else: aux = str(p_value) # Decode the received string # Note that we prepend a sharp to all strings in order to avoid # getting empty encodings in the case of empty strings, so we need # to remove it when decoding aux = base64.b64decode(aux.encode()) aux = aux[1:] if aux: ####### # Check if the string is really an object # Required in order to recover objects passed as parameters. # - Option object_conversion real_value = aux try: # try to recover the real object if IS_PYTHON3: # decode removes double backslash, and encode returns # the result as binary p_bin_str = aux.decode(STR_ESCAPE).encode() aux = deserialize_from_string(p_bin_str, show_exception=False) else: # decode removes double backslash, and str casts the output aux = deserialize_from_string(str(aux.decode(STR_ESCAPE)), show_exception=False) except (SerializerException, ValueError, EOFError): # was not an object aux = str(real_value.decode()) ####### if IS_PYTHON3 and isinstance(aux, bytes): aux = aux.decode('utf-8') if __debug__: logger.debug("\t * Value: %s" % aux) return Parameter(content_type=p_type, stream=p_stream, prefix=p_prefix, name=p_name, content=aux, extra_content_type=p_c_type), num_substrings else: # Basic numeric types. These are passed as command line arguments # and only a cast is needed val = None if p_type == parameter.TYPE.INT: val = int(p_value) # noqa elif p_type == parameter.TYPE.LONG: val = PYCOMPSS_LONG(p_value) if val > JAVA_MAX_INT or val < JAVA_MIN_INT: # A Python in parameter was converted to a Java long to prevent # overflow. We are sure we will not overflow Python int, # otherwise this would have been passed as a serialized object. val = int(val) elif p_type == parameter.TYPE.DOUBLE: val = float(p_value) # noqa p_type = parameter.TYPE.FLOAT if __debug__: logger.debug("Changing type from DOUBLE to FLOAT") elif p_type == parameter.TYPE.BOOLEAN: val = (p_value == 'true') return Parameter(content=val, content_type=p_type, stream=p_stream, prefix=p_prefix, name=p_name, extra_content_type=p_c_type), 0
def execute_task( process_name, # type: str storage_conf, # type: str params, # type: list tracing, # type: bool logger, # type: ... logger_cfg, # type: str log_files, # type: tuple python_mpi=False, # type: bool collections_layouts=None, # type: list cache_queue=None, # type: ... cache_ids=None, # type: ... ): # type: (...) -> (str, list, list, bool, str) """ ExecuteTask main method. :param process_name: Process name. :param storage_conf: Storage configuration file path. :param params: List of parameters. :param tracing: Tracing flag. :param logger: Logger to use. :param logger_cfg: Logger configuration file :param log_files: Tuple with (out filename, err filename). None to avoid stdout and sdterr fd redirection. :param python_mpi: If it is a MPI task. :param collections_layouts: collections layouts for python MPI tasks :param cache_queue: Cache tracker communication queue :param cache_ids: Cache proxy dictionary (read-only) :return: updated_args, exit_code, new_types, new_values, timed_out and except_msg """ if __debug__: logger.debug("BEGIN TASK execution in %s" % process_name) persistent_storage = False if storage_conf != 'null': persistent_storage = True # Retrieve the parameters from the params argument path = params[0] method_name = params[1] num_slaves = int(params[3]) time_out = int(params[2]) slaves = [] for i in range(3, 3 + num_slaves): slaves.append(params[i]) arg_position = 4 + num_slaves args = params[arg_position:] cus = args[0] # noqa args = args[1:] has_target = args[0] # Next parameter: return_type = args[1] return_length = int(args[2]) num_params = int(args[3]) args = args[4:] # COMPSs keywords for tasks (ie: tracing, process name...) # compss_key is included to be checked in the @task decorator, so that # the task knows if it has been called from the worker or from the # user code (reason: ignore @task decorator if called from another task # or decide if submit to runtime if nesting is enabled). compss_kwargs = { 'compss_key': True, 'compss_tracing': tracing, 'compss_process_name': process_name, 'compss_storage_conf': storage_conf, 'compss_return_length': return_length, 'compss_logger': logger, 'compss_log_cfg': logger_cfg, 'compss_log_files': log_files, 'compss_python_MPI': python_mpi, 'compss_collections_layouts': collections_layouts, 'cache_queue': cache_queue, 'cache_ids': cache_ids } if __debug__: logger.debug("COMPSs parameters:") logger.debug("\t- Storage conf: %s" % str(storage_conf)) if log_files: logger.debug("\t- Log out file: %s" % str(log_files[0])) logger.debug("\t- Log err file: %s" % str(log_files[1])) else: logger.debug("\t- Log out and err not redirected") logger.debug("\t- Params: %s" % str(params)) logger.debug("\t- Path: %s" % str(path)) logger.debug("\t- Method name: %s" % str(method_name)) logger.debug("\t- Num slaves: %s" % str(num_slaves)) logger.debug("\t- Slaves: %s" % str(slaves)) logger.debug("\t- Cus: %s" % str(cus)) logger.debug("\t- Has target: %s" % str(has_target)) logger.debug("\t- Num Params: %s" % str(num_params)) logger.debug("\t- Return Length: %s" % str(return_length)) logger.debug("\t- Args: %r" % args) logger.debug("\t- COMPSs kwargs:") for k, v in compss_kwargs.items(): logger.debug("\t\t- %s: %s" % (str(k), str(v))) # Get all parameter values if __debug__: logger.debug("Processing parameters:") # logger.debug(args) values = get_task_params(num_params, logger, args) types = [x.content_type for x in values] if __debug__: logger.debug("RUN TASK with arguments:") logger.debug("\t- Path: %s" % path) logger.debug("\t- Method/function name: %s" % method_name) logger.debug("\t- Has target: %s" % str(has_target)) logger.debug("\t- # parameters: %s" % str(num_params)) # Next parameters are the values: # logger.debug("\t- Values:") # for v in values: # logger.debug("\t\t %r" % v) # logger.debug("\t- COMPSs types:") # for t in types: # logger.debug("\t\t %s" % str(t)) import_error = False if __debug__: logger.debug("LOAD TASK:") try: # Try to import the module (for functions) if __debug__: logger.debug("\t- Trying to import the user module: %s" % path) module = import_user_module(path, logger) except ImportError: if __debug__: msg = "\t- Could not import the module. Reason: Method in class." logger.debug(msg) import_error = True if __debug__: logger.debug("EXECUTE TASK:") if not import_error: # Module method declared as task result = task_execution(logger, process_name, module, method_name, time_out, types, values, compss_kwargs, persistent_storage, storage_conf) exit_code = result[0] new_types = result[1] new_values = result[2] # Next result: target_direction = result[3] timed_out = result[4] except_msg = result[5] else: # Method declared as task in class # Not the path of a module, it ends with a class name class_name = path.split('.')[-1] if '.' in path: module_name = '.'.join(path.split('.')[0:-1]) else: module_name = path try: module = __import__(module_name, fromlist=[class_name]) klass = getattr(module, class_name) except Exception: # noqa exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception(exc_type, exc_value, exc_traceback) logger.exception("EXCEPTION IMPORTING MODULE IN %s" % process_name) logger.exception(''.join(line for line in lines)) return 1, [], [], False, None, [] if __debug__: logger.debug("Method in class %s of module %s" % (class_name, module_name)) logger.debug("Has target: %s" % str(has_target)) if has_target == 'true': # Instance method # The self object needs to be an object in order to call the # function. So, it can not be done in the @task decorator. # Since the args structure is parameters + self + returns we pop # the corresponding considering the return_length notified by the # runtime (-1 due to index starts from 0). self_index = num_params - return_length - 1 self_elem = values.pop(self_index) self_type = types.pop(self_index) if self_type == parameter.TYPE.EXTERNAL_PSCO: if __debug__: logger.debug("Last element (self) is a PSCO with id: %s" % str(self_elem.content)) obj = get_by_id(self_elem.content) else: obj = None file_name = None if self_elem.content is None: file_name = self_elem.file_name.original_path if __debug__: logger.debug("\t- Deserialize self from file.") try: obj = deserialize_from_file(file_name) except Exception: # noqa exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception( exc_type, exc_value, exc_traceback) logger.exception("EXCEPTION DESERIALIZING SELF IN %s" % process_name) # noqa: E501 logger.exception(''.join(line for line in lines)) return 1, [], [], False, None, [] if __debug__: logger.debug("Deserialized self object is: %s" % self_elem.content) logger.debug( "Processing callee, a hidden object of %s in file %s" % # noqa: E501 (file_name, type(self_elem.content))) values.insert(0, obj) # noqa if not self_type == parameter.TYPE.EXTERNAL_PSCO: types.insert(0, parameter.TYPE.OBJECT) else: types.insert(0, parameter.TYPE.EXTERNAL_PSCO) result = task_execution(logger, process_name, klass, method_name, time_out, types, values, compss_kwargs, persistent_storage, storage_conf) exit_code = result[0] new_types = result[1] new_values = result[2] target_direction = result[3] timed_out = result[4] except_msg = result[5] # Depending on the target_direction option, it is necessary to # serialize again self or not. Since this option is only visible # within the task decorator, the task_execution returns the value # of target_direction in order to know here if self has to be # serialized. This solution avoids to use inspect. if target_direction is not None and \ (target_direction.direction == parameter.DIRECTION.INOUT or target_direction.direction == parameter.DIRECTION.COMMUTATIVE): # noqa: E501 if is_psco(obj): # There is no explicit update if self is a PSCO. # Consequently, the changes on the PSCO must have been # pushed into the storage automatically on each PSCO # modification. if __debug__: logger.debug("The changes on the PSCO must have been" + " automatically updated by the storage.") else: if __debug__: logger.debug("Serializing self to file: %s" % file_name) try: serialize_to_file(obj, file_name) except Exception: # noqa # Catch any serialization exception exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception( exc_type, exc_value, exc_traceback) logger.exception("EXCEPTION SERIALIZING SELF IN %s" % process_name) # noqa: E501 logger.exception(''.join(line for line in lines)) exit_code = 1 if __debug__: logger.debug("Obj: %r" % obj) else: # Class method - class is not included in values (e.g. values=[7]) types.append(None) # class must be first type result = task_execution(logger, process_name, klass, method_name, time_out, types, values, compss_kwargs, persistent_storage, storage_conf) exit_code = result[0] new_types = result[1] new_values = result[2] # Next return: target_direction = result[3] timed_out = result[4] except_msg = result[5] if __debug__: if exit_code != 0: logger.debug("EXECUTE TASK FAILED: Exit code: %s" % str(exit_code)) else: logger.debug("END TASK execution. Status: Ok") return exit_code, new_types, new_values, timed_out, except_msg