def init_streaming(streaming_backend, streaming_master_name, streaming_master_port, logger): """ Initialize the streaming client. :param streaming_backend: Streaming backend. :param streaming_master_name: Streaming backend master node name. :param streaming_master_port: Streaming backend master port. :param logger: Logger :return: <Boolean> True if initialized successfully, False otherwise. """ # Fix options if necessary if streaming_master_name is None or not streaming_master_name or streaming_master_name == "null": streaming_master_name = "localhost" # Check if the stream backend is enabled streaming_enabled = streaming_backend is not None \ and streaming_backend \ and streaming_backend != "null" \ and streaming_backend != "NONE" # Init stream backend if needed if streaming_enabled: if __debug__: logger.debug("Starting DistroStream library") DistroStreamClientHandler.init_and_start( master_ip=streaming_master_name, master_port=int(streaming_master_port)) # Return whether the streaming backend is enabled or not return streaming_enabled
def init_streaming(streaming_backend, streaming_master_name, streaming_master_port): # type: (str, str, str) -> bool """ Initialize the streaming client. :param streaming_backend: Streaming backend. :param streaming_master_name: Streaming backend master node name. :param streaming_master_port: Streaming backend master port. :return: True if initialized successfully, False otherwise. """ # Fix options if necessary if streaming_master_name == "" or \ not streaming_master_name or \ streaming_master_name == "null": streaming_master_name = "localhost" if streaming_master_port == "" or \ not streaming_master_port or \ streaming_master_port == "null": streaming_master_port = "49049" # Check if the stream backend is enabled if streaming_backend != "" \ and streaming_backend != "null" \ and streaming_backend != "None" \ and streaming_backend != "NONE": streaming_enabled = True else: streaming_enabled = False # Init stream backend if needed if streaming_enabled: if __debug__: logger.debug("Starting DistroStream library") DistroStreamClientHandler.init_and_start(master_ip=streaming_master_name, # noqa: E501 master_port=streaming_master_port) # noqa: E501 # Return whether the streaming backend is enabled or not return streaming_enabled
def main(): # Emit sync event if tracing is enabled tracing = sys.argv[1] == 'true' task_id = int(sys.argv[2]) log_level = sys.argv[3] storage_conf = sys.argv[4] stream_backend = sys.argv[5] stream_master_name = sys.argv[6] stream_master_port = sys.argv[7] # method_type = sys.argv[8] params = sys.argv[9:] # class_name = sys.argv[9] # method_name = sys.argv[10] # num_slaves = sys.argv[11] # i = 11 + num_slaves # slaves = sys.argv[11..i] # numCus = sys.argv[i+1] # has_target = sys.argv[i+2] == 'true' # num_params = int(sys.argv[i+3]) # params = sys.argv[i+4..] print("tracing = " + str(tracing)) print("task_id = " + str(task_id)) print("log_level = " + str(log_level)) print("storage_conf = " + str(storage_conf)) persistent_storage = False if storage_conf != 'null': persistent_storage = True streaming = False if stream_backend not in [None, 'null', 'NONE']: streaming = True with trace_multiprocessing_worker() if tracing else dummy_context(): if streaming: # Start streaming DistroStreamClientHandler.init_and_start( master_ip=stream_master_name, master_port=stream_master_port) # Load log level configuration file worker_path = os.path.dirname(os.path.realpath(__file__)) if log_level == 'true' or log_level == "debug": # Debug init_logging_worker( worker_path + '/../../../log/logging_gat_worker_debug.json', tracing) elif log_level == "info" or log_level == "off": # Info or no debug init_logging_worker( worker_path + '/../../../log/logging_gat_worker_off.json', tracing) else: # Default init_logging_worker( worker_path + '/../../../log/logging_gat_worker.json', tracing) if persistent_storage: # Initialize storage with event(INIT_STORAGE_AT_WORKER_EVENT): from storage.api import initWorker as initStorageAtWorker initStorageAtWorker(config_file_path=storage_conf) # Init worker exit_code = compss_worker(tracing, str(task_id), storage_conf, params) if streaming: # Finish streaming DistroStreamClientHandler.set_stop() if persistent_storage: # Finish storage with event(FINISH_STORAGE_AT_WORKER_EVENT): from storage.api import finishWorker as finishStorageAtWorker finishStorageAtWorker() if exit_code == 1: exit(1)
def executor(queue, process_name, pipe, conf): """ Thread main body - Overrides Threading run method. Iterates over the input pipe in order to receive tasks (with their parameters) and process them. Notifies the runtime when each task has finished with the corresponding output value. Finishes when the "quit" message is received. :param queue: Queue where to put exception messages :param process_name: Process name (Thread-X, where X is the thread id). :param pipe: Pipe to receive and send messages from/to the runtime. :param conf: configuration of the executor :return: None """ # Replace Python Worker's SIGTERM handler. signal.signal(signal.SIGTERM, shutdown_handler) tracing = conf.tracing storage_conf = conf.storage_conf logger = conf.logger storage_loggers = conf.storage_loggers # Get a copy of the necessary information from the logger to re-establish # after each task logger_handlers = copy.copy(logger.handlers) logger_level = logger.getEffectiveLevel() logger_formatter = logging.Formatter(logger_handlers[0].formatter._fmt) storage_loggers_handlers = [] for storage_logger in storage_loggers: storage_loggers_handlers.append(copy.copy(storage_logger.handlers)) if storage_conf != 'null': try: from storage.api import initWorkerPostFork initWorkerPostFork() except ImportError: if __debug__: logger.info( HEADER + "[%s] Could not find initWorkerPostFork storage call. Ignoring it." % # noqa: E501 str(process_name)) # Start the streaming backend if necessary streaming = False if conf.stream_backend not in [None, 'null', 'NONE']: streaming = True if streaming: # Initialize streaming logger.debug(HEADER + "Starting streaming for process " + str(process_name)) try: DistroStreamClientHandler.init_and_start( master_ip=conf.stream_master_ip, master_port=int(conf.stream_master_port)) except Exception as e: logger.error(e) raise e # Process properties alive = True if __debug__: logger.debug(HEADER + "[%s] Starting process" % str(process_name)) # MAIN EXECUTOR LOOP while alive: # Runtime -> pipe - Read command from pipe command = pipe.read_command(retry_period=0.5) if command != "": logger.debug(HEADER + "Received %s" % command) # Process the command alive = process_task(command, process_name, pipe, queue, tracing, logger, logger_handlers, logger_level, logger_formatter, storage_conf, storage_loggers, storage_loggers_handlers) # Stop storage if storage_conf != 'null': try: from storage.api import finishWorkerPostFork finishWorkerPostFork() except ImportError: if __debug__: logger.info( HEADER + "[%s] Could not find finishWorkerPostFork storage call. Ignoring it." % # noqa: E501 str(process_name)) # Stop streaming if streaming: logger.debug(HEADER + "Stopping streaming for process " + str(process_name)) DistroStreamClientHandler.set_stop() sys.stdout.flush() sys.stderr.flush() if __debug__: logger.debug(HEADER + "[%s] Exiting process " % str(process_name)) pipe.write(QUIT_TAG) pipe.close()
def main(): # Emit sync event if tracing is enabled tracing = sys.argv[1] == 'true' task_id = int(sys.argv[2]) log_level = sys.argv[3] storage_conf = sys.argv[4] stream_backend = sys.argv[5] stream_master_name = sys.argv[6] stream_master_port = sys.argv[7] # method_type = sys.argv[8] params = sys.argv[9:] # class_name = sys.argv[9] # method_name = sys.argv[10] # num_slaves = sys.argv[11] # i = 11 + num_slaves # slaves = sys.argv[11..i] # numCus = sys.argv[i+1] # has_target = sys.argv[i+2] == 'true' # num_params = int(sys.argv[i+3]) # params = sys.argv[i+4..] print("tracing = " + str(tracing)) print("task_id = " + str(task_id)) print("log_level = " + str(log_level)) print("storage_conf = " + str(storage_conf)) persistent_storage = False if storage_conf != 'null': persistent_storage = True from storage.api import initWorker as initStorageAtWorker from storage.api import finishWorker as finishStorageAtWorker streaming = False if stream_backend not in [None, 'null', 'NONE']: streaming = True if tracing: # Start tracing import pyextrae.multiprocessing as pyextrae pyextrae.eventandcounters(SYNC_EVENTS, task_id) # pyextrae.eventandcounters(TASK_EVENTS, 0) pyextrae.eventandcounters(TASK_EVENTS, WORKER_RUNNING_EVENT) if streaming: # Start streaming DistroStreamClientHandler.init_and_start( master_ip=stream_master_name, master_port=stream_master_port) # Load log level configuration file worker_path = os.path.dirname(os.path.realpath(__file__)) if log_level == 'true' or log_level == "debug": # Debug init_logging_worker(worker_path + '/../../../log/logging_debug.json') elif log_level == "info" or log_level == "off": # Info or no debug init_logging_worker(worker_path + '/../../../log/logging_off.json') else: # Default init_logging_worker(worker_path + '/../../../log/logging.json') if persistent_storage: # Initialize storage initStorageAtWorker(config_file_path=storage_conf) # Init worker exit_code = compss_worker(tracing, str(task_id), storage_conf, params) if tracing: # Finish tracing pyextrae.eventandcounters(TASK_EVENTS, 0) # pyextrae.eventandcounters(TASK_EVENTS, PROCESS_DESTRUCTION) pyextrae.eventandcounters(SYNC_EVENTS, task_id) if streaming: # Finish streaming DistroStreamClientHandler.set_stop() if persistent_storage: # Finish storage finishStorageAtWorker() if exit_code == 1: exit(1)
def main(): # type: () -> None """ GAT worker main code. Executes the task provided by parameters. :return: None """ # Emit sync event if tracing is enabled tracing = sys.argv[1] == 'true' task_id = int(sys.argv[2]) log_level = sys.argv[3] storage_conf = sys.argv[4] stream_backend = sys.argv[5] stream_master_name = sys.argv[6] stream_master_port = sys.argv[7] # Next: method_type = sys.argv[8] params = sys.argv[9:] # Next parameters: # class_name = sys.argv[10] # method_name = sys.argv[11] # num_slaves = sys.argv[12] # i = 13 + num_slaves # slaves = sys.argv[12..i] # numCus = sys.argv[i+1] # has_target = sys.argv[i+2] == 'true' # num_params = int(sys.argv[i+3]) # params = sys.argv[i+4..] if log_level == "true" or log_level == "debug": print("Tracing = " + str(tracing)) print("Task id = " + str(task_id)) print("Log level = " + str(log_level)) print("Storage conf = " + str(storage_conf)) persistent_storage = False if storage_conf != "null": persistent_storage = True streaming = False if stream_backend not in [None, "null", "NONE"]: streaming = True with trace_multiprocessing_worker() if tracing else dummy_context(): if streaming: # Start streaming DistroStreamClientHandler.init_and_start( master_ip=stream_master_name, master_port=stream_master_port) # Load log level configuration file worker_path = os.path.dirname(os.path.realpath(__file__)) if log_level == "true" or log_level == "debug": # Debug log_json = "".join( (worker_path, "/../../../log/logging_gat_worker_debug.json")) elif log_level == "info" or log_level == "off": # Info or no debug log_json = "".join( (worker_path, "/../../../log/logging_gat_worker_off.json")) else: # Default log_json = "".join( (worker_path, "/../../../log/logging_gat_worker.json")) init_logging_worker(log_json, tracing) if persistent_storage: # Initialize storage with event_worker(INIT_STORAGE_AT_WORKER_EVENT): from storage.api import initWorker as initStorageAtWorker # noqa initStorageAtWorker(config_file_path=storage_conf) # Init worker exit_code = compss_worker(tracing, str(task_id), storage_conf, params, log_json) if streaming: # Finish streaming DistroStreamClientHandler.set_stop() if persistent_storage: # Finish storage with event_worker(FINISH_STORAGE_AT_WORKER_EVENT): from storage.api import finishWorker as finishStorageAtWorker # noqa finishStorageAtWorker() if exit_code == 1: exit(1)
def executor(queue, process_name, pipe, conf): # type: (typing.Union[None, Queue], str, Pipe, typing.Any) -> None """Thread main body - Overrides Threading run method. Iterates over the input pipe in order to receive tasks (with their parameters) and process them. Notifies the runtime when each task has finished with the corresponding output value. Finishes when the "quit" message is received. :param queue: Queue where to put exception messages. :param process_name: Process name (Thread-X, where X is the thread id). :param pipe: Pipe to receive and send messages from/to the runtime. :param conf: configuration of the executor. :return: None """ try: # Replace Python Worker's SIGTERM handler. signal.signal(signal.SIGTERM, shutdown_handler) if len(conf.logger.handlers) == 0: # Logger has not been inherited correctly. Happens in MacOS. set_temporary_directory(conf.tmp_dir, create_tmpdir=False) # Reload logger conf.logger, conf.logger_cfg, conf.storage_loggers, _ = \ load_loggers(conf.debug, conf.persistent_storage) # Set the binding in worker mode too context.set_pycompss_context(context.WORKER) logger = conf.logger tracing = conf.tracing storage_conf = conf.storage_conf storage_loggers = conf.storage_loggers # Get a copy of the necessary information from the logger to # re-establish after each task logger_handlers = copy.copy(logger.handlers) logger_level = logger.getEffectiveLevel() logger_formatter = logging.Formatter( logger_handlers[0].formatter._fmt) # noqa storage_loggers_handlers = [] for storage_logger in storage_loggers: storage_loggers_handlers.append(copy.copy(storage_logger.handlers)) # Establish link with the binding-commons to enable task nesting if __debug__: logger.debug(HEADER + "Establishing link with runtime in process " + str(process_name)) # noqa: E501 COMPSs.load_runtime(external_process=False, _logger=logger) COMPSs.set_pipes(pipe.output_pipe, pipe.input_pipe) if storage_conf != "null": try: from storage.api import initWorkerPostFork # noqa with event_worker(INIT_WORKER_POSTFORK_EVENT): initWorkerPostFork() except (ImportError, AttributeError): if __debug__: logger.info( HEADER + "[%s] Could not find initWorkerPostFork storage call. Ignoring it." % # noqa: E501 str(process_name)) # Start the streaming backend if necessary streaming = False if conf.stream_backend not in [None, "null", "NONE"]: streaming = True if streaming: # Initialize streaming logger.debug(HEADER + "Starting streaming for process " + str(process_name)) try: DistroStreamClientHandler.init_and_start( master_ip=conf.stream_master_ip, master_port=conf.stream_master_port) except Exception as e: logger.error(e) raise e # Connect to Shared memory manager if conf.cache_queue: load_shared_memory_manager() # Process properties alive = True if __debug__: logger.debug(HEADER + "[%s] Starting process" % str(process_name)) # MAIN EXECUTOR LOOP while alive: # Runtime -> pipe - Read command from pipe command = COMPSs.read_pipes() if command != "": if __debug__: logger.debug(HEADER + "[%s] Received command %s" % (str(process_name), str(command))) # Process the command alive = process_message( command, process_name, pipe, queue, tracing, logger, conf.logger_cfg, logger_handlers, logger_level, logger_formatter, storage_conf, storage_loggers, storage_loggers_handlers, conf.cache_queue, conf.cache_ids, conf.cache_profiler) # Stop storage if storage_conf != "null": try: from storage.api import finishWorkerPostFork # noqa with event_worker(FINISH_WORKER_POSTFORK_EVENT): finishWorkerPostFork() except (ImportError, AttributeError): if __debug__: logger.info( HEADER + "[%s] Could not find finishWorkerPostFork storage call. Ignoring it." % # noqa: E501 str(process_name)) # Stop streaming if streaming: logger.debug(HEADER + "Stopping streaming for process " + str(process_name)) DistroStreamClientHandler.set_stop() sys.stdout.flush() sys.stderr.flush() if __debug__: logger.debug(HEADER + "[%s] Exiting process " % str(process_name)) pipe.write(QUIT_TAG) pipe.close() except Exception as e: logger.error(e) raise e