def compss_persistent_executor(config): # type: (PiperWorkerConfiguration) -> None """ Persistent executor main function. Retrieves the initial configuration and spawns the worker processes. :param config: Piper Worker Configuration description. :return: None """ COMM.gather(str(os.getpid()), root=0) # Catch SIGTERM sent by bindings_piper signal.signal(signal.SIGTERM, shutdown_handler) # Catch SIGUSER2 to solve strange behaviour with mpi4py signal.signal(signal.SIGUSR2, user_signal_handler) # Set the binding in worker mode import pycompss.util.context as context context.set_pycompss_context(context.WORKER) persistent_storage = (config.storage_conf != "null") logger, logger_cfg, storage_loggers, _ = load_loggers( config.debug, persistent_storage) cache_profiler = False if config.cache_profiler.lower() == 'true': cache_profiler = True if persistent_storage: # Initialize storage with event_worker(INIT_STORAGE_AT_WORKER_EVENT): from storage.api import initWorker as initStorageAtWorker # noqa initStorageAtWorker(config_file_path=config.storage_conf) process_name = "".join(("Rank-", str(RANK))) conf = ExecutorConf(config.debug, get_temporary_directory(), TRACING, config.storage_conf, logger, logger_cfg, persistent_storage, storage_loggers, config.stream_backend, config.stream_master_name, config.stream_master_port, CACHE_IDS, CACHE_QUEUE, cache_profiler) executor(None, process_name, config.pipes[RANK - 1], conf) if persistent_storage: # Finish storage if __debug__: logger.debug(HEADER + "Stopping persistent storage") with event_worker(FINISH_STORAGE_AT_WORKER_EVENT): from storage.api import finishWorker as finishStorageAtWorker # noqa finishStorageAtWorker()
def compss_persistent_executor(config): """ Persistent worker main function. Retrieves the initial configuration and spawns the worker processes. :param config: Piper Worker Configuration description :return: None """ import os COMM.gather(str(os.getpid()), root=0) # Catch SIGTERM sent by bindings_piper signal.signal(signal.SIGTERM, shutdown_handler) # Catch SIGUSER2 to solve strange behaviour with mpi4py signal.signal(signal.SIGUSR2, user_signal_handler) # Set the binding in worker mode import pycompss.util.context as context context.set_pycompss_context(context.WORKER) persistent_storage = (config.storage_conf != 'null') logger, storage_loggers = load_loggers(config.debug, persistent_storage, config.tracing) if persistent_storage: # Initialize storage with event(INIT_STORAGE_AT_WORKER_EVENT): from storage.api import initWorker as initStorageAtWorker initStorageAtWorker(config_file_path=config.storage_conf) process_name = 'Rank-' + str(RANK) conf = ExecutorConf(TRACING, config.storage_conf, logger, storage_loggers, config.stream_backend, config.stream_master_name, config.stream_master_port) executor(None, process_name, config.pipes[RANK - 1], conf) if persistent_storage: # Finish storage if __debug__: logger.debug(HEADER + "Stopping persistent storage") with event(FINISH_STORAGE_AT_WORKER_EVENT): from storage.api import finishWorker as finishStorageAtWorker finishStorageAtWorker()
def main(): # Emit sync event if tracing is enabled tracing = sys.argv[1] == 'true' task_id = int(sys.argv[2]) log_level = sys.argv[3] storage_conf = sys.argv[4] stream_backend = sys.argv[5] stream_master_name = sys.argv[6] stream_master_port = sys.argv[7] # method_type = sys.argv[8] params = sys.argv[9:] # class_name = sys.argv[9] # method_name = sys.argv[10] # num_slaves = sys.argv[11] # i = 11 + num_slaves # slaves = sys.argv[11..i] # numCus = sys.argv[i+1] # has_target = sys.argv[i+2] == 'true' # num_params = int(sys.argv[i+3]) # params = sys.argv[i+4..] print("tracing = " + str(tracing)) print("task_id = " + str(task_id)) print("log_level = " + str(log_level)) print("storage_conf = " + str(storage_conf)) persistent_storage = False if storage_conf != 'null': persistent_storage = True streaming = False if stream_backend not in [None, 'null', 'NONE']: streaming = True with trace_multiprocessing_worker() if tracing else dummy_context(): if streaming: # Start streaming DistroStreamClientHandler.init_and_start( master_ip=stream_master_name, master_port=stream_master_port) # Load log level configuration file worker_path = os.path.dirname(os.path.realpath(__file__)) if log_level == 'true' or log_level == "debug": # Debug init_logging_worker( worker_path + '/../../../log/logging_gat_worker_debug.json', tracing) elif log_level == "info" or log_level == "off": # Info or no debug init_logging_worker( worker_path + '/../../../log/logging_gat_worker_off.json', tracing) else: # Default init_logging_worker( worker_path + '/../../../log/logging_gat_worker.json', tracing) if persistent_storage: # Initialize storage with event(INIT_STORAGE_AT_WORKER_EVENT): from storage.api import initWorker as initStorageAtWorker initStorageAtWorker(config_file_path=storage_conf) # Init worker exit_code = compss_worker(tracing, str(task_id), storage_conf, params) if streaming: # Finish streaming DistroStreamClientHandler.set_stop() if persistent_storage: # Finish storage with event(FINISH_STORAGE_AT_WORKER_EVENT): from storage.api import finishWorker as finishStorageAtWorker finishStorageAtWorker() if exit_code == 1: exit(1)
def compss_persistent_worker(): # Get args debug = (sys.argv[1] == 'true') tracing = (sys.argv[2] == 'true') storage_conf = sys.argv[3] tasks_x_node = int(sys.argv[4]) in_pipes = sys.argv[5:5 + tasks_x_node] out_pipes = sys.argv[5 + tasks_x_node:] if tracing: import pyextrae.multiprocessing as pyextrae pyextrae.eventandcounters(SYNC_EVENTS, 1) pyextrae.eventandcounters(TASK_EVENTS, WORKER_RUNNING) if debug: assert tasks_x_node == len(in_pipes) assert tasks_x_node == len(out_pipes) persistent_storage = False if storage_conf != 'null': persistent_storage = True from storage.api import initWorker as initStorageAtWorker from storage.api import finishWorker as finishStorageAtWorker # Load log level configuration file worker_path = os.path.dirname(os.path.realpath(__file__)) if debug: # Debug init_logging_worker(worker_path + '/../../log/logging.json.debug') else: # Default init_logging_worker(worker_path + '/../../log/logging.json.off') if __debug__: logger = logging.getLogger('pycompss.worker.worker') logger.debug("[PYTHON WORKER] piper_worker.py wake up") logger.debug("[PYTHON WORKER] -----------------------------") logger.debug("[PYTHON WORKER] Persistent worker parameters:") logger.debug("[PYTHON WORKER] -----------------------------") logger.debug("[PYTHON WORKER] Debug : " + str(debug)) logger.debug("[PYTHON WORKER] Tracing : " + str(tracing)) logger.debug("[PYTHON WORKER] Tasks per node : " + str(tasks_x_node)) logger.debug("[PYTHON WORKER] In Pipes : " + str(in_pipes)) logger.debug("[PYTHON WORKER] Out Pipes : " + str(out_pipes)) logger.debug("[PYTHON WORKER] Storage conf. : " + str(storage_conf)) logger.debug("[PYTHON WORKER] -----------------------------") if persistent_storage: # Initialize storage initStorageAtWorker(config_file_path=storage_conf) # Create new threads queues = [] for i in range(0, tasks_x_node): if __debug__: logger.debug("[PYTHON WORKER] Launching process " + str(i)) process_name = 'Process-' + str(i) queues.append(Queue()) def create_threads(): processes.append(Process(target=worker, args=(queues[i], process_name, in_pipes[i], out_pipes[i], storage_conf))) processes[i].start() create_threads() # Catch SIGTERM send by bindings_piper to exit all processes signal.signal(signal.SIGTERM, shutdown_handler) # Wait for all threads for i in range(0, tasks_x_node): processes[i].join() # Check if there is any exception message from the threads for i in range(0, tasks_x_node): if not queues[i].empty: print(queues[i].get()) for q in queues: q.close() q.join_thread() if persistent_storage: # Finish storage finishStorageAtWorker() if __debug__: logger.debug("[PYTHON WORKER] Finished") if tracing: pyextrae.eventandcounters(TASK_EVENTS, 0) pyextrae.eventandcounters(SYNC_EVENTS, 0)
# pyextrae.eventandcounters(TASK_EVENTS, 0) pyextrae.eventandcounters(TASK_EVENTS, WORKER_INITIALIZATION) # Load log level configuration file worker_path = os.path.dirname(os.path.realpath(__file__)) if log_level == 'true' or log_level == "debug": # Debug init_logging_worker(worker_path + '/../../log/logging.json.debug') elif log_level == "info" or log_level == "off": # Info or no debug init_logging_worker(worker_path + '/../../log/logging.json.off') else: # Default init_logging_worker(worker_path + '/../../log/logging.json') if persistent_storage: # Initialize storage initStorageAtWorker(config_file_path=storage_conf) # Init worker compss_worker(persistent_storage) if tracing: pyextrae.eventandcounters(TASK_EVENTS, 0) # pyextrae.eventandcounters(TASK_EVENTS, PROCESS_DESTRUCTION) pyextrae.eventandcounters(SYNC_EVENTS, taskId) if persistent_storage: # Finish storage finishStorageAtWorker()
def main(): # Emit sync event if tracing is enabled tracing = sys.argv[1] == 'true' task_id = int(sys.argv[2]) log_level = sys.argv[3] storage_conf = sys.argv[4] stream_backend = sys.argv[5] stream_master_name = sys.argv[6] stream_master_port = sys.argv[7] # method_type = sys.argv[8] params = sys.argv[9:] # class_name = sys.argv[9] # method_name = sys.argv[10] # num_slaves = sys.argv[11] # i = 11 + num_slaves # slaves = sys.argv[11..i] # numCus = sys.argv[i+1] # has_target = sys.argv[i+2] == 'true' # num_params = int(sys.argv[i+3]) # params = sys.argv[i+4..] print("tracing = " + str(tracing)) print("task_id = " + str(task_id)) print("log_level = " + str(log_level)) print("storage_conf = " + str(storage_conf)) persistent_storage = False if storage_conf != 'null': persistent_storage = True from storage.api import initWorker as initStorageAtWorker from storage.api import finishWorker as finishStorageAtWorker streaming = False if stream_backend not in [None, 'null', 'NONE']: streaming = True if tracing: # Start tracing import pyextrae.multiprocessing as pyextrae pyextrae.eventandcounters(SYNC_EVENTS, task_id) # pyextrae.eventandcounters(TASK_EVENTS, 0) pyextrae.eventandcounters(TASK_EVENTS, WORKER_RUNNING_EVENT) if streaming: # Start streaming DistroStreamClientHandler.init_and_start( master_ip=stream_master_name, master_port=stream_master_port) # Load log level configuration file worker_path = os.path.dirname(os.path.realpath(__file__)) if log_level == 'true' or log_level == "debug": # Debug init_logging_worker(worker_path + '/../../../log/logging_debug.json') elif log_level == "info" or log_level == "off": # Info or no debug init_logging_worker(worker_path + '/../../../log/logging_off.json') else: # Default init_logging_worker(worker_path + '/../../../log/logging.json') if persistent_storage: # Initialize storage initStorageAtWorker(config_file_path=storage_conf) # Init worker exit_code = compss_worker(tracing, str(task_id), storage_conf, params) if tracing: # Finish tracing pyextrae.eventandcounters(TASK_EVENTS, 0) # pyextrae.eventandcounters(TASK_EVENTS, PROCESS_DESTRUCTION) pyextrae.eventandcounters(SYNC_EVENTS, task_id) if streaming: # Finish streaming DistroStreamClientHandler.set_stop() if persistent_storage: # Finish storage finishStorageAtWorker() if exit_code == 1: exit(1)
def compss_persistent_worker(config): # type: (PiperWorkerConfiguration) -> None """ Persistent worker main function. Retrieves the initial configuration and spawns the worker processes. :param config: Piper Worker Configuration description. :return: None """ pids = COMM.gather(str(os.getpid()), root=0) # Catch SIGTERM sent by bindings_piper signal.signal(signal.SIGTERM, shutdown_handler) # Catch SIGUSER2 to solve strange behaviour with mpi4py signal.signal(signal.SIGUSR2, user_signal_handler) # Set the binding in worker mode import pycompss.util.context as context context.set_pycompss_context(context.WORKER) persistent_storage = (config.storage_conf != 'null') logger, _, _ = load_loggers(config.debug, persistent_storage) if __debug__: logger.debug(HEADER + "mpi_piper_worker.py rank: " + str(RANK) + " wake up") config.print_on_logger(logger) # Start storage if persistent_storage: # Initialize storage if __debug__: logger.debug(HEADER + "Starting persistent storage") from storage.api import initWorker as initStorageAtWorker # noqa initStorageAtWorker(config_file_path=config.storage_conf) for i in range(0, config.tasks_x_node): child_in_pipe = config.pipes[i].input_pipe try: child_pid = pids[i + 1] except IndexError: child_pid = pids[i] PROCESSES[child_in_pipe] = child_pid if __debug__: logger.debug(HEADER + "Starting alive") logger.debug(HEADER + "Control pipe: " + str(config.control_pipe)) # Read command from control pipe alive = True control_pipe = config.control_pipe while alive: command = control_pipe.read_command() if command != "": line = command.split() if line[0] == ADD_EXECUTOR_TAG: in_pipe = line[1] out_pipe = line[2] control_pipe.write(" ".join( (ADD_EXECUTOR_FAILED_TAG, out_pipe, in_pipe, str(0)))) elif line[0] == REMOVE_EXECUTOR_TAG: in_pipe = line[1] out_pipe = line[2] PROCESSES.pop(in_pipe, None) control_pipe.write(" ".join( (REMOVED_EXECUTOR_TAG, out_pipe, in_pipe))) elif line[0] == QUERY_EXECUTOR_ID_TAG: in_pipe = line[1] out_pipe = line[2] pid = PROCESSES.get(in_pipe) control_pipe.write(" ".join( (REPLY_EXECUTOR_ID_TAG, out_pipe, in_pipe, str(pid)))) elif line[0] == CANCEL_TASK_TAG: in_pipe = line[1] pid = PROCESSES.get(in_pipe) logger.debug(HEADER + "Signaling process with PID " + pid + " to cancel a task") os.kill( int(pid), signal.SIGUSR2) # NOSONAR cancellation produced by COMPSs elif line[0] == PING_TAG: control_pipe.write(PONG_TAG) elif line[0] == QUIT_TAG: alive = False else: logger.debug(HEADER + "ERROR: UNKNOWN COMMAND: " + command) alive = False # Stop storage if persistent_storage: # Finish storage logger.debug(HEADER + "Stopping persistent storage") from storage.api import finishWorker as finishStorageAtWorker # noqa finishStorageAtWorker() if __debug__: logger.debug(HEADER + "Finished") control_pipe.write(QUIT_TAG) control_pipe.close()
def main(): # type: () -> None """ GAT worker main code. Executes the task provided by parameters. :return: None """ # Emit sync event if tracing is enabled tracing = sys.argv[1] == 'true' task_id = int(sys.argv[2]) log_level = sys.argv[3] storage_conf = sys.argv[4] stream_backend = sys.argv[5] stream_master_name = sys.argv[6] stream_master_port = sys.argv[7] # Next: method_type = sys.argv[8] params = sys.argv[9:] # Next parameters: # class_name = sys.argv[10] # method_name = sys.argv[11] # num_slaves = sys.argv[12] # i = 13 + num_slaves # slaves = sys.argv[12..i] # numCus = sys.argv[i+1] # has_target = sys.argv[i+2] == 'true' # num_params = int(sys.argv[i+3]) # params = sys.argv[i+4..] if log_level == "true" or log_level == "debug": print("Tracing = " + str(tracing)) print("Task id = " + str(task_id)) print("Log level = " + str(log_level)) print("Storage conf = " + str(storage_conf)) persistent_storage = False if storage_conf != "null": persistent_storage = True streaming = False if stream_backend not in [None, "null", "NONE"]: streaming = True with trace_multiprocessing_worker() if tracing else dummy_context(): if streaming: # Start streaming DistroStreamClientHandler.init_and_start( master_ip=stream_master_name, master_port=stream_master_port) # Load log level configuration file worker_path = os.path.dirname(os.path.realpath(__file__)) if log_level == "true" or log_level == "debug": # Debug log_json = "".join( (worker_path, "/../../../log/logging_gat_worker_debug.json")) elif log_level == "info" or log_level == "off": # Info or no debug log_json = "".join( (worker_path, "/../../../log/logging_gat_worker_off.json")) else: # Default log_json = "".join( (worker_path, "/../../../log/logging_gat_worker.json")) init_logging_worker(log_json, tracing) if persistent_storage: # Initialize storage with event_worker(INIT_STORAGE_AT_WORKER_EVENT): from storage.api import initWorker as initStorageAtWorker # noqa initStorageAtWorker(config_file_path=storage_conf) # Init worker exit_code = compss_worker(tracing, str(task_id), storage_conf, params, log_json) if streaming: # Finish streaming DistroStreamClientHandler.set_stop() if persistent_storage: # Finish storage with event_worker(FINISH_STORAGE_AT_WORKER_EVENT): from storage.api import finishWorker as finishStorageAtWorker # noqa finishStorageAtWorker() if exit_code == 1: exit(1)
def compss_persistent_worker(config): # type: (PiperWorkerConfiguration) -> None """ Persistent worker main function. Retrieves the initial configuration and spawns the worker processes. :param config: Piper Worker Configuration description. :return: None """ global CACHE global CACHE_PROCESS # Catch SIGTERM sent by bindings_piper signal.signal(signal.SIGTERM, shutdown_handler) # Set the binding in worker mode context.set_pycompss_context(context.WORKER) persistent_storage = (config.storage_conf != 'null') logger, logger_cfg, storage_loggers, log_dir = load_loggers(config.debug, persistent_storage) if __debug__: logger.debug(HEADER + "piper_worker.py wake up") config.print_on_logger(logger) if persistent_storage: # Initialize storage logger.debug(HEADER + "Starting persistent storage") with event_worker(INIT_STORAGE_AT_WORKER_EVENT): from storage.api import initWorker as initStorageAtWorker # noqa initStorageAtWorker(config_file_path=config.storage_conf) # Create new processes queues = [] cache_profiler = False if config.cache_profiler.lower() == 'true': cache_profiler = True # Setup cache if is_cache_enabled(str(config.cache)): # Deploy the necessary processes CACHE = True cache_params = start_cache(logger, str(config.cache), cache_profiler, log_dir) else: # No cache CACHE = False cache_params = (None, None, None, None) # type: ignore smm, CACHE_PROCESS, cache_queue, cache_ids = cache_params # Create new executor processes conf = ExecutorConf(config.debug, get_temporary_directory(), TRACING, config.storage_conf, logger, logger_cfg, persistent_storage, storage_loggers, config.stream_backend, config.stream_master_name, config.stream_master_port, cache_ids, cache_queue, cache_profiler) for i in range(0, config.tasks_x_node): if __debug__: logger.debug(HEADER + "Launching process " + str(i)) process_name = "".join(("Process-", str(i))) pid, queue = create_executor_process(process_name, conf, config.pipes[i]) queues.append(queue) # Read command from control pipe alive = True process_counter = config.tasks_x_node control_pipe = config.control_pipe # type: typing.Any while alive: command = control_pipe.read_command(retry_period=1) if command != "": line = command.split() if line[0] == ADD_EXECUTOR_TAG: process_name = "".join(("Process-", str(process_counter))) process_counter = process_counter + 1 in_pipe = line[1] out_pipe = line[2] pipe = Pipe(in_pipe, out_pipe) pid, queue = create_executor_process(process_name, conf, pipe) queues.append(queue) control_pipe.write(" ".join((ADDED_EXECUTOR_TAG, out_pipe, in_pipe, str(pid)))) elif line[0] == QUERY_EXECUTOR_ID_TAG: in_pipe = line[1] out_pipe = line[2] proc = PROCESSES.get(in_pipe) # type: typing.Any pid = proc.pid control_pipe.write(" ".join((REPLY_EXECUTOR_ID_TAG, out_pipe, in_pipe, str(pid)))) elif line[0] == CANCEL_TASK_TAG: in_pipe = line[1] cancel_proc = PROCESSES.get(in_pipe) # type: typing.Any cancel_pid = cancel_proc.pid if __debug__: logger.debug(HEADER + "Signaling process with PID " + str(cancel_pid) + " to cancel a task") os.kill(cancel_pid, signal.SIGUSR2) # NOSONAR cancellation produced by COMPSs elif line[0] == REMOVE_EXECUTOR_TAG: in_pipe = line[1] out_pipe = line[2] proc = PROCESSES.pop(in_pipe, None) if proc: if proc.is_alive(): logger.warn(HEADER + "Forcing terminate on : " + proc.name) proc.terminate() proc.join() control_pipe.write(" ".join((REMOVED_EXECUTOR_TAG, out_pipe, in_pipe))) elif line[0] == PING_TAG: control_pipe.write(PONG_TAG) elif line[0] == QUIT_TAG: alive = False # Wait for all threads for proc in PROCESSES.values(): proc.join() # Check if there is any exception message from the threads for i in range(0, config.tasks_x_node): if not queues[i].empty: logger.error(HEADER + "Exception in threads queue: " + str(queues[i].get())) for queue in queues: queue.close() queue.join_thread() if CACHE: stop_cache(smm, cache_queue, cache_profiler, CACHE_PROCESS) # noqa if persistent_storage: # Finish storage if __debug__: logger.debug(HEADER + "Stopping persistent storage") with event_worker(FINISH_STORAGE_AT_WORKER_EVENT): from storage.api import finishWorker as finishStorageAtWorker # noqa finishStorageAtWorker() if __debug__: logger.debug(HEADER + "Finished") control_pipe.write(QUIT_TAG) control_pipe.close()
def compss_persistent_worker(config): """ Persistent worker main function. Retrieves the initial configuration and spawns the worker processes. :param config: Piper Worker Configuration description :return: None """ # Catch SIGTERM sent by bindings_piper signal.signal(signal.SIGTERM, shutdown_handler) # Set the binding in worker mode context.set_pycompss_context(context.WORKER) if TRACING: try: user_paths = os.environ['PYTHONPATH'] except KeyError: user_paths = "" print("PYTHON PATH = " + user_paths) import pyextrae.multiprocessing as pyextrae pyextrae.eventandcounters(SYNC_EVENTS, 1) pyextrae.eventandcounters(TASK_EVENTS, WORKER_RUNNING_EVENT) persistent_storage = (config.storage_conf != 'null') logger, storage_loggers = load_loggers(config.debug, persistent_storage) if __debug__: logger.debug(HEADER + "piper_worker.py wake up") config.print_on_logger(logger) if persistent_storage: # Initialize storage logger.debug(HEADER + "Starting persitent storage") from storage.api import initWorker as initStorageAtWorker initStorageAtWorker(config_file_path=config.storage_conf) # Create new threads queues = [] for i in range(0, config.tasks_x_node): if __debug__: logger.debug(HEADER + "Launching process " + str(i)) process_name = 'Process-' + str(i) queue = Queue() queues.append(queue) conf = ExecutorConf(TRACING, config.storage_conf, logger, storage_loggers, config.stream_backend, config.stream_master_name, config.stream_master_port) process = Process(target=executor, args=(queue, process_name, config.pipes[i], conf)) PROCESSES[config.pipes[i].input_pipe] = process process.start() # Read command from control pipe alive = True process_counter = config.tasks_x_node control_pipe = config.control_pipe while alive: command = control_pipe.read_command(retry_period=1) if command != "": line = command.split() if line[0] == ADD_EXECUTOR_TAG: process_name = 'Process-' + str(process_counter) process_counter = process_counter + 1 in_pipe = line[1] out_pipe = line[2] pipe = Pipe(in_pipe, out_pipe) pid = create_threads(process_name, pipe) control_pipe.write(ADDED_EXECUTOR_TAG + " " + out_pipe + " " + in_pipe + " " + str(pid)) elif line[0] == QUERY_EXECUTOR_ID_TAG: in_pipe = line[1] out_pipe = line[2] proc = PROCESSES.get(in_pipe) pid = proc.pid control_pipe.write(REPLY_EXECUTOR_ID_TAG + " " + out_pipe + " " + in_pipe + " " + str(pid)) elif line[0] == CANCEL_TASK_TAG: in_pipe = line[1] proc = PROCESSES.get(in_pipe) pid = proc.pid logger.debug("[PYTHON WORKER] Signaling process with PID " + str(pid) + " to cancel a task") kill(pid, signal.SIGUSR2) elif line[0] == REMOVE_EXECUTOR_TAG: in_pipe = line[1] out_pipe = line[2] proc = PROCESSES.pop(in_pipe, None) if proc: if proc.is_alive(): logger.warn(HEADER + "Forcing terminate on : " + proc.name) proc.terminate() proc.join() control_pipe.write(REMOVED_EXECUTOR_TAG + " " + out_pipe + " " + in_pipe) elif line[0] == PING_TAG: control_pipe.write(PONG_TAG) elif line[0] == QUIT_TAG: alive = False # Wait for all threads for proc in PROCESSES.values(): proc.join() # Check if there is any exception message from the threads for i in range(0, config.tasks_x_node): if not queues[i].empty: logger.error(HEADER + "Exception in threads queue: " + str(queues[i].get())) for queue in queues: queue.close() queue.join_thread() if persistent_storage: # Finish storage logger.debug(HEADER + "Stopping persistent storage") from storage.api import finishWorker as finishStorageAtWorker finishStorageAtWorker() if __debug__: logger.debug(HEADER + "Finished") if TRACING: pyextrae.eventandcounters(TASK_EVENTS, 0) pyextrae.eventandcounters(SYNC_EVENTS, 0) control_pipe.write(QUIT_TAG) control_pipe.close()