def compss_persistent_executor(config):
# type: (PiperWorkerConfiguration) -> None
""" Persistent executor main function.
Retrieves the initial configuration and spawns the worker processes.
:param config: Piper Worker Configuration description.
:return: None
"""
COMM.gather(str(os.getpid()), root=0)
# Catch SIGTERM sent by bindings_piper
signal.signal(signal.SIGTERM, shutdown_handler)
# Catch SIGUSER2 to solve strange behaviour with mpi4py
signal.signal(signal.SIGUSR2, user_signal_handler)
# Set the binding in worker mode
import pycompss.util.context as context
context.set_pycompss_context(context.WORKER)
persistent_storage = (config.storage_conf != 'null')
logger, logger_cfg, storage_loggers = load_loggers(config.debug,
persistent_storage)
if persistent_storage:
# Initialize storage
with event(INIT_STORAGE_AT_WORKER_EVENT):
from storage.api import initWorker as initStorageAtWorker # noqa
initStorageAtWorker(config_file_path=config.storage_conf)
process_name = "".join(("Rank-", str(RANK)))
conf = ExecutorConf(TRACING, config.storage_conf, logger, logger_cfg,
storage_loggers, config.stream_backend,
config.stream_master_name, config.stream_master_port,
CACHE_IDS, CACHE_QUEUE)
executor(None, process_name, config.pipes[RANK - 1], conf)
if persistent_storage:
# Finish storage
if __debug__:
logger.debug(HEADER + "Stopping persistent storage")
with event(FINISH_STORAGE_AT_WORKER_EVENT):
from storage.api import finishWorker as finishStorageAtWorker # noqa
finishStorageAtWorker()
def compss_persistent_worker(config):
"""
Persistent worker main function.
Retrieves the initial configuration and spawns the worker processes.
:param config: Piper Worker Configuration description
:return: None
"""
import os
pids = COMM.gather(str(os.getpid()), root=0)
# Catch SIGTERM sent by bindings_piper
signal.signal(signal.SIGTERM, shutdown_handler)
# Catch SIGUSER2 to solve strange behaviour with mpi4py
signal.signal(signal.SIGUSR2, user_signal_handler)
# Set the binding in worker mode
import pycompss.util.context as context
context.set_pycompss_context(context.WORKER)
persistent_storage = (config.storage_conf != 'null')
logger, storage_loggers = load_loggers(config.debug, persistent_storage,
config.tracing)
if __debug__:
logger.debug(HEADER + "mpi_piper_worker.py rank: " + str(RANK) +
" wake up")
config.print_on_logger(logger)
# Start storage
if persistent_storage:
# Initialize storage
logger.debug(HEADER + "Starting persistent storage")
from storage.api import initWorker
initWorker(config_file_path=config.storage_conf)
for i in range(0, config.tasks_x_node):
child_in_pipe = config.pipes[i].input_pipe
child_pid = pids[i + 1]
PROCESSES[child_in_pipe] = child_pid
if __debug__:
logger.debug(HEADER + "Starting alive")
logger.debug(HEADER + "Control pipe: " + str(config.control_pipe))
# Read command from control pipe
alive = True
control_pipe = config.control_pipe
while alive:
command = control_pipe.read_command()
if command != "":
line = command.split()
if line[0] == REMOVE_EXECUTOR_TAG:
in_pipe = line[1]
out_pipe = line[2]
pid = PROCESSES.pop(in_pipe, None)
control_pipe.write(REMOVED_EXECUTOR_TAG + " " + out_pipe +
" " + in_pipe)
elif line[0] == QUERY_EXECUTOR_ID_TAG:
in_pipe = line[1]
out_pipe = line[2]
pid = PROCESSES.get(in_pipe)
control_pipe.write(REPLY_EXECUTOR_ID_TAG + " " + out_pipe +
" " + in_pipe + " " + str(pid))
elif line[0] == CANCEL_TASK_TAG:
in_pipe = line[1]
pid = PROCESSES.get(in_pipe)
logger.debug(HEADER + "Signaling process with PID " + pid +
" to cancel a task")
kill(int(pid), signal.SIGUSR2)
elif line[0] == PING_TAG:
control_pipe.write(PONG_TAG)
elif line[0] == QUIT_TAG:
alive = False
else:
logger.debug(HEADER + "ERROR: UNKNOWN COMMAND: " + command)
alive = False
# Stop storage
if persistent_storage:
# Finish storage
logger.debug(HEADER + "Stopping persistent storage")
from storage.api import finishWorker
finishWorker()
if __debug__:
logger.debug(HEADER + "Finished")
control_pipe.write(QUIT_TAG)
control_pipe.close()
def compss_persistent_worker(config):
# type: (PiperWorkerConfiguration) -> None
""" Persistent worker main function.
Retrieves the initial configuration and spawns the worker processes.
:param config: Piper Worker Configuration description.
:return: None
"""
global CACHE
global CACHE_PROCESS
# Catch SIGTERM sent by bindings_piper
signal.signal(signal.SIGTERM, shutdown_handler)
# Set the binding in worker mode
context.set_pycompss_context(context.WORKER)
persistent_storage = (config.storage_conf != 'null')
logger, logger_cfg, storage_loggers = load_loggers(config.debug,
persistent_storage)
if __debug__:
logger.debug(HEADER + "piper_worker.py wake up")
config.print_on_logger(logger)
if persistent_storage:
# Initialize storage
logger.debug(HEADER + "Starting persistent storage")
with event(INIT_STORAGE_AT_WORKER_EVENT):
from storage.api import initWorker as initStorageAtWorker # noqa
initStorageAtWorker(config_file_path=config.storage_conf)
# Create new processes
queues = []
# Setup cache
if is_cache_enabled(config.cache):
# Deploy the necessary processes
CACHE = True
cache_params = start_cache(logger, config.cache)
else:
# No cache
CACHE = False
cache_params = (None, None, None, None)
smm, CACHE_PROCESS, cache_queue, cache_ids = cache_params
# Create new executor processes
conf = ExecutorConf(TRACING, config.storage_conf, logger, logger_cfg,
storage_loggers, config.stream_backend,
config.stream_master_name, config.stream_master_port,
cache_ids, cache_queue)
for i in range(0, config.tasks_x_node):
if __debug__:
logger.debug(HEADER + "Launching process " + str(i))
process_name = "".join(("Process-", str(i)))
pid, queue = create_executor_process(process_name, conf,
config.pipes[i])
queues.append(queue)
# Read command from control pipe
alive = True
process_counter = config.tasks_x_node
control_pipe = config.control_pipe
while alive:
command = control_pipe.read_command(retry_period=1)
if command != "":
line = command.split()
if line[0] == ADD_EXECUTOR_TAG:
process_name = "".join(("Process-", str(process_counter)))
process_counter = process_counter + 1
in_pipe = line[1]
out_pipe = line[2]
pipe = Pipe(in_pipe, out_pipe)
pid, queue = create_executor_process(process_name, conf, pipe)
queues.append(queue)
control_pipe.write(" ".join(
(ADDED_EXECUTOR_TAG, out_pipe, in_pipe, str(pid))))
elif line[0] == QUERY_EXECUTOR_ID_TAG:
in_pipe = line[1]
out_pipe = line[2]
proc = PROCESSES.get(in_pipe)
pid = proc.pid
control_pipe.write(" ".join(
(REPLY_EXECUTOR_ID_TAG, out_pipe, in_pipe, str(pid))))
elif line[0] == CANCEL_TASK_TAG:
in_pipe = line[1]
proc = PROCESSES.get(in_pipe)
pid = proc.pid
if __debug__:
logger.debug(HEADER + "Signaling process with PID " +
str(pid) + " to cancel a task")
os.kill(
pid,
signal.SIGUSR2) # NOSONAR cancellation produced by COMPSs
elif line[0] == REMOVE_EXECUTOR_TAG:
in_pipe = line[1]
out_pipe = line[2]
proc = PROCESSES.pop(in_pipe, None)
if proc:
if proc.is_alive():
logger.warn(HEADER + "Forcing terminate on : " +
proc.name)
proc.terminate()
proc.join()
control_pipe.write(" ".join(
(REMOVED_EXECUTOR_TAG, out_pipe, in_pipe)))
elif line[0] == PING_TAG:
control_pipe.write(PONG_TAG)
elif line[0] == QUIT_TAG:
alive = False
# Wait for all threads
for proc in PROCESSES.values():
proc.join()
# Check if there is any exception message from the threads
for i in range(0, config.tasks_x_node):
if not queues[i].empty:
logger.error(HEADER + "Exception in threads queue: " +
str(queues[i].get()))
for queue in queues:
queue.close()
queue.join_thread()
if CACHE:
stop_cache(smm, cache_queue, CACHE_PROCESS) # noqa
if persistent_storage:
# Finish storage
if __debug__:
logger.debug(HEADER + "Stopping persistent storage")
with event(FINISH_STORAGE_AT_WORKER_EVENT):
from storage.api import finishWorker as finishStorageAtWorker # noqa
finishStorageAtWorker()
if __debug__:
logger.debug(HEADER + "Finished")
control_pipe.write(QUIT_TAG)
control_pipe.close()
def compss_persistent_worker(config):
"""
Persistent worker main function.
Retrieves the initial configuration and spawns the worker processes.
:param config: Piper Worker Configuration description
:return: None
"""
# Catch SIGTERM sent by bindings_piper
signal.signal(signal.SIGTERM, shutdown_handler)
# Set the binding in worker mode
context.set_pycompss_context(context.WORKER)
persistent_storage = (config.storage_conf != 'null')
logger, storage_loggers = load_loggers(config.debug,
persistent_storage,
config.tracing)
if __debug__:
logger.debug(HEADER + "piper_worker.py wake up")
config.print_on_logger(logger)
if persistent_storage:
# Initialize storage
logger.debug(HEADER + "Starting persistent storage")
with event(INIT_STORAGE_AT_WORKER_EVENT):
from storage.api import initWorker as initStorageAtWorker
initStorageAtWorker(config_file_path=config.storage_conf)
# Create new threads
queues = []
for i in range(0, config.tasks_x_node):
if __debug__:
logger.debug(HEADER + "Launching process " + str(i))
process_name = 'Process-' + str(i)
queue = Queue()
queues.append(queue)
conf = ExecutorConf(TRACING,
config.storage_conf,
logger,
storage_loggers,
config.stream_backend,
config.stream_master_name,
config.stream_master_port)
process = Process(target=executor, args=(queue,
process_name,
config.pipes[i],
conf))
PROCESSES[config.pipes[i].input_pipe] = process
process.start()
# Read command from control pipe
alive = True
process_counter = config.tasks_x_node
control_pipe = config.control_pipe
while alive:
command = control_pipe.read_command(retry_period=1)
if command != "":
line = command.split()
if line[0] == ADD_EXECUTOR_TAG:
process_name = 'Process-' + str(process_counter)
process_counter = process_counter + 1
in_pipe = line[1]
out_pipe = line[2]
pipe = Pipe(in_pipe, out_pipe)
pid = create_threads(process_name, pipe)
control_pipe.write(ADDED_EXECUTOR_TAG + " " +
out_pipe + " " +
in_pipe + " " +
str(pid))
elif line[0] == QUERY_EXECUTOR_ID_TAG:
in_pipe = line[1]
out_pipe = line[2]
proc = PROCESSES.get(in_pipe)
pid = proc.pid
control_pipe.write(REPLY_EXECUTOR_ID_TAG + " " +
out_pipe + " " +
in_pipe + " " +
str(pid))
elif line[0] == CANCEL_TASK_TAG:
in_pipe = line[1]
proc = PROCESSES.get(in_pipe)
pid = proc.pid
logger.debug("[PYTHON WORKER] Signaling process with PID " +
str(pid) + " to cancel a task")
kill(pid, signal.SIGUSR2)
elif line[0] == REMOVE_EXECUTOR_TAG:
in_pipe = line[1]
out_pipe = line[2]
proc = PROCESSES.pop(in_pipe, None)
if proc:
if proc.is_alive():
logger.warn(HEADER + "Forcing terminate on : " +
proc.name)
proc.terminate()
proc.join()
control_pipe.write(REMOVED_EXECUTOR_TAG + " " +
out_pipe + " " +
in_pipe)
elif line[0] == PING_TAG:
control_pipe.write(PONG_TAG)
elif line[0] == QUIT_TAG:
alive = False
# Wait for all threads
for proc in PROCESSES.values():
proc.join()
# Check if there is any exception message from the threads
for i in range(0, config.tasks_x_node):
if not queues[i].empty:
logger.error(HEADER + "Exception in threads queue: " +
str(queues[i].get()))
for queue in queues:
queue.close()
queue.join_thread()
if persistent_storage:
# Finish storage
if __debug__:
logger.debug(HEADER + "Stopping persistent storage")
with event(FINISH_STORAGE_AT_WORKER_EVENT):
from storage.api import finishWorker as finishStorageAtWorker
finishStorageAtWorker()
if __debug__:
logger.debug(HEADER + "Finished")
control_pipe.write(QUIT_TAG)
control_pipe.close()