def main():
# type: () -> None
""" Main mpi piper worker
:return: None
"""
# Configure the global tracing variable from the argument
global TRACING
global WORKER_CONF
global CACHE_IDS
global CACHE_QUEUE
TRACING = (int(sys.argv[4]) > 0)
# Enable coverage if performed
if "COVERAGE_PROCESS_START" in os.environ:
import coverage
coverage.process_startup()
# Configure the piper worker with the arguments
WORKER_CONF = PiperWorkerConfiguration()
WORKER_CONF.update_params(sys.argv)
persistent_storage = (WORKER_CONF.storage_conf != 'null')
_, _, _, log_dir = load_loggers(WORKER_CONF.debug, persistent_storage)
cache_profiler = False
if WORKER_CONF.cache_profiler.lower() == 'true':
cache_profiler = True
if is_worker():
# Setup cache
if is_cache_enabled(str(WORKER_CONF.cache)):
# Deploy the necessary processes
cache = True
cache_params = start_cache(None, str(WORKER_CONF.cache),
cache_profiler, log_dir)
else:
# No cache
cache = False
cache_params = (None, None, None, None) # type: ignore
else:
# Otherwise it is an executor
cache = False # to stop only the cache from the main process
cache_params = (None, None, None, None) # type: ignore
smm, cache_process, CACHE_QUEUE, CACHE_IDS = cache_params
if is_worker():
with trace_mpi_worker() if TRACING else dummy_context():
compss_persistent_worker(WORKER_CONF)
else:
with trace_mpi_executor() if TRACING else dummy_context():
compss_persistent_executor(WORKER_CONF)
if cache and is_worker():
stop_cache(smm, CACHE_QUEUE, cache_profiler, cache_process) # noqa
def main():
global TRACING
global WORKER_CONF
# Configure the global tracing variable from the argument
TRACING = (int(sys.argv[4]) > 0)
with trace_multiprocessing_worker() if TRACING else dummy_context():
# Configure the piper worker with the arguments
WORKER_CONF = PiperWorkerConfiguration()
WORKER_CONF.update_params(sys.argv)
compss_persistent_worker(WORKER_CONF)
def main():
# Configure the global tracing variable from the argument
global TRACING
global WORKER_CONF
TRACING = (int(sys.argv[4]) > 0)
# Configure the piper worker with the arguments
WORKER_CONF = PiperWorkerConfiguration()
WORKER_CONF.update_params(sys.argv)
if is_worker():
with trace_mpi_worker() if TRACING else dummy_context():
compss_persistent_worker(WORKER_CONF)
else:
with trace_mpi_executor() if TRACING else dummy_context():
compss_persistent_executor(WORKER_CONF)
def main():
# Configure the global tracing variable from the argument
global TRACING
global WORKER_CONF
global CACHE_IDS
global CACHE_QUEUE
TRACING = (int(sys.argv[4]) > 0)
# Enable coverage if performed
if "COVERAGE_PROCESS_START" in os.environ:
import coverage
coverage.process_startup()
# Configure the piper worker with the arguments
WORKER_CONF = PiperWorkerConfiguration()
WORKER_CONF.update_params(sys.argv)
if is_worker():
# Setup cache
if is_cache_enabled(WORKER_CONF.cache):
# Deploy the necessary processes
cache = True
cache_params = start_cache(None, WORKER_CONF.cache)
else:
# No cache
cache = False
cache_params = (None, None, None, None)
else:
# Otherwise it is an executor
cache = False # to stop only the cache from the main process
cache_params = (None, None, None, None)
smm, cache_process, CACHE_QUEUE, CACHE_IDS = cache_params
if is_worker():
with trace_mpi_worker() if TRACING else dummy_context():
compss_persistent_worker(WORKER_CONF)
else:
with trace_mpi_executor() if TRACING else dummy_context():
compss_persistent_executor(WORKER_CONF)
if cache and is_worker():
stop_cache(smm, CACHE_QUEUE, cache_process) # noqa
config.stream_master_port)
executor(None, process_name, config.pipes[RANK - 1], conf)
if persistent_storage:
# Finish storage
if __debug__:
logger.debug(HEADER + "Stopping persistent storage")
with event(FINISH_STORAGE_AT_WORKER_EVENT):
from storage.api import finishWorker as finishStorageAtWorker
finishStorageAtWorker()
############################
# Main -> Calls main method
############################
if __name__ == '__main__':
# Configure the global tracing variable from the argument
TRACING = (int(sys.argv[2]) > 0)
# Configure the piper worker with the arguments
WORKER_CONF = PiperWorkerConfiguration()
WORKER_CONF.update_params(sys.argv)
if is_worker():
with trace_mpi_worker() if TRACING else dummy_context():
compss_persistent_worker(WORKER_CONF)
else:
with trace_mpi_executor() if TRACING else dummy_context():
compss_persistent_executor(WORKER_CONF)