def init_streaming(streaming_backend, streaming_master_name,
streaming_master_port, logger):
"""
Initialize the streaming client.
:param streaming_backend: Streaming backend.
:param streaming_master_name: Streaming backend master node name.
:param streaming_master_port: Streaming backend master port.
:param logger: Logger
:return: <Boolean> True if initialized successfully, False otherwise.
"""
# Fix options if necessary
if streaming_master_name is None or not streaming_master_name or streaming_master_name == "null":
streaming_master_name = "localhost"
# Check if the stream backend is enabled
streaming_enabled = streaming_backend is not None \
and streaming_backend \
and streaming_backend != "null" \
and streaming_backend != "NONE"
# Init stream backend if needed
if streaming_enabled:
if __debug__:
logger.debug("Starting DistroStream library")
DistroStreamClientHandler.init_and_start(
master_ip=streaming_master_name,
master_port=int(streaming_master_port))
# Return whether the streaming backend is enabled or not
return streaming_enabled
def poll(self, timeout=None):
logger.info("Polling new stream items...")
# Send request to server
req = PollRequest(self.id)
DistroStreamClientHandler.request(req)
# Retrieve answer
req.wait_processed()
error = req.get_error_code()
if error != 0:
raise BackendException(error, req.get_error_msg())
# Parse answer
info = req.get_response_msg()
if __debug__:
logger.debug("Retrieved stream items: " + str(info))
from pycompss.util.storages.persistent import get_by_id
retrieved_pscos = []
if info is not None and info and info != "null":
for psco_id in info.split():
psco = get_by_id(psco_id)
retrieved_pscos.append(psco)
return retrieved_pscos
def _psco_publish(self, psco):
# Persist the psco if its not
if __debug__:
logger.debug("Persisting user PSCO...")
if psco.getID() is None:
import uuid
alias = str(uuid.uuid4())
psco.makePersistent(alias)
psco_id = psco.getID()
# Register the psco on the server
if __debug__:
logger.debug("Registering PSCO publish...")
req = PublishRequest(self.id, psco_id)
DistroStreamClientHandler.request(req)
# Retrieve answer
req.wait_processed()
error = req.get_error_code()
if error != 0:
raise BackendException(error, req.get_error_msg())
# Parse answer
answer = req.get_response_msg() # noqa
if __debug__:
logger.debug("Publish stream answer: " + str(answer))
def stop_streaming():
# type: () -> None
""" Stop the streaming backend.
:return: None
"""
if __debug__:
logger.debug("Stopping DistroStream library")
DistroStreamClientHandler.set_stop()
def stop_streaming(logger):
"""
Stop the streaming backend.
:param logger: Logger
:return: None
"""
if __debug__:
logger.debug("Stopping DistroStream library")
DistroStreamClientHandler.set_stop()
def close(self):
if __debug__:
logger.debug("Closing stream " + str(self.id))
# Ask for stream closure
req = CloseStreamRequest(self.id)
DistroStreamClientHandler.request(req)
req.wait_processed()
error = req.get_error_code()
if error != 0:
logger.error("ERROR: Cannot close stream")
logger.error(" - Internal Error Code: " + str(error))
logger.error(" - Internal Error Msg: " + str(req.get_error_msg()))
def is_closed(self):
if __debug__:
logger.debug("Checking if stream " + str(self.id) + " is closed")
# Ask for stream status
req = StreamStatusRequest(self.id)
DistroStreamClientHandler.request(req)
req.wait_processed()
error = req.get_error_code()
if error != 0:
logger.error("ERROR: Cannot retrieve stream status")
logger.error(" - Internal Error Code: " + str(error))
logger.error(" - Internal Error Msg: " + str(req.get_error_msg()))
return str2bool(req.get_response_msg())
def _request_bootstrap_server_info():
logger.info("Requesting bootstrap server...")
req = BootstrapServerRequest()
DistroStreamClientHandler.request(req)
# Retrieve answer
req.wait_processed()
error = req.get_error_code()
if error != 0:
raise BackendException(error, req.get_error_msg())
# Parse answer
answer = req.get_response_msg()
if __debug__:
logger.debug("Retrieved bootstrap server information: " + answer)
return answer
def poll(self, timeout=None):
logger.info("Polling new stream items...")
# Send request to server
req = PollRequest(self.id)
DistroStreamClientHandler.request(req)
# Retrieve answer
req.wait_processed()
error = req.get_error_code()
if error != 0:
raise BackendException(error, req.get_error_msg())
# Parse answer
info = req.get_response_msg()
if __debug__:
logger.debug("Retrieved stream items: " + str(info))
if info is not None and info and info != "null":
return info.split()
else:
return []
def init_streaming(streaming_backend,
streaming_master_name,
streaming_master_port):
# type: (str, str, str) -> bool
""" Initialize the streaming client.
:param streaming_backend: Streaming backend.
:param streaming_master_name: Streaming backend master node name.
:param streaming_master_port: Streaming backend master port.
:return: True if initialized successfully, False otherwise.
"""
# Fix options if necessary
if streaming_master_name == "" or \
not streaming_master_name or \
streaming_master_name == "null":
streaming_master_name = "localhost"
if streaming_master_port == "" or \
not streaming_master_port or \
streaming_master_port == "null":
streaming_master_port = "49049"
# Check if the stream backend is enabled
if streaming_backend != "" \
and streaming_backend != "null" \
and streaming_backend != "None" \
and streaming_backend != "NONE":
streaming_enabled = True
else:
streaming_enabled = False
# Init stream backend if needed
if streaming_enabled:
if __debug__:
logger.debug("Starting DistroStream library")
DistroStreamClientHandler.init_and_start(master_ip=streaming_master_name, # noqa: E501
master_port=streaming_master_port) # noqa: E501
# Return whether the streaming backend is enabled or not
return streaming_enabled
def __init__(self,
alias=None,
stream_type=None,
access_mode=AT_MOST_ONCE,
internal_stream_info=None):
"""
Creates a new DistroStream instance.
:param alias: Stream alias.
+ type: string
:param stream_type: Internal stream type.
+ type: StreamType
:param access_mode: Stream access mode.
+ type: ConsumerMode
:param internal_stream_info: Implementation specific information.
+ type: List<T>
:raise RegistrationException: When client cannot register the stream
into the server.
"""
super(DistroStreamImpl, self).__init__()
if __debug__:
logger.debug("Registering new stream...")
self.alias = alias
self.stream_type = stream_type
self.access_mode = access_mode
# Retrieve registration id
req = RegisterStreamRequest(self.alias, self.stream_type,
self.access_mode, internal_stream_info)
DistroStreamClientHandler.request(req)
req.wait_processed()
error = req.get_error_code()
if error != 0:
raise RegistrationException(error, req.get_error_msg())
self.id = req.get_response_msg()
def main():
# Emit sync event if tracing is enabled
tracing = sys.argv[1] == 'true'
task_id = int(sys.argv[2])
log_level = sys.argv[3]
storage_conf = sys.argv[4]
stream_backend = sys.argv[5]
stream_master_name = sys.argv[6]
stream_master_port = sys.argv[7]
# method_type = sys.argv[8]
params = sys.argv[9:]
# class_name = sys.argv[9]
# method_name = sys.argv[10]
# num_slaves = sys.argv[11]
# i = 11 + num_slaves
# slaves = sys.argv[11..i]
# numCus = sys.argv[i+1]
# has_target = sys.argv[i+2] == 'true'
# num_params = int(sys.argv[i+3])
# params = sys.argv[i+4..]
print("tracing = " + str(tracing))
print("task_id = " + str(task_id))
print("log_level = " + str(log_level))
print("storage_conf = " + str(storage_conf))
persistent_storage = False
if storage_conf != 'null':
persistent_storage = True
streaming = False
if stream_backend not in [None, 'null', 'NONE']:
streaming = True
with trace_multiprocessing_worker() if tracing else dummy_context():
if streaming:
# Start streaming
DistroStreamClientHandler.init_and_start(
master_ip=stream_master_name, master_port=stream_master_port)
# Load log level configuration file
worker_path = os.path.dirname(os.path.realpath(__file__))
if log_level == 'true' or log_level == "debug":
# Debug
init_logging_worker(
worker_path + '/../../../log/logging_gat_worker_debug.json',
tracing)
elif log_level == "info" or log_level == "off":
# Info or no debug
init_logging_worker(
worker_path + '/../../../log/logging_gat_worker_off.json',
tracing)
else:
# Default
init_logging_worker(
worker_path + '/../../../log/logging_gat_worker.json', tracing)
if persistent_storage:
# Initialize storage
with event(INIT_STORAGE_AT_WORKER_EVENT):
from storage.api import initWorker as initStorageAtWorker
initStorageAtWorker(config_file_path=storage_conf)
# Init worker
exit_code = compss_worker(tracing, str(task_id), storage_conf, params)
if streaming:
# Finish streaming
DistroStreamClientHandler.set_stop()
if persistent_storage:
# Finish storage
with event(FINISH_STORAGE_AT_WORKER_EVENT):
from storage.api import finishWorker as finishStorageAtWorker
finishStorageAtWorker()
if exit_code == 1:
exit(1)
def executor(queue, process_name, pipe, conf):
"""
Thread main body - Overrides Threading run method.
Iterates over the input pipe in order to receive tasks (with their
parameters) and process them.
Notifies the runtime when each task has finished with the
corresponding output value.
Finishes when the "quit" message is received.
:param queue: Queue where to put exception messages
:param process_name: Process name (Thread-X, where X is the thread id).
:param pipe: Pipe to receive and send messages from/to the runtime.
:param conf: configuration of the executor
:return: None
"""
# Replace Python Worker's SIGTERM handler.
signal.signal(signal.SIGTERM, shutdown_handler)
tracing = conf.tracing
storage_conf = conf.storage_conf
logger = conf.logger
storage_loggers = conf.storage_loggers
# Get a copy of the necessary information from the logger to re-establish
# after each task
logger_handlers = copy.copy(logger.handlers)
logger_level = logger.getEffectiveLevel()
logger_formatter = logging.Formatter(logger_handlers[0].formatter._fmt)
storage_loggers_handlers = []
for storage_logger in storage_loggers:
storage_loggers_handlers.append(copy.copy(storage_logger.handlers))
if storage_conf != 'null':
try:
from storage.api import initWorkerPostFork
initWorkerPostFork()
except ImportError:
if __debug__:
logger.info(
HEADER +
"[%s] Could not find initWorkerPostFork storage call. Ignoring it."
% # noqa: E501
str(process_name))
# Start the streaming backend if necessary
streaming = False
if conf.stream_backend not in [None, 'null', 'NONE']:
streaming = True
if streaming:
# Initialize streaming
logger.debug(HEADER + "Starting streaming for process " +
str(process_name))
try:
DistroStreamClientHandler.init_and_start(
master_ip=conf.stream_master_ip,
master_port=int(conf.stream_master_port))
except Exception as e:
logger.error(e)
raise e
# Process properties
alive = True
if __debug__:
logger.debug(HEADER + "[%s] Starting process" % str(process_name))
# MAIN EXECUTOR LOOP
while alive:
# Runtime -> pipe - Read command from pipe
command = pipe.read_command(retry_period=0.5)
if command != "":
logger.debug(HEADER + "Received %s" % command)
# Process the command
alive = process_task(command, process_name, pipe, queue, tracing,
logger, logger_handlers, logger_level,
logger_formatter, storage_conf,
storage_loggers, storage_loggers_handlers)
# Stop storage
if storage_conf != 'null':
try:
from storage.api import finishWorkerPostFork
finishWorkerPostFork()
except ImportError:
if __debug__:
logger.info(
HEADER +
"[%s] Could not find finishWorkerPostFork storage call. Ignoring it."
% # noqa: E501
str(process_name))
# Stop streaming
if streaming:
logger.debug(HEADER + "Stopping streaming for process " +
str(process_name))
DistroStreamClientHandler.set_stop()
sys.stdout.flush()
sys.stderr.flush()
if __debug__:
logger.debug(HEADER + "[%s] Exiting process " % str(process_name))
pipe.write(QUIT_TAG)
pipe.close()
def main():
# Emit sync event if tracing is enabled
tracing = sys.argv[1] == 'true'
task_id = int(sys.argv[2])
log_level = sys.argv[3]
storage_conf = sys.argv[4]
stream_backend = sys.argv[5]
stream_master_name = sys.argv[6]
stream_master_port = sys.argv[7]
# method_type = sys.argv[8]
params = sys.argv[9:]
# class_name = sys.argv[9]
# method_name = sys.argv[10]
# num_slaves = sys.argv[11]
# i = 11 + num_slaves
# slaves = sys.argv[11..i]
# numCus = sys.argv[i+1]
# has_target = sys.argv[i+2] == 'true'
# num_params = int(sys.argv[i+3])
# params = sys.argv[i+4..]
print("tracing = " + str(tracing))
print("task_id = " + str(task_id))
print("log_level = " + str(log_level))
print("storage_conf = " + str(storage_conf))
persistent_storage = False
if storage_conf != 'null':
persistent_storage = True
from storage.api import initWorker as initStorageAtWorker
from storage.api import finishWorker as finishStorageAtWorker
streaming = False
if stream_backend not in [None, 'null', 'NONE']:
streaming = True
if tracing:
# Start tracing
import pyextrae.multiprocessing as pyextrae
pyextrae.eventandcounters(SYNC_EVENTS, task_id)
# pyextrae.eventandcounters(TASK_EVENTS, 0)
pyextrae.eventandcounters(TASK_EVENTS, WORKER_RUNNING_EVENT)
if streaming:
# Start streaming
DistroStreamClientHandler.init_and_start(
master_ip=stream_master_name, master_port=stream_master_port)
# Load log level configuration file
worker_path = os.path.dirname(os.path.realpath(__file__))
if log_level == 'true' or log_level == "debug":
# Debug
init_logging_worker(worker_path + '/../../../log/logging_debug.json')
elif log_level == "info" or log_level == "off":
# Info or no debug
init_logging_worker(worker_path + '/../../../log/logging_off.json')
else:
# Default
init_logging_worker(worker_path + '/../../../log/logging.json')
if persistent_storage:
# Initialize storage
initStorageAtWorker(config_file_path=storage_conf)
# Init worker
exit_code = compss_worker(tracing, str(task_id), storage_conf, params)
if tracing:
# Finish tracing
pyextrae.eventandcounters(TASK_EVENTS, 0)
# pyextrae.eventandcounters(TASK_EVENTS, PROCESS_DESTRUCTION)
pyextrae.eventandcounters(SYNC_EVENTS, task_id)
if streaming:
# Finish streaming
DistroStreamClientHandler.set_stop()
if persistent_storage:
# Finish storage
finishStorageAtWorker()
if exit_code == 1:
exit(1)
def main():
# type: () -> None
""" GAT worker main code.
Executes the task provided by parameters.
:return: None
"""
# Emit sync event if tracing is enabled
tracing = sys.argv[1] == 'true'
task_id = int(sys.argv[2])
log_level = sys.argv[3]
storage_conf = sys.argv[4]
stream_backend = sys.argv[5]
stream_master_name = sys.argv[6]
stream_master_port = sys.argv[7]
# Next: method_type = sys.argv[8]
params = sys.argv[9:]
# Next parameters:
# class_name = sys.argv[10]
# method_name = sys.argv[11]
# num_slaves = sys.argv[12]
# i = 13 + num_slaves
# slaves = sys.argv[12..i]
# numCus = sys.argv[i+1]
# has_target = sys.argv[i+2] == 'true'
# num_params = int(sys.argv[i+3])
# params = sys.argv[i+4..]
if log_level == "true" or log_level == "debug":
print("Tracing = " + str(tracing))
print("Task id = " + str(task_id))
print("Log level = " + str(log_level))
print("Storage conf = " + str(storage_conf))
persistent_storage = False
if storage_conf != "null":
persistent_storage = True
streaming = False
if stream_backend not in [None, "null", "NONE"]:
streaming = True
with trace_multiprocessing_worker() if tracing else dummy_context():
if streaming:
# Start streaming
DistroStreamClientHandler.init_and_start(
master_ip=stream_master_name, master_port=stream_master_port)
# Load log level configuration file
worker_path = os.path.dirname(os.path.realpath(__file__))
if log_level == "true" or log_level == "debug":
# Debug
log_json = "".join(
(worker_path, "/../../../log/logging_gat_worker_debug.json"))
elif log_level == "info" or log_level == "off":
# Info or no debug
log_json = "".join(
(worker_path, "/../../../log/logging_gat_worker_off.json"))
else:
# Default
log_json = "".join(
(worker_path, "/../../../log/logging_gat_worker.json"))
init_logging_worker(log_json, tracing)
if persistent_storage:
# Initialize storage
with event_worker(INIT_STORAGE_AT_WORKER_EVENT):
from storage.api import initWorker as initStorageAtWorker # noqa
initStorageAtWorker(config_file_path=storage_conf)
# Init worker
exit_code = compss_worker(tracing, str(task_id), storage_conf, params,
log_json)
if streaming:
# Finish streaming
DistroStreamClientHandler.set_stop()
if persistent_storage:
# Finish storage
with event_worker(FINISH_STORAGE_AT_WORKER_EVENT):
from storage.api import finishWorker as finishStorageAtWorker # noqa
finishStorageAtWorker()
if exit_code == 1:
exit(1)
def executor(queue, process_name, pipe, conf):
# type: (typing.Union[None, Queue], str, Pipe, typing.Any) -> None
"""Thread main body - Overrides Threading run method.
Iterates over the input pipe in order to receive tasks (with their
parameters) and process them.
Notifies the runtime when each task has finished with the
corresponding output value.
Finishes when the "quit" message is received.
:param queue: Queue where to put exception messages.
:param process_name: Process name (Thread-X, where X is the thread id).
:param pipe: Pipe to receive and send messages from/to the runtime.
:param conf: configuration of the executor.
:return: None
"""
try:
# Replace Python Worker's SIGTERM handler.
signal.signal(signal.SIGTERM, shutdown_handler)
if len(conf.logger.handlers) == 0:
# Logger has not been inherited correctly. Happens in MacOS.
set_temporary_directory(conf.tmp_dir, create_tmpdir=False)
# Reload logger
conf.logger, conf.logger_cfg, conf.storage_loggers, _ = \
load_loggers(conf.debug, conf.persistent_storage)
# Set the binding in worker mode too
context.set_pycompss_context(context.WORKER)
logger = conf.logger
tracing = conf.tracing
storage_conf = conf.storage_conf
storage_loggers = conf.storage_loggers
# Get a copy of the necessary information from the logger to
# re-establish after each task
logger_handlers = copy.copy(logger.handlers)
logger_level = logger.getEffectiveLevel()
logger_formatter = logging.Formatter(
logger_handlers[0].formatter._fmt) # noqa
storage_loggers_handlers = []
for storage_logger in storage_loggers:
storage_loggers_handlers.append(copy.copy(storage_logger.handlers))
# Establish link with the binding-commons to enable task nesting
if __debug__:
logger.debug(HEADER +
"Establishing link with runtime in process " +
str(process_name)) # noqa: E501
COMPSs.load_runtime(external_process=False, _logger=logger)
COMPSs.set_pipes(pipe.output_pipe, pipe.input_pipe)
if storage_conf != "null":
try:
from storage.api import initWorkerPostFork # noqa
with event_worker(INIT_WORKER_POSTFORK_EVENT):
initWorkerPostFork()
except (ImportError, AttributeError):
if __debug__:
logger.info(
HEADER +
"[%s] Could not find initWorkerPostFork storage call. Ignoring it."
% # noqa: E501
str(process_name))
# Start the streaming backend if necessary
streaming = False
if conf.stream_backend not in [None, "null", "NONE"]:
streaming = True
if streaming:
# Initialize streaming
logger.debug(HEADER + "Starting streaming for process " +
str(process_name))
try:
DistroStreamClientHandler.init_and_start(
master_ip=conf.stream_master_ip,
master_port=conf.stream_master_port)
except Exception as e:
logger.error(e)
raise e
# Connect to Shared memory manager
if conf.cache_queue:
load_shared_memory_manager()
# Process properties
alive = True
if __debug__:
logger.debug(HEADER + "[%s] Starting process" % str(process_name))
# MAIN EXECUTOR LOOP
while alive:
# Runtime -> pipe - Read command from pipe
command = COMPSs.read_pipes()
if command != "":
if __debug__:
logger.debug(HEADER + "[%s] Received command %s" %
(str(process_name), str(command)))
# Process the command
alive = process_message(
command, process_name, pipe, queue, tracing, logger,
conf.logger_cfg, logger_handlers, logger_level,
logger_formatter, storage_conf, storage_loggers,
storage_loggers_handlers, conf.cache_queue, conf.cache_ids,
conf.cache_profiler)
# Stop storage
if storage_conf != "null":
try:
from storage.api import finishWorkerPostFork # noqa
with event_worker(FINISH_WORKER_POSTFORK_EVENT):
finishWorkerPostFork()
except (ImportError, AttributeError):
if __debug__:
logger.info(
HEADER +
"[%s] Could not find finishWorkerPostFork storage call. Ignoring it."
% # noqa: E501
str(process_name))
# Stop streaming
if streaming:
logger.debug(HEADER + "Stopping streaming for process " +
str(process_name))
DistroStreamClientHandler.set_stop()
sys.stdout.flush()
sys.stderr.flush()
if __debug__:
logger.debug(HEADER + "[%s] Exiting process " % str(process_name))
pipe.write(QUIT_TAG)
pipe.close()
except Exception as e:
logger.error(e)
raise e
