Exemple #1
0
def executor(queue, process_name, pipe, conf):
    # type: (..., str, Pipe, ...) -> None
    """Thread main body - Overrides Threading run method.

    Iterates over the input pipe in order to receive tasks (with their
    parameters) and process them.
    Notifies the runtime when each task  has finished with the
    corresponding output value.
    Finishes when the "quit" message is received.

    :param queue: Queue where to put exception messages.
    :param process_name: Process name (Thread-X, where X is the thread id).
    :param pipe: Pipe to receive and send messages from/to the runtime.
    :param conf: configuration of the executor.
    :return: None
    """
    logger = conf.logger
    try:
        # Replace Python Worker's SIGTERM handler.
        signal.signal(signal.SIGTERM, shutdown_handler)

        tracing = conf.tracing
        storage_conf = conf.storage_conf
        storage_loggers = conf.storage_loggers

        # Get a copy of the necessary information from the logger to
        # re-establish after each task
        logger_handlers = copy.copy(logger.handlers)
        logger_level = logger.getEffectiveLevel()
        logger_formatter = logging.Formatter(
            logger_handlers[0].formatter._fmt)  # noqa
        storage_loggers_handlers = []
        for storage_logger in storage_loggers:
            storage_loggers_handlers.append(copy.copy(storage_logger.handlers))

        # Establish link with the binding-commons to enable task nesting
        if __debug__:
            logger.debug(HEADER +
                         "Establishing link with runtime in process " +
                         str(process_name))  # noqa: E501
        COMPSs.load_runtime(external_process=False, _logger=logger)
        COMPSs.set_pipes(pipe.output_pipe, pipe.input_pipe)

        if storage_conf != 'null':
            try:
                from storage.api import initWorkerPostFork  # noqa
                with event(INIT_WORKER_POSTFORK_EVENT):
                    initWorkerPostFork()
            except ImportError:
                if __debug__:
                    logger.info(
                        HEADER +
                        "[%s] Could not find initWorkerPostFork storage call. Ignoring it."
                        %  # noqa: E501
                        str(process_name))

        # Start the streaming backend if necessary
        streaming = False
        if conf.stream_backend not in [None, 'null', 'NONE']:
            streaming = True

        if streaming:
            # Initialize streaming
            logger.debug(HEADER + "Starting streaming for process " +
                         str(process_name))
            try:
                DistroStreamClientHandler.init_and_start(
                    master_ip=conf.stream_master_ip,
                    master_port=int(conf.stream_master_port))
            except Exception as e:
                logger.error(e)
                raise e

        # Process properties
        alive = True

        if __debug__:
            logger.debug(HEADER + "[%s] Starting process" % str(process_name))

        # MAIN EXECUTOR LOOP
        while alive:
            # Runtime -> pipe - Read command from pipe
            command = pipe.read_command(retry_period=0.5)
            if command != "":
                if __debug__:
                    logger.debug(HEADER + "Received %s" % command)
                # Process the command
                alive = process_message(command, process_name, pipe, queue,
                                        tracing, logger, logger_handlers,
                                        logger_level, logger_formatter,
                                        storage_conf, storage_loggers,
                                        storage_loggers_handlers)

        # Stop storage
        if storage_conf != 'null':
            try:
                from storage.api import finishWorkerPostFork  # noqa
                with event(FINISH_WORKER_POSTFORK_EVENT):
                    finishWorkerPostFork()
            except ImportError:
                if __debug__:
                    logger.info(
                        HEADER +
                        "[%s] Could not find finishWorkerPostFork storage call. Ignoring it."
                        %  # noqa: E501
                        str(process_name))

        # Stop streaming
        if streaming:
            logger.debug(HEADER + "Stopping streaming for process " +
                         str(process_name))
            DistroStreamClientHandler.set_stop()

        sys.stdout.flush()
        sys.stderr.flush()
        if __debug__:
            logger.debug(HEADER + "[%s] Exiting process " % str(process_name))
        pipe.write(QUIT_TAG)
        pipe.close()
    except BaseException as e:
        logger.error(e)
        raise e
Exemple #2
0
def executor(queue, process_name, pipe, conf):
    # type: (typing.Union[None, Queue], str, Pipe, typing.Any) -> None
    """Thread main body - Overrides Threading run method.

    Iterates over the input pipe in order to receive tasks (with their
    parameters) and process them.
    Notifies the runtime when each task  has finished with the
    corresponding output value.
    Finishes when the "quit" message is received.

    :param queue: Queue where to put exception messages.
    :param process_name: Process name (Thread-X, where X is the thread id).
    :param pipe: Pipe to receive and send messages from/to the runtime.
    :param conf: configuration of the executor.
    :return: None
    """
    try:
        # Replace Python Worker's SIGTERM handler.
        signal.signal(signal.SIGTERM, shutdown_handler)

        if len(conf.logger.handlers) == 0:
            # Logger has not been inherited correctly. Happens in MacOS.
            set_temporary_directory(conf.tmp_dir, create_tmpdir=False)
            # Reload logger
            conf.logger, conf.logger_cfg, conf.storage_loggers, _ = \
                load_loggers(conf.debug, conf.persistent_storage)
            # Set the binding in worker mode too
            context.set_pycompss_context(context.WORKER)
        logger = conf.logger

        tracing = conf.tracing
        storage_conf = conf.storage_conf
        storage_loggers = conf.storage_loggers

        # Get a copy of the necessary information from the logger to
        # re-establish after each task
        logger_handlers = copy.copy(logger.handlers)
        logger_level = logger.getEffectiveLevel()
        logger_formatter = logging.Formatter(
            logger_handlers[0].formatter._fmt)  # noqa
        storage_loggers_handlers = []
        for storage_logger in storage_loggers:
            storage_loggers_handlers.append(copy.copy(storage_logger.handlers))

        # Establish link with the binding-commons to enable task nesting
        if __debug__:
            logger.debug(HEADER +
                         "Establishing link with runtime in process " +
                         str(process_name))  # noqa: E501
        COMPSs.load_runtime(external_process=False, _logger=logger)
        COMPSs.set_pipes(pipe.output_pipe, pipe.input_pipe)

        if storage_conf != "null":
            try:
                from storage.api import initWorkerPostFork  # noqa
                with event_worker(INIT_WORKER_POSTFORK_EVENT):
                    initWorkerPostFork()
            except (ImportError, AttributeError):
                if __debug__:
                    logger.info(
                        HEADER +
                        "[%s] Could not find initWorkerPostFork storage call. Ignoring it."
                        %  # noqa: E501
                        str(process_name))

        # Start the streaming backend if necessary
        streaming = False
        if conf.stream_backend not in [None, "null", "NONE"]:
            streaming = True

        if streaming:
            # Initialize streaming
            logger.debug(HEADER + "Starting streaming for process " +
                         str(process_name))
            try:
                DistroStreamClientHandler.init_and_start(
                    master_ip=conf.stream_master_ip,
                    master_port=conf.stream_master_port)
            except Exception as e:
                logger.error(e)
                raise e

        # Connect to Shared memory manager
        if conf.cache_queue:
            load_shared_memory_manager()

        # Process properties
        alive = True

        if __debug__:
            logger.debug(HEADER + "[%s] Starting process" % str(process_name))

        # MAIN EXECUTOR LOOP
        while alive:
            # Runtime -> pipe - Read command from pipe
            command = COMPSs.read_pipes()
            if command != "":
                if __debug__:
                    logger.debug(HEADER + "[%s] Received command %s" %
                                 (str(process_name), str(command)))
                # Process the command
                alive = process_message(
                    command, process_name, pipe, queue, tracing, logger,
                    conf.logger_cfg, logger_handlers, logger_level,
                    logger_formatter, storage_conf, storage_loggers,
                    storage_loggers_handlers, conf.cache_queue, conf.cache_ids,
                    conf.cache_profiler)
        # Stop storage
        if storage_conf != "null":
            try:
                from storage.api import finishWorkerPostFork  # noqa
                with event_worker(FINISH_WORKER_POSTFORK_EVENT):
                    finishWorkerPostFork()
            except (ImportError, AttributeError):
                if __debug__:
                    logger.info(
                        HEADER +
                        "[%s] Could not find finishWorkerPostFork storage call. Ignoring it."
                        %  # noqa: E501
                        str(process_name))

        # Stop streaming
        if streaming:
            logger.debug(HEADER + "Stopping streaming for process " +
                         str(process_name))
            DistroStreamClientHandler.set_stop()

        sys.stdout.flush()
        sys.stderr.flush()
        if __debug__:
            logger.debug(HEADER + "[%s] Exiting process " % str(process_name))
        pipe.write(QUIT_TAG)
        pipe.close()
    except Exception as e:
        logger.error(e)
        raise e