Exemplo n.º 1
0
class Platform:

    SERVICE_QUERY = 'system.service.query'

    def __init__(self,
                 work_dir: str = None,
                 log_file: str = None,
                 log_level: str = None,
                 max_threads: int = None,
                 network_connector: str = None):
        if sys.version_info.major < 3:
            python_version = str(sys.version_info.major) + "." + str(
                sys.version_info.minor)
            raise RuntimeError("Requires python 3.6 and above. Actual: " +
                               python_version)

        self.util = Utility()
        self.origin = 'py' + (''.join(str(uuid.uuid4()).split('-')))
        config = AppConfig()
        my_log_file = (config.LOG_FILE if hasattr(config, 'LOG_FILE') else
                       None) if log_file is None else log_file
        my_log_level = config.LOG_LEVEL if log_level is None else log_level
        self._max_threads = config.MAX_THREADS if max_threads is None else max_threads
        self.work_dir = config.WORK_DIRECTORY if work_dir is None else work_dir
        self.log = LoggingService(
            log_dir=self.util.normalize_path(self.work_dir + "/log"),
            log_file=my_log_file,
            log_level=my_log_level).get_logger()
        self._loop = asyncio.new_event_loop()
        my_distributed_trace = DistributedTrace(
            self, config.DISTRIBUTED_TRACE_PROCESSOR)
        my_connector = config.NETWORK_CONNECTOR if network_connector is None else network_connector
        self._cloud = NetworkConnector(self, my_distributed_trace, self._loop,
                                       my_connector, self.origin)
        self._function_queues = dict()
        self._executor = concurrent.futures.ThreadPoolExecutor(
            max_workers=self._max_threads)
        self.log.info("Concurrent thread pool = " + str(self._max_threads))
        #
        # Before we figure out how to solve blocking file I/O, we will regulate event output rate.
        #
        my_test_dir = self.util.normalize_path(self.work_dir + "/test")
        if not os.path.exists(my_test_dir):
            os.makedirs(my_test_dir)
        self._throttle = Throttle(self.util.normalize_path(my_test_dir +
                                                           "/to_be_deleted"),
                                  log=self.log)
        self._seq = 0
        self.util.cleanup_dir(my_test_dir)
        self.log.debug("Estimated processing rate is " +
                       format(self._throttle.get_tps(), ',d') +
                       " events per second for this computer")
        self.running = True
        self.stopped = False
        # distributed trace sessions
        self._traces = {}

        # start event loop in a new thread to avoid blocking the main thread
        def main_event_loop():
            self.log.info("Event system started")
            self._loop.run_forever()
            self.log.info("Event system stopped")
            self._loop.close()

        threading.Thread(target=main_event_loop).start()

    def get_origin(self):
        """
        get the origin ID of this application instance
        :return: origin ID
        """
        return self.origin

    def get_trace_id(self) -> str:
        """
        get trace ID for a transaction
        :return: trace ID
        """
        trace_info = self.get_trace()
        return trace_info.get_id() if trace_info is not None else None

    def get_trace(self) -> TraceInfo:
        """
        get trace info for a transaction
        :return:
        """
        thread_id = threading.get_ident()
        return self._traces[thread_id] if thread_id in self._traces else None

    def annotate_trace(self, key: str, value: str):
        """
        Annotate a trace at a point of a transaction
        :param key: any key
        :param value: any value
        :return:
        """
        trace_info = self.get_trace()
        if trace_info is not None and isinstance(trace_info, TraceInfo):
            trace_info.annotate(key, value)

    def start_tracing(self,
                      route: str,
                      trace_id: str = None,
                      trace_path: str = None):
        """
        IMPORTANT: This method is reserved for system use. DO NOT call this from a user application.
        :param route: route name
        :param trace_id: id
        :param trace_path: path such as URI
        :return: None
        """
        thread_id = threading.get_ident()
        self._traces[thread_id] = TraceInfo(route, trace_id, trace_path)

    def stop_tracing(self):
        """
        IMPORTANT: This method is reserved for system use. DO NOT call this from a user application.
        :return: TraceInfo
        """
        thread_id = threading.get_ident()
        if thread_id in self._traces:
            trace_info = self.get_trace()
            self._traces.pop(thread_id)
            return trace_info

    def run_forever(self):
        """
        Tell the platform to run in the background until user presses CTL-C or the application is stopped by admin
        :return: None
        """
        def graceful_shutdown(signum, frame):
            self.log.warn("Control-C detected" if signal.SIGINT ==
                          signum else "KILL signal detected")
            self.running = False

        if threading.current_thread() is threading.main_thread():
            signal.signal(signal.SIGTERM, graceful_shutdown)
            signal.signal(signal.SIGINT, graceful_shutdown)
            # keep the main thread running so CTL-C can be detected
            self.log.info("To stop this application, press Control-C")
            while self.running:
                time.sleep(0.1)
            # exit forever loop and ask platform to end event loop
            self.stop()
        else:
            raise ValueError(
                'Unable to register Control-C and KILL signals because this is not the main thread'
            )

    def register(self,
                 route: str,
                 user_function: any,
                 total_instances: int,
                 is_private: bool = False) -> None:
        """
        Register a user function
        :param route: ID of the function
        :param user_function: the lambda function given by you
        :param total_instances: 1 for singleton or more for concurrency
        :param is_private: true if internal function within this application instance
        :return:
        """
        self.util.validate_service_name(route)
        if route in self._function_queues:
            raise ValueError("route " + route + " already registered")
        if not isinstance(total_instances, int):
            raise ValueError("Expect total_instances to be int, actual: " +
                             str(type(total_instances)))
        if total_instances < 1:
            raise ValueError("total_instances must be at least 1")
        if total_instances > self._max_threads:
            raise ValueError(
                "total_instances must not exceed max threads of " +
                str(self._max_threads))
        function_type = self.util.get_function_type(user_function)
        if function_type == FunctionType.NOT_SUPPORTED:
            raise ValueError(
                "Function signature should be (headers: dict, body: any, instance: int) or "
                + "(headers: dict, body: any) or (event: EventEnvelope)")

        queue = asyncio.Queue(loop=self._loop)
        if function_type == FunctionType.INTERCEPTOR:
            self._function_queues[route] = {
                'queue': queue,
                'private': is_private,
                'instances': 1
            }
            ServiceQueue(self._loop, self._executor, queue, route,
                         user_function, 0)
        elif function_type == FunctionType.REGULAR:
            self._function_queues[route] = {
                'queue': queue,
                'private': is_private,
                'instances': total_instances
            }
            ServiceQueue(self._loop, self._executor, queue, route,
                         user_function, total_instances)
        else:
            # function_type == FunctionType.SINGLETON
            self._function_queues[route] = {
                'queue': queue,
                'private': is_private,
                'instances': 1
            }
            ServiceQueue(self._loop, self._executor, queue, route,
                         user_function, -1)
        # advertise the new route to the network
        if self._cloud.is_ready() and not is_private:
            self._cloud.send_payload({'type': 'add', 'route': route})

    def cloud_ready(self):
        return self._cloud.is_ready()

    def release(self, route: str) -> None:
        # this will un-register a route
        if not isinstance(route, str):
            raise ValueError("Expect route to be str, actual: " +
                             str(type(route)))
        if route not in self._function_queues:
            raise ValueError("route " + route + " not found")
        # advertise the deleted route to the network
        if self._cloud.is_ready() and self.route_is_private(route):
            self._cloud.send_payload({'type': 'remove', 'route': route})
        self._remove_route(route)

    def has_route(self, route: str) -> bool:
        if not isinstance(route, str):
            raise ValueError("Expect route to be str, actual: " +
                             str(type(route)))
        return route in self._function_queues

    def get_routes(self, options: str = 'all'):
        result = list()
        if 'public' == options:
            for route in self._function_queues:
                if not self.route_is_private(route):
                    result.append(route)
            return result
        elif 'private' == options:
            for route in self._function_queues:
                if self.route_is_private(route):
                    result.append(route)
            return result
        elif 'all' == options:
            return list(self._function_queues.keys())
        else:
            return result

    def route_is_private(self, route: str) -> bool:
        config = self._function_queues[route]
        if config and 'private' in config:
            return config['private']
        else:
            return False

    def route_instances(self, route: str) -> int:
        config = self._function_queues[route]
        if config and 'instances' in config:
            return config['instances']
        else:
            return 0

    def parallel_request(self, events: list, timeout_seconds: float):
        timeout_value = self.util.get_float(timeout_seconds)
        if timeout_value <= 0:
            raise ValueError(
                "timeout value in seconds must be positive number")
        if not isinstance(events, list):
            raise ValueError("events must be a list of EventEnvelope")
        if len(events) == 0:
            raise ValueError("event list is empty")
        if len(events) == 1:
            result = list()
            result.append(self.request(events[0], timeout_value))
            return result
        for evt in events:
            if not isinstance(evt, EventEnvelope):
                raise ValueError("events must be a list of EventEnvelope")

        # retrieve distributed tracing info if any
        trace_info = self.get_trace()
        # emulate RPC
        inbox = Inbox(self)
        temp_route = inbox.get_route()
        inbox_queue = inbox.get_queue()
        try:
            for evt in events:
                # restore distributed tracing info from current thread
                if trace_info:
                    if trace_info.get_route(
                    ) is not None and evt.get_from() is None:
                        evt.set_from(trace_info.get_route())
                    if trace_info.get_id() is not None and trace_info.get_path(
                    ) is not None:
                        evt.set_trace(trace_info.get_id(),
                                      trace_info.get_path())

                route = evt.get_to()
                evt.set_reply_to(temp_route, me=True)
                if route in self._function_queues:
                    self._loop.call_soon_threadsafe(self._send, route,
                                                    evt.to_map())
                else:
                    if self._cloud.is_connected():
                        self._cloud.send_payload({
                            'type': 'event',
                            'event': evt.to_map()
                        })
                    else:
                        raise ValueError("route " + route + " not found")

            total_requests = len(events)
            result_list = list()
            while True:
                try:
                    # wait until all response events are delivered to the inbox
                    result_list.append(inbox_queue.get(True, timeout_value))
                    if len(result_list) == len(events):
                        return result_list
                except Empty:
                    raise TimeoutError('Requests timeout for ' +
                                       format(timeout_value, '.3f') +
                                       " seconds. Expect: " +
                                       str(total_requests) +
                                       " responses, actual: " +
                                       str(len(result_list)))
        finally:
            inbox.close()

    def request(self, event: EventEnvelope, timeout_seconds: float):
        timeout_value = self.util.get_float(timeout_seconds)
        if timeout_value <= 0:
            raise ValueError(
                "timeout value in seconds must be positive number")
        if not isinstance(event, EventEnvelope):
            raise ValueError("event object must be an EventEnvelope")
        # restore distributed tracing info from current thread
        trace_info = self.get_trace()
        if trace_info:
            if trace_info.get_route() is not None and event.get_from() is None:
                event.set_from(trace_info.get_route())
            if trace_info.get_id() is not None and trace_info.get_path(
            ) is not None:
                event.set_trace(trace_info.get_id(), trace_info.get_path())
        # emulate RPC
        inbox = Inbox(self)
        temp_route = inbox.get_route()
        inbox_queue = inbox.get_queue()
        try:
            route = event.get_to()
            event.set_reply_to(temp_route, me=True)
            if route in self._function_queues:
                self._loop.call_soon_threadsafe(self._send, route,
                                                event.to_map())
            else:
                if self._cloud.is_connected():
                    self._cloud.send_payload({
                        'type': 'event',
                        'event': event.to_map()
                    })
                else:
                    raise ValueError("route " + route + " not found")
            # wait until response event is delivered to the inbox
            return inbox_queue.get(True, timeout_value)
        except Empty:
            raise TimeoutError('Route ' + event.get_to() + ' timeout for ' +
                               format(timeout_value, '.3f') + " seconds")
        finally:
            inbox.close()

    def send_event(self, event: EventEnvelope, broadcast=False) -> None:
        if not isinstance(event, EventEnvelope):
            raise ValueError("event object must be an EventEnvelope class")
        # restore distributed tracing info from current thread
        trace_info = self.get_trace()
        if trace_info:
            if trace_info.get_route() is not None and event.get_from() is None:
                event.set_from(trace_info.get_route())
            if trace_info.get_id() is not None and trace_info.get_path(
            ) is not None:
                event.set_trace(trace_info.get_id(), trace_info.get_path())
        # regulate rate for best performance
        self._seq += 1
        self._throttle.regulate_rate(self._seq)
        route = event.get_to()
        if broadcast:
            event.set_broadcast(True)
        reply_to = event.get_reply_to()
        if reply_to:
            target = reply_to[2:] if reply_to.startswith('->') else reply_to
            if route == target:
                raise ValueError("route and reply_to must not be the same")
        if route in self._function_queues:
            if event.is_broadcast() and self._cloud.is_connected():
                self._cloud.send_payload({
                    'type': 'event',
                    'event': event.to_map()
                })
            else:
                self._loop.call_soon_threadsafe(self._send, route,
                                                event.to_map())
        else:
            if self._cloud.is_connected():
                self._cloud.send_payload({
                    'type': 'event',
                    'event': event.to_map()
                })
            else:
                raise ValueError("route " + route + " not found")

    def exists(self, routes: any):
        if isinstance(routes, str):
            single_route = routes
            if self.has_route(single_route):
                return True
            if self.cloud_ready():
                event = EventEnvelope()
                event.set_to(self.SERVICE_QUERY).set_header(
                    'type', 'find').set_header('route', single_route)
                result = self.request(event, 8.0)
                if isinstance(result, EventEnvelope):
                    if result.get_body() is not None:
                        return result.get_body()
        if isinstance(routes, list):
            if len(routes) > 0:
                remote_routes = list()
                for r in routes:
                    if not self.platform.has_route(r):
                        remote_routes.append(r)
                if len(remote_routes) == 0:
                    return True
                if self.platform.cloud_ready():
                    # tell service query to use the route list in body
                    event = EventEnvelope()
                    event.set_to(self.SERVICE_QUERY).set_header('type', 'find')
                    event.set_header('route', '*').set_body(routes)
                    result = self.request(event, 8.0)
                    if isinstance(
                            result,
                            EventEnvelope) and result.get_body() is not None:
                        return result.get_body()
        return False

    def _remove_route(self, route):
        if route in self._function_queues:
            self._send(route, None)
            self._function_queues.pop(route)

    def _send(self, route, event):
        if route in self._function_queues:
            config = self._function_queues[route]
            if 'queue' in config:
                config['queue'].put_nowait(event)

    def connect_to_cloud(self):
        self._loop.run_in_executor(self._executor,
                                   self._cloud.start_connection)

    def stop(self):
        #
        # to allow user application to invoke the "stop" method from a registered service,
        # the system must start a new thread so that the service can finish first.
        #
        if not self.stopped:
            self.log.info('Bye')
            # guarantee this stop function to execute only once
            self.stopped = True
            # exit the run_forever loop if any
            self.running = False
            # in case the calling function has just send an event asynchronously
            time.sleep(0.5)
            threading.Thread(target=self._bye).start()

    def _bye(self):
        def stopping():
            route_list = []
            for route in self.get_routes():
                route_list.append(route)
            for route in route_list:
                self._remove_route(route)
            self._loop.create_task(full_stop())

        async def full_stop():
            # give time for registered services to stop
            await asyncio.sleep(1.0)
            queue_dir = self.util.normalize_path(self.work_dir + "/queues/" +
                                                 self.get_origin())
            self.util.cleanup_dir(queue_dir)
            self._loop.stop()

        self._cloud.close_connection(1000, 'bye', stop_engine=True)
        self._loop.call_soon_threadsafe(stopping)
Exemplo n.º 2
0
class Platform:
    """
    Event system platform instance
    """
    SERVICE_QUERY = 'system.service.query'

    def __init__(self, config_file: str = None):
        if sys.version_info.major < 3:
            python_version = f'{sys.version_info.major}.{sys.version_info.minor}'
            raise RuntimeError(
                f'Requires python 3.6 and above. Actual: {python_version}')
        self.origin = 'py-' + (''.join(str(uuid.uuid4()).split('-')))
        self.config = ConfigReader(config_file)
        self.util = Utility()
        log_level = self.config.get_property('log.level')
        self._max_threads = self.config.get('max.threads')
        self.work_dir = self.config.get_property('work.directory')
        self.log = LoggingService(log_level).get_logger()
        self._loop = asyncio.new_event_loop()
        # DO NOT CHANGE 'distributed.trace.processor' which is an optional user defined trace aggregator
        my_tracer = DistributedTrace(self, 'distributed.trace.processor')
        my_nc = self.config.get_property('network.connector')
        self._cloud = NetworkConnector(self, my_tracer, self._loop, my_nc,
                                       self.origin)
        self._function_queues = dict()
        self._executor = concurrent.futures.ThreadPoolExecutor(
            max_workers=self._max_threads)
        self.log.info(f'Concurrent thread pool = {self._max_threads}')
        #
        # Before we figure out how to solve blocking file I/O, we will regulate event output rate.
        #
        my_test_dir = self.util.normalize_path(
            f'{self.work_dir}/safe_to_delete_when_apps_stop')
        if not os.path.exists(my_test_dir):
            os.makedirs(my_test_dir, exist_ok=True)
        self._throttle = Throttle(self.util.normalize_path(f'{my_test_dir}/' +
                                                           self.origin),
                                  log=self.log)
        self._seq = 0
        self.log.info(
            f'Estimated performance is {format(self._throttle.get_tps(), ",d")} events per second'
        )
        self.running = True
        self.stopped = False
        # distributed trace sessions
        self._traces = {}
        self.trace_aggregation = True

        # start event loop in a new thread to avoid blocking the main thread
        def main_event_loop():
            self.log.info('Event system started')
            self._loop.run_forever()
            self.log.info('Event system stopped')
            self._loop.close()

        threading.Thread(target=main_event_loop).start()

    def get_origin(self):
        """
        Get the origin ID of this application instance

        Returns: origin ID

        """
        return self.origin

    def get_logger(self):
        """
        Get Logger

        Returns: logger instance

        """
        return self.log

    def is_trace_supported(self):
        return self.trace_aggregation

    def set_trace_support(self, enabled: bool = True):
        self.trace_aggregation = enabled
        status = 'ON' if enabled else 'OFF'
        self.log.info(f'Trace aggregation is {status}')

    def get_trace_id(self) -> str:
        """
        Get trace ID for a transaction

        Returns: trace ID

        """
        trace_info = self.get_trace()
        return trace_info.get_id() if trace_info is not None else None

    def get_trace(self) -> TraceInfo:
        """
        Get trace info for a transaction

        Returns: trace info

        """
        thread_id = threading.get_ident()
        return self._traces[thread_id] if thread_id in self._traces else None

    def annotate_trace(self, key: str, value: str) -> None:
        """
        Annotate a trace at the current point of a transaction

        Args:
            key: any key
            value: any value

        Returns: None

        """
        trace_info = self.get_trace()
        if trace_info is not None and isinstance(trace_info, TraceInfo):
            trace_info.annotate(key, value)

    def start_tracing(self,
                      route: str,
                      trace_id: str = None,
                      trace_path: str = None) -> None:
        """
        This method is reserved for system use. DO NOT call this from a user application.

        Args:
            route: route name
            trace_id: id
            trace_path: path such as Method and URI

        Returns: None

        """
        thread_id = threading.get_ident()
        self._traces[thread_id] = TraceInfo(route, trace_id, trace_path)

    def stop_tracing(self) -> TraceInfo:
        """
        This method is reserved for system use. DO NOT call this from a user application.

        Returns: trace info

        """
        thread_id = threading.get_ident()
        if thread_id in self._traces:
            trace_info = self.get_trace()
            self._traces.pop(thread_id)
            return trace_info

    def run_forever(self) -> None:
        """
        Tell the platform to run in the background until user presses CTL-C or the application is stopped by admin

        Returns: None

        """
        def graceful_shutdown(signum, frame):
            self.running = False
            if frame is not None:
                self.log.warn('Control-C detected' if signal.SIGINT ==
                              signum else 'KILL signal detected')

        if threading.current_thread() is threading.main_thread():
            signal.signal(signal.SIGTERM, graceful_shutdown)
            signal.signal(signal.SIGINT, graceful_shutdown)
            # keep the main thread running so CTL-C can be detected
            self.log.info('To stop this application, press Control-C')
            while self.running:
                time.sleep(0.1)
            # exit forever loop and ask platform to end event loop
            self.stop()
        else:
            raise ValueError(
                'Unable to register Control-C and KILL signals because this is not the main thread'
            )

    def register(self,
                 route: str,
                 user_function: any,
                 total_instances: int = 1,
                 is_private: bool = False) -> None:
        """
        Register a user function

        Args:
            route: ID of the function
            user_function: the lambda function given by you
            total_instances: 1 for singleton or more for concurrency
            is_private: true if internal function within this application instance

        Returns: None

        """
        self.util.validate_service_name(route)
        if not isinstance(total_instances, int):
            raise ValueError(
                f'Expect total_instances to be int, actual: {type(total_instances)}'
            )
        if total_instances < 1:
            raise ValueError('total_instances must be at least 1')
        if total_instances > self._max_threads:
            raise ValueError(
                f'total_instances must not exceed max threads of {self._max_threads}'
            )
        function_type = self.util.get_function_type(user_function)
        if function_type == FunctionType.NOT_SUPPORTED:
            raise ValueError(
                'Function signature should be (headers: dict, body: any, instance: int) or '
                + '(headers: dict, body: any) or (event: EventEnvelope)')
        if route in self._function_queues:
            self.log.warn(f'{route} will be reloaded')
            self.release(route)
        queue = asyncio.Queue()
        if function_type == FunctionType.INTERCEPTOR:
            self._function_queues[route] = {
                'queue': queue,
                'private': is_private,
                'instances': 1
            }
            ServiceQueue(self._loop, self._executor, queue, route,
                         user_function, 0)
        elif function_type == FunctionType.REGULAR:
            self._function_queues[route] = {
                'queue': queue,
                'private': is_private,
                'instances': total_instances
            }
            ServiceQueue(self._loop, self._executor, queue, route,
                         user_function, total_instances)
        else:
            # function_type == FunctionType.SINGLETON
            self._function_queues[route] = {
                'queue': queue,
                'private': is_private,
                'instances': 1
            }
            ServiceQueue(self._loop, self._executor, queue, route,
                         user_function, -1)
        # advertise the new route to the network
        if self._cloud.is_ready() and not is_private:
            self._cloud.send_payload({'type': 'add', 'route': route})

    def cloud_ready(self):
        return self._cloud.is_ready()

    def subscribe_life_cycle(self, callback: str):
        self._cloud.subscribe_life_cycle(callback)

    def unsubscribe_life_cycle(self, callback: str):
        self._cloud.unsubscribe_life_cycle(callback)

    def release(self, route: str) -> None:
        # this will un-register a route
        if not isinstance(route, str):
            raise ValueError(f'Expect route to be str, actual: {type(route)}')
        if route not in self._function_queues:
            raise ValueError(f'route {route} not found')
        # advertise the deleted route to the network
        if self._cloud.is_ready() and self.route_is_private(route):
            self._cloud.send_payload({'type': 'remove', 'route': route})
        self._remove_route(route)

    def has_route(self, route: str) -> bool:
        if not isinstance(route, str):
            raise ValueError(f'Expect route to be str, actual: {type(route)}')
        return route in self._function_queues

    def get_routes(self, options: str = 'all'):
        result = list()
        if 'public' == options:
            for route in self._function_queues:
                if not self.route_is_private(route):
                    result.append(route)
            return result
        elif 'private' == options:
            for route in self._function_queues:
                if self.route_is_private(route):
                    result.append(route)
            return result
        elif 'all' == options:
            return list(self._function_queues.keys())
        else:
            return result

    def route_is_private(self, route: str) -> bool:
        config = self._function_queues[route]
        if config and 'private' in config:
            return config['private']
        else:
            return False

    def route_instances(self, route: str) -> int:
        config = self._function_queues[route]
        if config and 'instances' in config:
            return config['instances']
        else:
            return 0

    def parallel_request(self, events: list, timeout_seconds: float):
        timeout_value = self.util.get_float(timeout_seconds)
        if timeout_value <= 0:
            raise ValueError(
                'timeout value in seconds must be positive number')
        if not isinstance(events, list):
            raise ValueError('events must be a list of EventEnvelope')
        if len(events) == 0:
            raise ValueError('event list is empty')
        if len(events) == 1:
            result = list()
            result.append(self.request(events[0], timeout_value))
            return result
        for evt in events:
            if not isinstance(evt, EventEnvelope):
                raise ValueError('events must be a list of EventEnvelope')

        # retrieve distributed tracing info if any
        trace_info = self.get_trace()
        # emulate RPC
        inbox = Inbox(self)
        temp_route = inbox.get_route()
        inbox_queue = inbox.get_queue()
        try:
            for evt in events:
                # restore distributed tracing info from current thread
                if trace_info:
                    if trace_info.get_route(
                    ) is not None and evt.get_from() is None:
                        evt.set_from(trace_info.get_route())
                    if trace_info.get_id() is not None and trace_info.get_path(
                    ) is not None:
                        evt.set_trace(trace_info.get_id(),
                                      trace_info.get_path())

                route = evt.get_to()
                evt.set_reply_to(temp_route, me=True)
                if route in self._function_queues:
                    self._loop.call_soon_threadsafe(self._send, route,
                                                    evt.to_map())
                else:
                    if self._cloud.is_connected():
                        self._cloud.send_payload({
                            'type': 'event',
                            'event': evt.to_map()
                        })
                    else:
                        raise ValueError(f'route {route} not found')

            total_requests = len(events)
            result_list = list()
            while True:
                try:
                    # wait until all response events are delivered to the inbox
                    result_list.append(inbox_queue.get(True, timeout_value))
                    if len(result_list) == len(events):
                        return result_list
                except Empty:
                    raise TimeoutError(
                        f'Requests timeout for {round(timeout_value, 3)} seconds. '
                        f'Expect: {total_requests} responses, actual: {len(result_list)}'
                    )
        finally:
            inbox.close()

    def request(self, event: EventEnvelope, timeout_seconds: float):
        timeout_value = self.util.get_float(timeout_seconds)
        if timeout_value <= 0:
            raise ValueError(
                'timeout value in seconds must be positive number')
        if not isinstance(event, EventEnvelope):
            raise ValueError('event object must be an EventEnvelope')
        # restore distributed tracing info from current thread
        trace_info = self.get_trace()
        if trace_info:
            if trace_info.get_route() is not None and event.get_from() is None:
                event.set_from(trace_info.get_route())
            if trace_info.get_id() is not None and trace_info.get_path(
            ) is not None:
                event.set_trace(trace_info.get_id(), trace_info.get_path())
        # emulate RPC
        inbox = Inbox(self)
        temp_route = inbox.get_route()
        inbox_queue = inbox.get_queue()
        try:
            route = event.get_to()
            event.set_reply_to(temp_route, me=True)
            if route in self._function_queues:
                self._loop.call_soon_threadsafe(self._send, route,
                                                event.to_map())
            else:
                if self._cloud.is_connected():
                    self._cloud.send_payload({
                        'type': 'event',
                        'event': event.to_map()
                    })
                else:
                    raise ValueError(f'route {route} not found')
            # wait until response event is delivered to the inbox
            return inbox_queue.get(True, timeout_value)
        except Empty:
            raise TimeoutError(
                f'Route {event.get_to()} timeout for {round(timeout_value, 3)} seconds'
            )
        finally:
            inbox.close()

    def send_event(self, event: EventEnvelope, broadcast=False) -> None:
        if not isinstance(event, EventEnvelope):
            raise ValueError('event object must be an EventEnvelope class')
        # restore distributed tracing info from current thread
        trace_info = self.get_trace()
        if trace_info:
            if trace_info.get_route() is not None and event.get_from() is None:
                event.set_from(trace_info.get_route())
            if trace_info.get_id() is not None and trace_info.get_path(
            ) is not None:
                event.set_trace(trace_info.get_id(), trace_info.get_path())
        # regulate rate for best performance
        self._seq += 1
        self._throttle.regulate_rate(self._seq)
        route = event.get_to()
        if broadcast:
            event.set_broadcast(True)
        reply_to = event.get_reply_to()
        if reply_to:
            target = reply_to[2:] if reply_to.startswith('->') else reply_to
            if route == target:
                raise ValueError('route and reply_to must not be the same')
        if route in self._function_queues:
            if event.is_broadcast() and self._cloud.is_connected():
                self._cloud.send_payload({
                    'type': 'event',
                    'event': event.to_map()
                })
            else:
                self._loop.call_soon_threadsafe(self._send, route,
                                                event.to_map())
        else:
            if self._cloud.is_connected():
                self._cloud.send_payload({
                    'type': 'event',
                    'event': event.to_map()
                })
            else:
                raise ValueError(f'route {route} not found')

    def send_event_later(self, event: EventEnvelope,
                         delay_in_seconds: float) -> None:
        self._loop.call_later(delay_in_seconds, self.send_event, event)

    def exists(self, routes: any):
        if isinstance(routes, str):
            single_route = routes
            if self.has_route(single_route):
                return True
            if self.cloud_ready():
                event = EventEnvelope()
                event.set_to(self.SERVICE_QUERY).set_header(
                    'type', 'find').set_header('route', single_route)
                result = self.request(event, 8.0)
                if isinstance(result, EventEnvelope):
                    if result.get_body() is not None:
                        return result.get_body()
        if isinstance(routes, list):
            if len(routes) > 0:
                remote_routes = list()
                for r in routes:
                    if not self.has_route(r):
                        remote_routes.append(r)
                if len(remote_routes) == 0:
                    return True
                if self.cloud_ready():
                    # tell service query to use the route list in body
                    event = EventEnvelope()
                    event.set_to(self.SERVICE_QUERY).set_header('type', 'find')
                    event.set_header('route', '*').set_body(routes)
                    result = self.request(event, 8.0)
                    if isinstance(
                            result,
                            EventEnvelope) and result.get_body() is not None:
                        return result.get_body()
        return False

    def _remove_route(self, route):
        if route in self._function_queues:
            self._send(route, None)
            self._function_queues.pop(route)

    def _send(self, route, event):
        if route in self._function_queues:
            config = self._function_queues[route]
            if 'queue' in config:
                config['queue'].put_nowait(event)

    def connect_to_cloud(self):
        self._loop.run_in_executor(self._executor,
                                   self._cloud.start_connection)

    def stop(self):
        #
        # to allow user application to invoke the "stop" method from a registered service,
        # the system must start a new thread so that the service can finish first.
        #
        if not self.stopped:
            self.log.info('Bye')
            # guarantee this stop function to execute only once
            self.stopped = True
            # exit the run_forever loop if any
            self.running = False
            # in case the calling function has just send an event asynchronously
            time.sleep(1.0)
            threading.Thread(target=self._bye).start()

    def _bye(self):
        def stopping():
            route_list = []
            for route in self.get_routes():
                route_list.append(route)
            for route in route_list:
                self._remove_route(route)
            self._loop.create_task(full_stop())

        async def full_stop():
            # give time for registered services to stop
            await asyncio.sleep(1.0)
            queue_dir = self.util.normalize_path(
                f'{self.work_dir}/queues/{self.get_origin()}')
            self.util.cleanup_dir(queue_dir)
            self._loop.stop()

        self._cloud.close_connection(
            1000,
            f'Application {self.get_origin()} is stopping',
            stop_engine=True)
        self._loop.call_soon_threadsafe(stopping)