class Platform: SERVICE_QUERY = 'system.service.query' def __init__(self, work_dir: str = None, log_file: str = None, log_level: str = None, max_threads: int = None, network_connector: str = None): if sys.version_info.major < 3: python_version = str(sys.version_info.major) + "." + str( sys.version_info.minor) raise RuntimeError("Requires python 3.6 and above. Actual: " + python_version) self.util = Utility() self.origin = 'py' + (''.join(str(uuid.uuid4()).split('-'))) config = AppConfig() my_log_file = (config.LOG_FILE if hasattr(config, 'LOG_FILE') else None) if log_file is None else log_file my_log_level = config.LOG_LEVEL if log_level is None else log_level self._max_threads = config.MAX_THREADS if max_threads is None else max_threads self.work_dir = config.WORK_DIRECTORY if work_dir is None else work_dir self.log = LoggingService( log_dir=self.util.normalize_path(self.work_dir + "/log"), log_file=my_log_file, log_level=my_log_level).get_logger() self._loop = asyncio.new_event_loop() my_distributed_trace = DistributedTrace( self, config.DISTRIBUTED_TRACE_PROCESSOR) my_connector = config.NETWORK_CONNECTOR if network_connector is None else network_connector self._cloud = NetworkConnector(self, my_distributed_trace, self._loop, my_connector, self.origin) self._function_queues = dict() self._executor = concurrent.futures.ThreadPoolExecutor( max_workers=self._max_threads) self.log.info("Concurrent thread pool = " + str(self._max_threads)) # # Before we figure out how to solve blocking file I/O, we will regulate event output rate. # my_test_dir = self.util.normalize_path(self.work_dir + "/test") if not os.path.exists(my_test_dir): os.makedirs(my_test_dir) self._throttle = Throttle(self.util.normalize_path(my_test_dir + "/to_be_deleted"), log=self.log) self._seq = 0 self.util.cleanup_dir(my_test_dir) self.log.debug("Estimated processing rate is " + format(self._throttle.get_tps(), ',d') + " events per second for this computer") self.running = True self.stopped = False # distributed trace sessions self._traces = {} # start event loop in a new thread to avoid blocking the main thread def main_event_loop(): self.log.info("Event system started") self._loop.run_forever() self.log.info("Event system stopped") self._loop.close() threading.Thread(target=main_event_loop).start() def get_origin(self): """ get the origin ID of this application instance :return: origin ID """ return self.origin def get_trace_id(self) -> str: """ get trace ID for a transaction :return: trace ID """ trace_info = self.get_trace() return trace_info.get_id() if trace_info is not None else None def get_trace(self) -> TraceInfo: """ get trace info for a transaction :return: """ thread_id = threading.get_ident() return self._traces[thread_id] if thread_id in self._traces else None def annotate_trace(self, key: str, value: str): """ Annotate a trace at a point of a transaction :param key: any key :param value: any value :return: """ trace_info = self.get_trace() if trace_info is not None and isinstance(trace_info, TraceInfo): trace_info.annotate(key, value) def start_tracing(self, route: str, trace_id: str = None, trace_path: str = None): """ IMPORTANT: This method is reserved for system use. DO NOT call this from a user application. :param route: route name :param trace_id: id :param trace_path: path such as URI :return: None """ thread_id = threading.get_ident() self._traces[thread_id] = TraceInfo(route, trace_id, trace_path) def stop_tracing(self): """ IMPORTANT: This method is reserved for system use. DO NOT call this from a user application. :return: TraceInfo """ thread_id = threading.get_ident() if thread_id in self._traces: trace_info = self.get_trace() self._traces.pop(thread_id) return trace_info def run_forever(self): """ Tell the platform to run in the background until user presses CTL-C or the application is stopped by admin :return: None """ def graceful_shutdown(signum, frame): self.log.warn("Control-C detected" if signal.SIGINT == signum else "KILL signal detected") self.running = False if threading.current_thread() is threading.main_thread(): signal.signal(signal.SIGTERM, graceful_shutdown) signal.signal(signal.SIGINT, graceful_shutdown) # keep the main thread running so CTL-C can be detected self.log.info("To stop this application, press Control-C") while self.running: time.sleep(0.1) # exit forever loop and ask platform to end event loop self.stop() else: raise ValueError( 'Unable to register Control-C and KILL signals because this is not the main thread' ) def register(self, route: str, user_function: any, total_instances: int, is_private: bool = False) -> None: """ Register a user function :param route: ID of the function :param user_function: the lambda function given by you :param total_instances: 1 for singleton or more for concurrency :param is_private: true if internal function within this application instance :return: """ self.util.validate_service_name(route) if route in self._function_queues: raise ValueError("route " + route + " already registered") if not isinstance(total_instances, int): raise ValueError("Expect total_instances to be int, actual: " + str(type(total_instances))) if total_instances < 1: raise ValueError("total_instances must be at least 1") if total_instances > self._max_threads: raise ValueError( "total_instances must not exceed max threads of " + str(self._max_threads)) function_type = self.util.get_function_type(user_function) if function_type == FunctionType.NOT_SUPPORTED: raise ValueError( "Function signature should be (headers: dict, body: any, instance: int) or " + "(headers: dict, body: any) or (event: EventEnvelope)") queue = asyncio.Queue(loop=self._loop) if function_type == FunctionType.INTERCEPTOR: self._function_queues[route] = { 'queue': queue, 'private': is_private, 'instances': 1 } ServiceQueue(self._loop, self._executor, queue, route, user_function, 0) elif function_type == FunctionType.REGULAR: self._function_queues[route] = { 'queue': queue, 'private': is_private, 'instances': total_instances } ServiceQueue(self._loop, self._executor, queue, route, user_function, total_instances) else: # function_type == FunctionType.SINGLETON self._function_queues[route] = { 'queue': queue, 'private': is_private, 'instances': 1 } ServiceQueue(self._loop, self._executor, queue, route, user_function, -1) # advertise the new route to the network if self._cloud.is_ready() and not is_private: self._cloud.send_payload({'type': 'add', 'route': route}) def cloud_ready(self): return self._cloud.is_ready() def release(self, route: str) -> None: # this will un-register a route if not isinstance(route, str): raise ValueError("Expect route to be str, actual: " + str(type(route))) if route not in self._function_queues: raise ValueError("route " + route + " not found") # advertise the deleted route to the network if self._cloud.is_ready() and self.route_is_private(route): self._cloud.send_payload({'type': 'remove', 'route': route}) self._remove_route(route) def has_route(self, route: str) -> bool: if not isinstance(route, str): raise ValueError("Expect route to be str, actual: " + str(type(route))) return route in self._function_queues def get_routes(self, options: str = 'all'): result = list() if 'public' == options: for route in self._function_queues: if not self.route_is_private(route): result.append(route) return result elif 'private' == options: for route in self._function_queues: if self.route_is_private(route): result.append(route) return result elif 'all' == options: return list(self._function_queues.keys()) else: return result def route_is_private(self, route: str) -> bool: config = self._function_queues[route] if config and 'private' in config: return config['private'] else: return False def route_instances(self, route: str) -> int: config = self._function_queues[route] if config and 'instances' in config: return config['instances'] else: return 0 def parallel_request(self, events: list, timeout_seconds: float): timeout_value = self.util.get_float(timeout_seconds) if timeout_value <= 0: raise ValueError( "timeout value in seconds must be positive number") if not isinstance(events, list): raise ValueError("events must be a list of EventEnvelope") if len(events) == 0: raise ValueError("event list is empty") if len(events) == 1: result = list() result.append(self.request(events[0], timeout_value)) return result for evt in events: if not isinstance(evt, EventEnvelope): raise ValueError("events must be a list of EventEnvelope") # retrieve distributed tracing info if any trace_info = self.get_trace() # emulate RPC inbox = Inbox(self) temp_route = inbox.get_route() inbox_queue = inbox.get_queue() try: for evt in events: # restore distributed tracing info from current thread if trace_info: if trace_info.get_route( ) is not None and evt.get_from() is None: evt.set_from(trace_info.get_route()) if trace_info.get_id() is not None and trace_info.get_path( ) is not None: evt.set_trace(trace_info.get_id(), trace_info.get_path()) route = evt.get_to() evt.set_reply_to(temp_route, me=True) if route in self._function_queues: self._loop.call_soon_threadsafe(self._send, route, evt.to_map()) else: if self._cloud.is_connected(): self._cloud.send_payload({ 'type': 'event', 'event': evt.to_map() }) else: raise ValueError("route " + route + " not found") total_requests = len(events) result_list = list() while True: try: # wait until all response events are delivered to the inbox result_list.append(inbox_queue.get(True, timeout_value)) if len(result_list) == len(events): return result_list except Empty: raise TimeoutError('Requests timeout for ' + format(timeout_value, '.3f') + " seconds. Expect: " + str(total_requests) + " responses, actual: " + str(len(result_list))) finally: inbox.close() def request(self, event: EventEnvelope, timeout_seconds: float): timeout_value = self.util.get_float(timeout_seconds) if timeout_value <= 0: raise ValueError( "timeout value in seconds must be positive number") if not isinstance(event, EventEnvelope): raise ValueError("event object must be an EventEnvelope") # restore distributed tracing info from current thread trace_info = self.get_trace() if trace_info: if trace_info.get_route() is not None and event.get_from() is None: event.set_from(trace_info.get_route()) if trace_info.get_id() is not None and trace_info.get_path( ) is not None: event.set_trace(trace_info.get_id(), trace_info.get_path()) # emulate RPC inbox = Inbox(self) temp_route = inbox.get_route() inbox_queue = inbox.get_queue() try: route = event.get_to() event.set_reply_to(temp_route, me=True) if route in self._function_queues: self._loop.call_soon_threadsafe(self._send, route, event.to_map()) else: if self._cloud.is_connected(): self._cloud.send_payload({ 'type': 'event', 'event': event.to_map() }) else: raise ValueError("route " + route + " not found") # wait until response event is delivered to the inbox return inbox_queue.get(True, timeout_value) except Empty: raise TimeoutError('Route ' + event.get_to() + ' timeout for ' + format(timeout_value, '.3f') + " seconds") finally: inbox.close() def send_event(self, event: EventEnvelope, broadcast=False) -> None: if not isinstance(event, EventEnvelope): raise ValueError("event object must be an EventEnvelope class") # restore distributed tracing info from current thread trace_info = self.get_trace() if trace_info: if trace_info.get_route() is not None and event.get_from() is None: event.set_from(trace_info.get_route()) if trace_info.get_id() is not None and trace_info.get_path( ) is not None: event.set_trace(trace_info.get_id(), trace_info.get_path()) # regulate rate for best performance self._seq += 1 self._throttle.regulate_rate(self._seq) route = event.get_to() if broadcast: event.set_broadcast(True) reply_to = event.get_reply_to() if reply_to: target = reply_to[2:] if reply_to.startswith('->') else reply_to if route == target: raise ValueError("route and reply_to must not be the same") if route in self._function_queues: if event.is_broadcast() and self._cloud.is_connected(): self._cloud.send_payload({ 'type': 'event', 'event': event.to_map() }) else: self._loop.call_soon_threadsafe(self._send, route, event.to_map()) else: if self._cloud.is_connected(): self._cloud.send_payload({ 'type': 'event', 'event': event.to_map() }) else: raise ValueError("route " + route + " not found") def exists(self, routes: any): if isinstance(routes, str): single_route = routes if self.has_route(single_route): return True if self.cloud_ready(): event = EventEnvelope() event.set_to(self.SERVICE_QUERY).set_header( 'type', 'find').set_header('route', single_route) result = self.request(event, 8.0) if isinstance(result, EventEnvelope): if result.get_body() is not None: return result.get_body() if isinstance(routes, list): if len(routes) > 0: remote_routes = list() for r in routes: if not self.platform.has_route(r): remote_routes.append(r) if len(remote_routes) == 0: return True if self.platform.cloud_ready(): # tell service query to use the route list in body event = EventEnvelope() event.set_to(self.SERVICE_QUERY).set_header('type', 'find') event.set_header('route', '*').set_body(routes) result = self.request(event, 8.0) if isinstance( result, EventEnvelope) and result.get_body() is not None: return result.get_body() return False def _remove_route(self, route): if route in self._function_queues: self._send(route, None) self._function_queues.pop(route) def _send(self, route, event): if route in self._function_queues: config = self._function_queues[route] if 'queue' in config: config['queue'].put_nowait(event) def connect_to_cloud(self): self._loop.run_in_executor(self._executor, self._cloud.start_connection) def stop(self): # # to allow user application to invoke the "stop" method from a registered service, # the system must start a new thread so that the service can finish first. # if not self.stopped: self.log.info('Bye') # guarantee this stop function to execute only once self.stopped = True # exit the run_forever loop if any self.running = False # in case the calling function has just send an event asynchronously time.sleep(0.5) threading.Thread(target=self._bye).start() def _bye(self): def stopping(): route_list = [] for route in self.get_routes(): route_list.append(route) for route in route_list: self._remove_route(route) self._loop.create_task(full_stop()) async def full_stop(): # give time for registered services to stop await asyncio.sleep(1.0) queue_dir = self.util.normalize_path(self.work_dir + "/queues/" + self.get_origin()) self.util.cleanup_dir(queue_dir) self._loop.stop() self._cloud.close_connection(1000, 'bye', stop_engine=True) self._loop.call_soon_threadsafe(stopping)
class PostOffice: """ Convenient class for making RPC, async and callback. """ def __init__(self): self.platform = Platform() self.util = Utility() def get_route(self): """ Obtain my route name for the currently running service. This is useful for Role Based Access Control (RBAC) to restrict certain user roles for a service. Note that RBAC is the responsibility of the user application. Returns: route name """ trace_info = self.get_trace() return "?" if trace_info is None else trace_info.get_route() def get_trace_id(self): return self.platform.get_trace_id() def get_trace(self): return self.platform.get_trace() def annotate_trace(self, key: str, value: str): self.platform.annotate_trace(key, value) def broadcast(self, route: str, headers: dict = None, body: any = None) -> None: self.util.validate_service_name(route) if headers is None and body is None: raise ValueError( 'Unable to broadcast because both headers and body are missing' ) event = EventEnvelope().set_to(route) if headers is not None: if not isinstance(headers, dict): raise ValueError('headers must be dict') for h in headers: event.set_header(h, str(headers[h])) if body is not None: event.set_body(body) self.platform.send_event(event, True) def send_later(self, route: str, headers: dict = None, body: any = None, reply_to: str = None, me=True, seconds: float = 1.0) -> None: self.util.validate_service_name(route, True) if isinstance(seconds, float) or isinstance(seconds, int): if seconds > 0: if headers is None and body is None: raise ValueError( 'Unable to send because both headers and body are missing' ) event = EventEnvelope().set_to(route) if headers is not None: if not isinstance(headers, dict): raise ValueError('headers must be dict') for h in headers: event.set_header(str(h), str(headers[h])) if body is not None: event.set_body(body) if reply_to is not None: if not isinstance(reply_to, str): raise ValueError('reply_to must be str') # encode 'me' in the "call back" if replying to this instance event.set_reply_to(reply_to, me) self.platform.send_event_later(event, seconds) else: raise ValueError('delay in seconds must be larger than zero') else: raise ValueError('delay in seconds must be int or float') def send_event_later(self, event: EventEnvelope, seconds: float = 1.0, me=True): if event.get_reply_to() is not None: event.set_reply_to(event.get_reply_to(), me) self.platform.send_event_later(event, seconds) def send(self, route: str, headers: dict = None, body: any = None, reply_to: str = None, me=True) -> None: self.util.validate_service_name(route, True) if headers is None and body is None: raise ValueError( 'Unable to send because both headers and body are missing') event = EventEnvelope().set_to(route) if headers is not None: if not isinstance(headers, dict): raise ValueError('headers must be dict') for h in headers: event.set_header(str(h), str(headers[h])) if body is not None: event.set_body(body) if reply_to is not None: if not isinstance(reply_to, str): raise ValueError('reply_to must be str') # encode 'me' in the "call back" if replying to this instance event.set_reply_to(reply_to, me) self.platform.send_event(event) def send_event(self, event: EventEnvelope, me=True): if event.get_reply_to() is not None: event.set_reply_to(event.get_reply_to(), me) self.platform.send_event(event) def request(self, route: str, timeout_seconds: float, headers: dict = None, body: any = None, correlation_id: str = None) -> EventEnvelope: self.util.validate_service_name(route, True) if headers is None and body is None: raise ValueError( 'Unable to make RPC call because both headers and body are missing' ) timeout_value = self.util.get_float(timeout_seconds) if timeout_value <= 0: raise ValueError( 'timeout value in seconds must be positive number') event = EventEnvelope().set_to(route) if headers is not None: if not isinstance(headers, dict): raise ValueError('headers must be dict') for h in headers: event.set_header(h, str(headers[h])) if body is not None: event.set_body(body) if correlation_id is not None: event.set_correlation_id(str(correlation_id)) response = self.platform.request(event, timeout_seconds) if isinstance(response, EventEnvelope): if response.get_tag('exception') is None: return response else: raise AppException(response.get_status(), response.get_body()) raise ValueError( f'Expect response is EventEnvelope, actual: ({response})') def single_request(self, event: EventEnvelope, timeout_seconds: float): response = self.platform.request(event, timeout_seconds) if isinstance(response, EventEnvelope): if response.get_tag('exception') is None: return response else: raise AppException(response.get_status(), response.get_body()) raise ValueError( f'Expect response is EventEnvelope, actual: ({response})') def parallel_request(self, events: list, timeout_seconds: float) -> list: return self.platform.parallel_request(events, timeout_seconds) def exists(self, routes: any): return self.platform.exists(routes)
class PostOffice: """ Convenient class for making RPC, async and callback. """ DEFERRED_DELIVERY = 'system.deferred.delivery' def __init__(self): self.platform = Platform() self.util = Utility() def get_route(self): """ Obtain my route name for the currently running service. This is useful for Role Based Access Control (RBAC) to restrict certain user roles for a service. Note that RBAC is the responsibility of the user application. :return: route name """ trace_info = self.get_trace() return "?" if trace_info is None else trace_info.get_route() def get_trace_id(self): return self.platform.get_trace_id() def get_trace(self): return self.platform.get_trace() def annotate_trace(self, key: str, value: str): self.platform.annotate_trace(key, value) def broadcast(self, route: str, headers: dict = None, body: any = None) -> None: self.util.validate_service_name(route) if headers is None and body is None: raise ValueError( 'Unable to broadcast because both headers and body are missing' ) event = EventEnvelope().set_to(route) if headers is not None: if not isinstance(headers, dict): raise ValueError('headers must be dict') for h in headers: event.set_header(h, str(headers[h])) if body is not None: event.set_body(body) self.platform.send_event(event, True) def send_later(self, route: str, headers: dict = None, body: any = None, seconds: float = 1.0) -> None: self.util.validate_service_name(route, True) if isinstance(seconds, float) or isinstance(seconds, int): relay = dict() relay['route'] = route if headers is not None: relay_headers = dict() for h in headers: relay_headers[str(h)] = str(headers[h]) relay['headers'] = relay_headers if body is not None: relay['body'] = body relay['seconds'] = seconds self.send(self.DEFERRED_DELIVERY, body=relay) else: raise ValueError('delay in seconds must be int or float') def send(self, route: str, headers: dict = None, body: any = None, reply_to: str = None, me=True) -> None: self.util.validate_service_name(route, True) if headers is None and body is None: raise ValueError( 'Unable to send because both headers and body are missing') event = EventEnvelope().set_to(route) if headers is not None: if not isinstance(headers, dict): raise ValueError('headers must be dict') for h in headers: event.set_header(str(h), str(headers[h])) if body is not None: event.set_body(body) if reply_to is not None: if not isinstance(reply_to, str): raise ValueError('reply_to must be str') # encode 'me' in the "call back" if replying to this instance event.set_reply_to(reply_to, me) self.platform.send_event(event) def request(self, route: str, timeout_seconds: float, headers: dict = None, body: any = None, correlation_id: str = None) -> EventEnvelope: self.util.validate_service_name(route, True) if headers is None and body is None: raise ValueError( 'Unable to make RPC call because both headers and body are missing' ) timeout_value = self.util.get_float(timeout_seconds) if timeout_value <= 0: raise ValueError( "timeout value in seconds must be positive number") event = EventEnvelope().set_to(route) if headers is not None: if not isinstance(headers, dict): raise ValueError('headers must be dict') for h in headers: event.set_header(h, str(headers[h])) if body is not None: event.set_body(body) if correlation_id is not None: event.set_correlation_id(str(correlation_id)) return self.platform.request(event, timeout_seconds) def parallel_request(self, events: list, timeout_seconds: float) -> list: return self.platform.parallel_request(events, timeout_seconds) def exists(self, routes: any): return self.platform.exists(routes)
class Platform: """ Event system platform instance """ SERVICE_QUERY = 'system.service.query' def __init__(self, config_file: str = None): if sys.version_info.major < 3: python_version = f'{sys.version_info.major}.{sys.version_info.minor}' raise RuntimeError( f'Requires python 3.6 and above. Actual: {python_version}') self.origin = 'py-' + (''.join(str(uuid.uuid4()).split('-'))) self.config = ConfigReader(config_file) self.util = Utility() log_level = self.config.get_property('log.level') self._max_threads = self.config.get('max.threads') self.work_dir = self.config.get_property('work.directory') self.log = LoggingService(log_level).get_logger() self._loop = asyncio.new_event_loop() # DO NOT CHANGE 'distributed.trace.processor' which is an optional user defined trace aggregator my_tracer = DistributedTrace(self, 'distributed.trace.processor') my_nc = self.config.get_property('network.connector') self._cloud = NetworkConnector(self, my_tracer, self._loop, my_nc, self.origin) self._function_queues = dict() self._executor = concurrent.futures.ThreadPoolExecutor( max_workers=self._max_threads) self.log.info(f'Concurrent thread pool = {self._max_threads}') # # Before we figure out how to solve blocking file I/O, we will regulate event output rate. # my_test_dir = self.util.normalize_path( f'{self.work_dir}/safe_to_delete_when_apps_stop') if not os.path.exists(my_test_dir): os.makedirs(my_test_dir, exist_ok=True) self._throttle = Throttle(self.util.normalize_path(f'{my_test_dir}/' + self.origin), log=self.log) self._seq = 0 self.log.info( f'Estimated performance is {format(self._throttle.get_tps(), ",d")} events per second' ) self.running = True self.stopped = False # distributed trace sessions self._traces = {} self.trace_aggregation = True # start event loop in a new thread to avoid blocking the main thread def main_event_loop(): self.log.info('Event system started') self._loop.run_forever() self.log.info('Event system stopped') self._loop.close() threading.Thread(target=main_event_loop).start() def get_origin(self): """ Get the origin ID of this application instance Returns: origin ID """ return self.origin def get_logger(self): """ Get Logger Returns: logger instance """ return self.log def is_trace_supported(self): return self.trace_aggregation def set_trace_support(self, enabled: bool = True): self.trace_aggregation = enabled status = 'ON' if enabled else 'OFF' self.log.info(f'Trace aggregation is {status}') def get_trace_id(self) -> str: """ Get trace ID for a transaction Returns: trace ID """ trace_info = self.get_trace() return trace_info.get_id() if trace_info is not None else None def get_trace(self) -> TraceInfo: """ Get trace info for a transaction Returns: trace info """ thread_id = threading.get_ident() return self._traces[thread_id] if thread_id in self._traces else None def annotate_trace(self, key: str, value: str) -> None: """ Annotate a trace at the current point of a transaction Args: key: any key value: any value Returns: None """ trace_info = self.get_trace() if trace_info is not None and isinstance(trace_info, TraceInfo): trace_info.annotate(key, value) def start_tracing(self, route: str, trace_id: str = None, trace_path: str = None) -> None: """ This method is reserved for system use. DO NOT call this from a user application. Args: route: route name trace_id: id trace_path: path such as Method and URI Returns: None """ thread_id = threading.get_ident() self._traces[thread_id] = TraceInfo(route, trace_id, trace_path) def stop_tracing(self) -> TraceInfo: """ This method is reserved for system use. DO NOT call this from a user application. Returns: trace info """ thread_id = threading.get_ident() if thread_id in self._traces: trace_info = self.get_trace() self._traces.pop(thread_id) return trace_info def run_forever(self) -> None: """ Tell the platform to run in the background until user presses CTL-C or the application is stopped by admin Returns: None """ def graceful_shutdown(signum, frame): self.running = False if frame is not None: self.log.warn('Control-C detected' if signal.SIGINT == signum else 'KILL signal detected') if threading.current_thread() is threading.main_thread(): signal.signal(signal.SIGTERM, graceful_shutdown) signal.signal(signal.SIGINT, graceful_shutdown) # keep the main thread running so CTL-C can be detected self.log.info('To stop this application, press Control-C') while self.running: time.sleep(0.1) # exit forever loop and ask platform to end event loop self.stop() else: raise ValueError( 'Unable to register Control-C and KILL signals because this is not the main thread' ) def register(self, route: str, user_function: any, total_instances: int = 1, is_private: bool = False) -> None: """ Register a user function Args: route: ID of the function user_function: the lambda function given by you total_instances: 1 for singleton or more for concurrency is_private: true if internal function within this application instance Returns: None """ self.util.validate_service_name(route) if not isinstance(total_instances, int): raise ValueError( f'Expect total_instances to be int, actual: {type(total_instances)}' ) if total_instances < 1: raise ValueError('total_instances must be at least 1') if total_instances > self._max_threads: raise ValueError( f'total_instances must not exceed max threads of {self._max_threads}' ) function_type = self.util.get_function_type(user_function) if function_type == FunctionType.NOT_SUPPORTED: raise ValueError( 'Function signature should be (headers: dict, body: any, instance: int) or ' + '(headers: dict, body: any) or (event: EventEnvelope)') if route in self._function_queues: self.log.warn(f'{route} will be reloaded') self.release(route) queue = asyncio.Queue() if function_type == FunctionType.INTERCEPTOR: self._function_queues[route] = { 'queue': queue, 'private': is_private, 'instances': 1 } ServiceQueue(self._loop, self._executor, queue, route, user_function, 0) elif function_type == FunctionType.REGULAR: self._function_queues[route] = { 'queue': queue, 'private': is_private, 'instances': total_instances } ServiceQueue(self._loop, self._executor, queue, route, user_function, total_instances) else: # function_type == FunctionType.SINGLETON self._function_queues[route] = { 'queue': queue, 'private': is_private, 'instances': 1 } ServiceQueue(self._loop, self._executor, queue, route, user_function, -1) # advertise the new route to the network if self._cloud.is_ready() and not is_private: self._cloud.send_payload({'type': 'add', 'route': route}) def cloud_ready(self): return self._cloud.is_ready() def subscribe_life_cycle(self, callback: str): self._cloud.subscribe_life_cycle(callback) def unsubscribe_life_cycle(self, callback: str): self._cloud.unsubscribe_life_cycle(callback) def release(self, route: str) -> None: # this will un-register a route if not isinstance(route, str): raise ValueError(f'Expect route to be str, actual: {type(route)}') if route not in self._function_queues: raise ValueError(f'route {route} not found') # advertise the deleted route to the network if self._cloud.is_ready() and self.route_is_private(route): self._cloud.send_payload({'type': 'remove', 'route': route}) self._remove_route(route) def has_route(self, route: str) -> bool: if not isinstance(route, str): raise ValueError(f'Expect route to be str, actual: {type(route)}') return route in self._function_queues def get_routes(self, options: str = 'all'): result = list() if 'public' == options: for route in self._function_queues: if not self.route_is_private(route): result.append(route) return result elif 'private' == options: for route in self._function_queues: if self.route_is_private(route): result.append(route) return result elif 'all' == options: return list(self._function_queues.keys()) else: return result def route_is_private(self, route: str) -> bool: config = self._function_queues[route] if config and 'private' in config: return config['private'] else: return False def route_instances(self, route: str) -> int: config = self._function_queues[route] if config and 'instances' in config: return config['instances'] else: return 0 def parallel_request(self, events: list, timeout_seconds: float): timeout_value = self.util.get_float(timeout_seconds) if timeout_value <= 0: raise ValueError( 'timeout value in seconds must be positive number') if not isinstance(events, list): raise ValueError('events must be a list of EventEnvelope') if len(events) == 0: raise ValueError('event list is empty') if len(events) == 1: result = list() result.append(self.request(events[0], timeout_value)) return result for evt in events: if not isinstance(evt, EventEnvelope): raise ValueError('events must be a list of EventEnvelope') # retrieve distributed tracing info if any trace_info = self.get_trace() # emulate RPC inbox = Inbox(self) temp_route = inbox.get_route() inbox_queue = inbox.get_queue() try: for evt in events: # restore distributed tracing info from current thread if trace_info: if trace_info.get_route( ) is not None and evt.get_from() is None: evt.set_from(trace_info.get_route()) if trace_info.get_id() is not None and trace_info.get_path( ) is not None: evt.set_trace(trace_info.get_id(), trace_info.get_path()) route = evt.get_to() evt.set_reply_to(temp_route, me=True) if route in self._function_queues: self._loop.call_soon_threadsafe(self._send, route, evt.to_map()) else: if self._cloud.is_connected(): self._cloud.send_payload({ 'type': 'event', 'event': evt.to_map() }) else: raise ValueError(f'route {route} not found') total_requests = len(events) result_list = list() while True: try: # wait until all response events are delivered to the inbox result_list.append(inbox_queue.get(True, timeout_value)) if len(result_list) == len(events): return result_list except Empty: raise TimeoutError( f'Requests timeout for {round(timeout_value, 3)} seconds. ' f'Expect: {total_requests} responses, actual: {len(result_list)}' ) finally: inbox.close() def request(self, event: EventEnvelope, timeout_seconds: float): timeout_value = self.util.get_float(timeout_seconds) if timeout_value <= 0: raise ValueError( 'timeout value in seconds must be positive number') if not isinstance(event, EventEnvelope): raise ValueError('event object must be an EventEnvelope') # restore distributed tracing info from current thread trace_info = self.get_trace() if trace_info: if trace_info.get_route() is not None and event.get_from() is None: event.set_from(trace_info.get_route()) if trace_info.get_id() is not None and trace_info.get_path( ) is not None: event.set_trace(trace_info.get_id(), trace_info.get_path()) # emulate RPC inbox = Inbox(self) temp_route = inbox.get_route() inbox_queue = inbox.get_queue() try: route = event.get_to() event.set_reply_to(temp_route, me=True) if route in self._function_queues: self._loop.call_soon_threadsafe(self._send, route, event.to_map()) else: if self._cloud.is_connected(): self._cloud.send_payload({ 'type': 'event', 'event': event.to_map() }) else: raise ValueError(f'route {route} not found') # wait until response event is delivered to the inbox return inbox_queue.get(True, timeout_value) except Empty: raise TimeoutError( f'Route {event.get_to()} timeout for {round(timeout_value, 3)} seconds' ) finally: inbox.close() def send_event(self, event: EventEnvelope, broadcast=False) -> None: if not isinstance(event, EventEnvelope): raise ValueError('event object must be an EventEnvelope class') # restore distributed tracing info from current thread trace_info = self.get_trace() if trace_info: if trace_info.get_route() is not None and event.get_from() is None: event.set_from(trace_info.get_route()) if trace_info.get_id() is not None and trace_info.get_path( ) is not None: event.set_trace(trace_info.get_id(), trace_info.get_path()) # regulate rate for best performance self._seq += 1 self._throttle.regulate_rate(self._seq) route = event.get_to() if broadcast: event.set_broadcast(True) reply_to = event.get_reply_to() if reply_to: target = reply_to[2:] if reply_to.startswith('->') else reply_to if route == target: raise ValueError('route and reply_to must not be the same') if route in self._function_queues: if event.is_broadcast() and self._cloud.is_connected(): self._cloud.send_payload({ 'type': 'event', 'event': event.to_map() }) else: self._loop.call_soon_threadsafe(self._send, route, event.to_map()) else: if self._cloud.is_connected(): self._cloud.send_payload({ 'type': 'event', 'event': event.to_map() }) else: raise ValueError(f'route {route} not found') def send_event_later(self, event: EventEnvelope, delay_in_seconds: float) -> None: self._loop.call_later(delay_in_seconds, self.send_event, event) def exists(self, routes: any): if isinstance(routes, str): single_route = routes if self.has_route(single_route): return True if self.cloud_ready(): event = EventEnvelope() event.set_to(self.SERVICE_QUERY).set_header( 'type', 'find').set_header('route', single_route) result = self.request(event, 8.0) if isinstance(result, EventEnvelope): if result.get_body() is not None: return result.get_body() if isinstance(routes, list): if len(routes) > 0: remote_routes = list() for r in routes: if not self.has_route(r): remote_routes.append(r) if len(remote_routes) == 0: return True if self.cloud_ready(): # tell service query to use the route list in body event = EventEnvelope() event.set_to(self.SERVICE_QUERY).set_header('type', 'find') event.set_header('route', '*').set_body(routes) result = self.request(event, 8.0) if isinstance( result, EventEnvelope) and result.get_body() is not None: return result.get_body() return False def _remove_route(self, route): if route in self._function_queues: self._send(route, None) self._function_queues.pop(route) def _send(self, route, event): if route in self._function_queues: config = self._function_queues[route] if 'queue' in config: config['queue'].put_nowait(event) def connect_to_cloud(self): self._loop.run_in_executor(self._executor, self._cloud.start_connection) def stop(self): # # to allow user application to invoke the "stop" method from a registered service, # the system must start a new thread so that the service can finish first. # if not self.stopped: self.log.info('Bye') # guarantee this stop function to execute only once self.stopped = True # exit the run_forever loop if any self.running = False # in case the calling function has just send an event asynchronously time.sleep(1.0) threading.Thread(target=self._bye).start() def _bye(self): def stopping(): route_list = [] for route in self.get_routes(): route_list.append(route) for route in route_list: self._remove_route(route) self._loop.create_task(full_stop()) async def full_stop(): # give time for registered services to stop await asyncio.sleep(1.0) queue_dir = self.util.normalize_path( f'{self.work_dir}/queues/{self.get_origin()}') self.util.cleanup_dir(queue_dir) self._loop.stop() self._cloud.close_connection( 1000, f'Application {self.get_origin()} is stopping', stop_engine=True) self._loop.call_soon_threadsafe(stopping)