def parse_args(args=None): parser = argparse.ArgumentParser( description="Lightbus management command.") subparsers = parser.add_subparsers(help="Commands", dest="subcommand") subparsers.required = True # Allow each command to set up its own arguments lightbus.commands.run.Command().setup(parser, subparsers) lightbus.commands.shell.Command().setup(parser, subparsers) lightbus.commands.dump_schema.Command().setup(parser, subparsers) lightbus.commands.dump_config_schema.Command().setup(parser, subparsers) lightbus.commands.inspect.Command().setup(parser, subparsers) lightbus.commands.version.Command().setup(parser, subparsers) # Create a temporary plugin registry in order to run the before_parse_args hook plugin_registry = PluginRegistry() plugin_registry.autoload_plugins(config=Config.load_dict({})) block( plugin_registry.execute_hook("before_parse_args", parser=parser, subparsers=subparsers), timeout=5, ) args = parser.parse_args(sys.argv[1:] if args is None else args) # Note that we don't have an after_parse_args plugin hook. Instead we use the receive_args # hook which is called once we have instantiated our plugins return args
def handle(self, args, config, plugin_registry: PluginRegistry): """Entrypoint for the inspect command""" self.setup_logging(args.log_level or "warning", config) bus_module, bus = self.import_bus(args) if args.internal or args.api: apis = bus.client.api_registry.all() else: apis = bus.client.api_registry.public() if args.api and args.api not in {a.meta.name for a in apis}: sys.stderr.write( f"Specified API was not found locally or within the schema on the bus. Cannot continue.\n" ) exit(1) try: for api in apis: if not args.api or self.wildcard_match(args.api, api.meta.name): logger.debug(f"Inspecting {api.meta.name}") block(self.search_in_api(args, api, bus)) else: logger.debug(f"API {api.meta.name} did not match {args.api}. Skipping") except KeyboardInterrupt: logger.info("Stopped by user")
async def test_block_within_event_loop(): # Should give an error async def co(): pass with pytest.raises(CannotBlockHere): block(co())
def close(self, _stop_worker=True): """Close the bus client This will cancel all tasks and close all transports/connections """ if self._closed: raise BusAlreadyClosed() block(self.close_async(_stop_worker=_stop_worker))
def _handle_worker_shutdown(self): # This method will be called within the worker thead, but after the worker # thread's event loop has stopped try: # Close _close_async_inner() because, we are in the worker thead. block(self._close_async_inner()) except BusAlreadyClosed: # In the case of a clean shutdown the bus will already be closed. pass
def handle(self, args, config, plugin_registry: PluginRegistry): command_utilities.setup_logging(args.log_level or "warning", config) bus_module, bus = command_utilities.import_bus(args) block(bus.client.lazy_load_now()) bus.schema.save_local(args.out) if args.out: sys.stderr.write("Schema for {} APIs saved to {}\n".format( len(bus.schema.api_names), Path(args.out).resolve()))
def parse_args(args=None): parser = argparse.ArgumentParser( description="Lightbus management command.") parser.add_argument( "--service-name", "-s", help="Name of service in which this process resides. YOU SHOULD " "LIKELY SET THIS IN PRODUCTION. Can also be set using the " "LIGHTBUS_SERVICE_NAME environment. Will default to a random string.", ) parser.add_argument( "--process-name", "-p", help= "A unique name of this process within the service. Can also be set using the " "LIGHTBUS_PROCESS_NAME environment. Will default to a random string.", ) parser.add_argument("--config", dest="config_file", help="Config file to load, JSON or YAML", metavar="FILE") parser.add_argument( "--log-level", help="Set the log level. Overrides any value set in config. " "One of debug, info, warning, critical, exception.", metavar="LOG_LEVEL", ) subparsers = parser.add_subparsers(help="Commands", dest="subcommand") subparsers.required = True lightbus.commands.run.Command().setup(parser, subparsers) lightbus.commands.shell.Command().setup(parser, subparsers) lightbus.commands.dump_schema.Command().setup(parser, subparsers) lightbus.commands.dump_schema.Command().setup(parser, subparsers) lightbus.commands.dump_config_schema.Command().setup(parser, subparsers) lightbus.commands.inspect.Command().setup(parser, subparsers) # Create a temporary plugin registry in order to run the before_parse_args hook plugin_registry = PluginRegistry() plugin_registry.autoload_plugins(config=Config.load_dict({})) block( plugin_registry.execute_hook("before_parse_args", parser=parser, subparsers=subparsers), timeout=5, ) args = parser.parse_args(sys.argv[1:] if args is None else args) # Note that we don't have an after_parse_args plugin hook. Instead we use the receive_args # hook which is called once we have instantiated our plugins return args
def handle(self, args, config, plugin_registry: PluginRegistry): """Entrypoint for the inspect command""" command_utilities.setup_logging(args.log_level or "warning", config) bus_module, bus = command_utilities.import_bus(args) api_names: List[str] block(bus.client.lazy_load_now()) # Locally registered APIs api_names = [api.meta.name for api in bus.client.api_registry.all()] # APIs registered to other services on the bus for api_name in bus.client.schema.api_names: if api_name not in api_names: api_names.append(api_name) if not args.internal and not args.api: # Hide internal APIs if we don't want them api_names = [ api_name for api_name in api_names if not api_name.startswith("internal.") ] if args.api and args.api not in api_names: sys.stderr.write( f"Specified API was not found locally or within the schema on the bus.\n" f"Ensure a Lightbus worker is running for this API.\n" f"Cannot continue.\n") sys.exit(1) api_names_to_inspect = [] for api_name in api_names: if not args.api or self.wildcard_match(args.api, api_name): api_names_to_inspect.append(api_name) if len(api_names_to_inspect) != 1 and args.follow: sys.stderr.write( f"The --follow option is only available when following a single API.\n" f"Please specify the --api option to select a single API to follow.\n" ) sys.exit(1) try: for api_name in api_names_to_inspect: if not args.api or self.wildcard_match(args.api, api_name): logger.debug(f"Inspecting {api_name}") block(self.search_in_api(args, api_name, bus)) else: logger.debug( f"API {api_name} did not match {args.api}. Skipping") except KeyboardInterrupt: logger.info("Stopped by user")
def test_block_outside_event_loop(): # Should not raise an error async def co(): return 1 resuult = block(co()) assert resuult == 1
def _handle(self, args, config, plugin_registry: PluginRegistry): self.setup_logging(override=getattr(args, "log_level", None), config=config) bus_module, bus = self.import_bus(args) # TODO: Move to lightbus.create()? if args.schema: if args.schema == "-": # if '-' read from stdin source = None else: source = args.schema bus.schema.load_local(source) restart_signals = (signal.SIGINT, signal.SIGTERM) # Handle incoming signals async def signal_handler(): # Stop handling signals now. If we receive the signal again # let the process quit naturally for signal_ in restart_signals: asyncio.get_event_loop().remove_signal_handler(signal_) logger.debug("Caught signal. Stopping main thread event loop") bus.client.shutdown_server(exit_code=0) for signal_ in restart_signals: asyncio.get_event_loop().add_signal_handler( signal_, lambda: asyncio.ensure_future(signal_handler())) try: block(plugin_registry.execute_hook("receive_args", args=args), timeout=5) if args.events_only: bus.client.run_forever(consume_rpcs=False) else: bus.client.run_forever() finally: # Cleanup signal handlers for signal_ in restart_signals: asyncio.get_event_loop().remove_signal_handler(signal_) if bus.client.exit_code: sys.exit(bus.client.exit_code)
def call(self, *args, bus_options: dict = None, **kwargs): """Call this BusPath node as an RPC" In contrast to __call__(), this method provides the ability to call with the additional `bus_options` argument. """ # Use a larger value of `rpc_timeout` because call_rpc_remote() should # handle timeout rpc_timeout = self.client.config.api(self.api_name).rpc_timeout * 1.5 return block(self.call_async(*args, **kwargs, bus_options=bus_options), timeout=rpc_timeout)
def listen(self, listener, *, listener_name: str, bus_options: dict = None): return block( self.listen_async(listener, listener_name=listener_name, bus_options=bus_options), timeout=self.client.config.api( self.api_name).event_listener_setup_timeout, )
def start_server(self, consume_rpcs=True): """Server startup procedure Must be called from within the main thread """ # Ensure an event loop exists get_event_loop() self._server_shutdown_queue = janus.Queue() self._server_tasks = set() async def server_shutdown_monitor(): exit_code = await self._server_shutdown_queue.async_q.get() self.exit_code = exit_code self.loop.stop() self._server_shutdown_queue.async_q.task_done() shutdown_monitor_task = asyncio.ensure_future(server_shutdown_monitor()) shutdown_monitor_task.add_done_callback(make_exception_checker(self, die=True)) self._shutdown_monitor_task = shutdown_monitor_task block(self._start_server_inner())
def handle(self, args, config, plugin_registry: PluginRegistry, fake_it=False): command_utilities.setup_logging(args.log_level or "warning", config) try: # pylint: disable=unused-import,cyclic-import,import-outside-toplevel import bpython from bpython.curtsies import main as bpython_main except ImportError: # pragma: no cover print( "Lightbus shell requires bpython. Run `pip install bpython` to install bpython." ) sys.exit(1) return # noqa lightbus_logger = logging.getLogger("lightbus") lightbus_logger.setLevel(logging.WARNING) bus_module, bus = command_utilities.import_bus(args) block(bus.client.lazy_load_now()) objects = {k: v for k, v in lightbus.__dict__.items() if isclass(v)} objects.update(bus=bus) block(plugin_registry.execute_hook("receive_args", args=args), timeout=5) # Ability to not start up the repl is useful for testing if not fake_it: bpython_main( args=["-i", "-q"], locals_=objects, welcome_message= "Welcome to the Lightbus shell. Use `bus` to access your bus.", )
def create(*args, **kwargs) -> BusPath: """ Create a new bus instance which can be used to access the bus. Typically this will be used as follows: import lightbus bus = lightbus.create() See Also: This function is a wrapper around `create_async()`, see `create_async()` for a list of arguments """ return block(create_async(*args, **kwargs), timeout=5)
def run_forever(self): block(self.start_worker()) self._actually_run_forever() logger.debug("Main thread event loop was stopped") # Close down the worker logger.debug("Stopping worker") block(self.stop_worker()) # Close down the client logger.debug("Closing bus") block(self.close_async()) return self.exit_code
def stop_server(self): block(cancel(self._shutdown_monitor_task)) block(self._stop_server_inner())
def two(): # Blocks! logger.debug("--> Two started") block(three()) logger.debug("--> Two finished")
def close(self): logger.debug("Bus is closing") self._call_queue.sync_q.join() block(cancel(self._perform_calls_task))
def worker(self, bus_client, after_shutdown: Callable = None): """ A note about error handling in the worker thread: There are two scenarios in which the worker thread my encounter an error. 1. The bus is being used as a client. A bus method is called by the client code, and this call raises an exception. This exception is propagated to the client code for it to deal with.1 2. The bus is being used as a server and has various coroutines running at any one time. In this case, if a coroutine encounters an error then it should cause the lightbus server to exit. In response to either of these cases the bus needs to shut itself down. Therefore, the worker needs to keep on running for a while in order to handle the various shutdown tasks. In case 1 above, we assume the developer will take responsibility for closing the bus correctly when they are done with it. In case 2 above, the worker needs to signal the main lightbus run process to tell it to begin the shutdown procedure """ logger.debug(f"Bus thread {self._thread.name} initialising") # Start a new event loop for this new thread asyncio.set_event_loop(asyncio.new_event_loop()) self._call_queue = janus.Queue() self._worker_shutdown_queue = janus.Queue() async def worker_shutdown_monitor(): await self._worker_shutdown_queue.async_q.get() asyncio.get_event_loop().stop() self._worker_shutdown_queue.async_q.task_done() shutdown_monitor_task = asyncio.ensure_future(worker_shutdown_monitor()) shutdown_monitor_task.add_done_callback(make_exception_checker(bus_client, die=True)) perform_calls_task = asyncio.ensure_future(self.perform_calls()) perform_calls_task.add_done_callback(make_exception_checker(bus_client, die=True)) self._ready.set() asyncio.get_event_loop().run_forever() logging.debug(f"Event loop stopped in bus worker thread {self._thread.name}. Closing down.") self._ready.clear() if after_shutdown: after_shutdown() logger.debug("Canceling worker tasks") block(cancel(perform_calls_task, shutdown_monitor_task)) logger.debug("Closing the call queue") self._call_queue.close() block(self._call_queue.wait_closed()) logger.debug("Closing the worker shutdown queue") self._worker_shutdown_queue.close() block(self._worker_shutdown_queue.wait_closed()) logger.debug("Worker shutdown complete")
def register_api(self, api: Api): block(self.register_api_async(api), timeout=5)
def fire(self, *args, bus_options: dict = None, **kwargs): """Fire an event for this BusPath node""" return block( self.fire_async(*args, **kwargs, bus_options=bus_options), timeout=self.client.config.api(self.api_name).event_fire_timeout, )
def handle(self, args, config, plugin_registry: PluginRegistry): try: self._handle(args, config, plugin_registry) except Exception as e: block(plugin_registry.execute_hook("exception", e=e), timeout=5) raise
def output(self, args, transport: EventTransport, message: EventMessage, bus: BusPath): """Print out the given message""" serialized = transport.serializer(message) if args.format in ("json", "pretty"): if args.format == "pretty": dumped = json.dumps(serialized, indent=4) else: dumped = json.dumps(serialized) sys.stdout.write(dumped) sys.stdout.write("\n") sys.stdout.flush() elif args.format == "human": print(Colors.BGreen, end="") print(f" {message.api_name}.{message.event_name} ".center(80, "=")) if hasattr(message, "datetime"): print(f" {message.datetime.strftime('%c')} ".center(80, " ")) print(Colors.Reset, end="") print(f"\n{Colors.BWhite}Metadata:{Colors.Reset}") for k, v in message.get_metadata().items(): print(f" {str(k).ljust(20)}: {v}") print(f"\n{Colors.BWhite}Data:{Colors.Reset}") for k, v in message.get_kwargs().items(): if isinstance(v, (dict, list)): v = json.dumps(v, indent=4) pad = " " * 24 v = "".join(pad + v for v in v.splitlines(keepends=True)).lstrip() print(f" {str(k).ljust(20)}: {v}") if args.validate or args.show_casting: print(f"\n{Colors.BWhite}Extra:{Colors.Reset}") if args.validate: try: bus.client.schema.validate_parameters( message.api_name, message.event_name, message.kwargs) except ValidationError as e: validation_message = f"{Colors.Red}{e}{Colors.Reset}" else: validation_message = f"{Colors.Green}Passed{Colors.Reset}" print(f" Validation: {validation_message}") if args.show_casting: block(bus.client.hook_registry.execute("before_worker_start")) for listener in bus.client.event_client._event_listeners: if (message.api_name, message.event_name) not in listener.events: continue hints = get_type_hints(listener.callable) casted = cast_to_signature(parameters=message.kwargs, callable=listener.callable) print( f"\n {Colors.BWhite}Casting for listener: {listener.name}{Colors.Reset}" ) for key, value in message.kwargs.items(): was = type(value) via = hints[key] now = type(casted[key]) color = Colors.Green if via == now else Colors.Red print(" " f"{color}{str(key).ljust(20)}: " f"Received a '{was.__name__}', " f"casted to a '{via.__name__}', " f"result was a '{now.__name__}'" f"{Colors.Reset}") print("\n") else: sys.stderr.write(f"Unknown output format '{args.format}'\n") sys.exit(1)
def setup(self, plugins: dict = None): block(self.setup_async(plugins), timeout=5)
def create_client( config: Union[dict, RootConfig] = None, *, config_file: str = None, service_name: str = None, process_name: str = None, features: List[Union[Feature, str]] = ALL_FEATURES, client_class: Type[BusClient] = BusClient, node_class: Type[BusPath] = BusPath, plugins=None, flask: bool = False, hook_registry: Optional[HookRegistry] = None, api_registry: Optional[ApiRegistry] = None, **kwargs, ) -> BusClient: """ Create a new bus instance which can be used to access the bus. Typically this will be used as follows: import lightbus bus = lightbus.create() This will be a `BusPath` instance. If you wish to access the lower level `BusClient` you can do so via `bus.client`. Args: config (dict, Config): The config object or dictionary to load config_file (str): The path to a config file to load (should end in .json or .yaml) service_name (str): The name of this service - will be used when creating event consumer groups process_name (str): The unique name of this process - used when retrieving unprocessed events following a crash client_class (Type[BusClient]): The class from which the bus client will be instantiated node_class (BusPath): The class from which the bus path will be instantiated plugins (list): A list of plugin instances to load flask (bool): Are we using flask? If so we will make sure we don't start lightbus in the reloader process **kwargs (): Any additional instantiation arguments to be passed to `client_class`. Returns: BusPath """ if flask: in_flask_server = sys.argv[0].endswith("flask") and "run" in sys.argv if in_flask_server and os.environ.get("WERKZEUG_RUN_MAIN", "").lower() != "true": # Flask has a reloader process that shouldn't start a lightbus client return # Ensure an event loop exists, as creating InternalQueue # objects requires that we have one. get_event_loop() # If were are running via the Lightbus CLI then we may have # some command line arguments we need to apply. # pylint: disable=cyclic-import,import-outside-toplevel from lightbus.commands import COMMAND_PARSED_ARGS config_file = COMMAND_PARSED_ARGS.get("config_file", None) or config_file service_name = COMMAND_PARSED_ARGS.get("service_name", None) or service_name process_name = COMMAND_PARSED_ARGS.get("process_name", None) or process_name if config is None: config = load_config(from_file=config_file, service_name=service_name, process_name=process_name) if isinstance(config, Mapping): config = Config.load_dict(config or {}) elif isinstance(config, RootConfig): config = Config(config) transport_registry = kwargs.pop( "transport_registry", None) or TransportRegistry().load_config(config) schema = Schema( schema_transport=transport_registry.get_schema_transport(), max_age_seconds=config.bus().schema.ttl, human_readable=config.bus().schema.human_readable, ) error_queue: ErrorQueueType = InternalQueue() # Plugin registry plugin_registry = PluginRegistry() if plugins is None: logger.debug("Auto-loading any installed Lightbus plugins...") plugin_registry.autoload_plugins(config) else: logger.debug("Loading explicitly specified Lightbus plugins....") plugin_registry.set_plugins(plugins) # Hook registry if not hook_registry: hook_registry = HookRegistry( error_queue=error_queue, execute_plugin_hooks=plugin_registry.execute_hook) # API registry if not api_registry: api_registry = ApiRegistry() api_registry.add(LightbusStateApi()) api_registry.add(LightbusMetricsApi()) events_queue_client_to_dock = InternalQueue() events_queue_dock_to_client = InternalQueue() event_client = EventClient( api_registry=api_registry, hook_registry=hook_registry, config=config, schema=schema, error_queue=error_queue, consume_from=events_queue_dock_to_client, produce_to=events_queue_client_to_dock, ) event_dock = EventDock( transport_registry=transport_registry, api_registry=api_registry, config=config, error_queue=error_queue, consume_from=events_queue_client_to_dock, produce_to=events_queue_dock_to_client, ) rpcs_queue_client_to_dock = InternalQueue() rpcs_queue_dock_to_client = InternalQueue() rpc_result_client = RpcResultClient( api_registry=api_registry, hook_registry=hook_registry, config=config, schema=schema, error_queue=error_queue, consume_from=rpcs_queue_dock_to_client, produce_to=rpcs_queue_client_to_dock, ) rpc_result_dock = RpcResultDock( transport_registry=transport_registry, api_registry=api_registry, config=config, error_queue=error_queue, consume_from=rpcs_queue_client_to_dock, produce_to=rpcs_queue_dock_to_client, ) client = client_class( config=config, hook_registry=hook_registry, plugin_registry=plugin_registry, features=features, schema=schema, api_registry=api_registry, event_client=event_client, rpc_result_client=rpc_result_client, error_queue=error_queue, transport_registry=transport_registry, **kwargs, ) # Pass the client to any hooks # (use a weakref to prevent circular references) hook_registry.set_extra_parameter("client", weakref.proxy(client)) # We don't do this normally as the docks do not need to be # accessed directly, but this is useful in testing # TODO: Testing flag removed, but these are only needed in testing. # Perhaps wrap them up in a way that makes this obvious client.event_dock = event_dock client.rpc_result_dock = rpc_result_dock log_welcome_message( logger=logger, transport_registry=transport_registry, schema=schema, plugin_registry=plugin_registry, config=config, ) # Wait until the docks are ready to go if not get_event_loop().is_running(): # If the event loop is running then we can assume that this setup # will happen momentarily anyway. Plus, if an event loop is running # then we cannot block. This is mainly to make sure everything # starts up in a sensible fashion when we are in non-worker-mode. block(event_dock.wait_until_ready(), timeout=2) block(rpc_result_dock.wait_until_ready(), timeout=2) return client
def _handle(self, args, config, plugin_registry: PluginRegistry): command_utilities.setup_logging(override=getattr( args, "log_level", None), config=config) bus_module, bus = command_utilities.import_bus(args) # Convert only & skip into a list of features to enable if args.only or args.skip: if args.only: features = args.only else: features = self.all_features for skip_feature in args.skip or []: if skip_feature in features: features.remove(skip_feature) elif os.environ.get("LIGHTBUS_FEATURES"): features = csv_type(os.environ.get("LIGHTBUS_FEATURES")) else: features = ALL_FEATURES bus.client.set_features(features) # TODO: Move to lightbus.create()? if args.schema: if args.schema == "-": # if '-' read from stdin source = None else: source = args.schema bus.schema.load_local(source) restart_signals = (signal.SIGINT, signal.SIGTERM) # Handle incoming signals async def signal_handler(): # Stop handling signals now. If we receive the signal again # let the process quit naturally for signal_ in restart_signals: asyncio.get_event_loop().remove_signal_handler(signal_) logger.debug("Caught signal. Stopping main thread event loop") bus.client.shutdown_server(exit_code=0) for signal_ in restart_signals: asyncio.get_event_loop().add_signal_handler( signal_, lambda: asyncio.ensure_future(signal_handler())) try: block(plugin_registry.execute_hook("receive_args", args=args), timeout=5) bus.client.run_forever() finally: # Cleanup signal handlers for signal_ in restart_signals: asyncio.get_event_loop().remove_signal_handler(signal_) if bus.client.exit_code: sys.exit(bus.client.exit_code)
def call(self, *, bus_options: dict = None, **kwargs): # Use a larger value of `rpc_timeout` because call_rpc_remote() should # handle timeout rpc_timeout = self.client.config.api(self.api_name).rpc_timeout * 1.5 return block(self.call_async(**kwargs, bus_options=bus_options), timeout=rpc_timeout)
def close(self): """Close the bus client This will cancel all tasks and close all transports/connections """ block(self.close_async())
def fire(self, *, bus_options: dict = None, **kwargs): return block( self.fire_async(**kwargs, bus_options=bus_options), timeout=self.client.config.api(self.api_name).event_fire_timeout, )