def parse_args(args=None):
parser = argparse.ArgumentParser(
description="Lightbus management command.")
subparsers = parser.add_subparsers(help="Commands", dest="subcommand")
subparsers.required = True
# Allow each command to set up its own arguments
lightbus.commands.run.Command().setup(parser, subparsers)
lightbus.commands.shell.Command().setup(parser, subparsers)
lightbus.commands.dump_schema.Command().setup(parser, subparsers)
lightbus.commands.dump_config_schema.Command().setup(parser, subparsers)
lightbus.commands.inspect.Command().setup(parser, subparsers)
lightbus.commands.version.Command().setup(parser, subparsers)
# Create a temporary plugin registry in order to run the before_parse_args hook
plugin_registry = PluginRegistry()
plugin_registry.autoload_plugins(config=Config.load_dict({}))
block(
plugin_registry.execute_hook("before_parse_args",
parser=parser,
subparsers=subparsers),
timeout=5,
)
args = parser.parse_args(sys.argv[1:] if args is None else args)
# Note that we don't have an after_parse_args plugin hook. Instead we use the receive_args
# hook which is called once we have instantiated our plugins
return args
def handle(self, args, config, plugin_registry: PluginRegistry):
"""Entrypoint for the inspect command"""
self.setup_logging(args.log_level or "warning", config)
bus_module, bus = self.import_bus(args)
if args.internal or args.api:
apis = bus.client.api_registry.all()
else:
apis = bus.client.api_registry.public()
if args.api and args.api not in {a.meta.name for a in apis}:
sys.stderr.write(
f"Specified API was not found locally or within the schema on the bus. Cannot continue.\n"
)
exit(1)
try:
for api in apis:
if not args.api or self.wildcard_match(args.api, api.meta.name):
logger.debug(f"Inspecting {api.meta.name}")
block(self.search_in_api(args, api, bus))
else:
logger.debug(f"API {api.meta.name} did not match {args.api}. Skipping")
except KeyboardInterrupt:
logger.info("Stopped by user")
async def test_block_within_event_loop():
# Should give an error
async def co():
pass
with pytest.raises(CannotBlockHere):
block(co())
def close(self, _stop_worker=True):
"""Close the bus client
This will cancel all tasks and close all transports/connections
"""
if self._closed:
raise BusAlreadyClosed()
block(self.close_async(_stop_worker=_stop_worker))
def _handle_worker_shutdown(self):
# This method will be called within the worker thead, but after the worker
# thread's event loop has stopped
try:
# Close _close_async_inner() because, we are in the worker thead.
block(self._close_async_inner())
except BusAlreadyClosed:
# In the case of a clean shutdown the bus will already be closed.
pass
def handle(self, args, config, plugin_registry: PluginRegistry):
command_utilities.setup_logging(args.log_level or "warning", config)
bus_module, bus = command_utilities.import_bus(args)
block(bus.client.lazy_load_now())
bus.schema.save_local(args.out)
if args.out:
sys.stderr.write("Schema for {} APIs saved to {}\n".format(
len(bus.schema.api_names),
Path(args.out).resolve()))
def parse_args(args=None):
parser = argparse.ArgumentParser(
description="Lightbus management command.")
parser.add_argument(
"--service-name",
"-s",
help="Name of service in which this process resides. YOU SHOULD "
"LIKELY SET THIS IN PRODUCTION. Can also be set using the "
"LIGHTBUS_SERVICE_NAME environment. Will default to a random string.",
)
parser.add_argument(
"--process-name",
"-p",
help=
"A unique name of this process within the service. Can also be set using the "
"LIGHTBUS_PROCESS_NAME environment. Will default to a random string.",
)
parser.add_argument("--config",
dest="config_file",
help="Config file to load, JSON or YAML",
metavar="FILE")
parser.add_argument(
"--log-level",
help="Set the log level. Overrides any value set in config. "
"One of debug, info, warning, critical, exception.",
metavar="LOG_LEVEL",
)
subparsers = parser.add_subparsers(help="Commands", dest="subcommand")
subparsers.required = True
lightbus.commands.run.Command().setup(parser, subparsers)
lightbus.commands.shell.Command().setup(parser, subparsers)
lightbus.commands.dump_schema.Command().setup(parser, subparsers)
lightbus.commands.dump_schema.Command().setup(parser, subparsers)
lightbus.commands.dump_config_schema.Command().setup(parser, subparsers)
lightbus.commands.inspect.Command().setup(parser, subparsers)
# Create a temporary plugin registry in order to run the before_parse_args hook
plugin_registry = PluginRegistry()
plugin_registry.autoload_plugins(config=Config.load_dict({}))
block(
plugin_registry.execute_hook("before_parse_args",
parser=parser,
subparsers=subparsers),
timeout=5,
)
args = parser.parse_args(sys.argv[1:] if args is None else args)
# Note that we don't have an after_parse_args plugin hook. Instead we use the receive_args
# hook which is called once we have instantiated our plugins
return args
def handle(self, args, config, plugin_registry: PluginRegistry):
"""Entrypoint for the inspect command"""
command_utilities.setup_logging(args.log_level or "warning", config)
bus_module, bus = command_utilities.import_bus(args)
api_names: List[str]
block(bus.client.lazy_load_now())
# Locally registered APIs
api_names = [api.meta.name for api in bus.client.api_registry.all()]
# APIs registered to other services on the bus
for api_name in bus.client.schema.api_names:
if api_name not in api_names:
api_names.append(api_name)
if not args.internal and not args.api:
# Hide internal APIs if we don't want them
api_names = [
api_name for api_name in api_names
if not api_name.startswith("internal.")
]
if args.api and args.api not in api_names:
sys.stderr.write(
f"Specified API was not found locally or within the schema on the bus.\n"
f"Ensure a Lightbus worker is running for this API.\n"
f"Cannot continue.\n")
sys.exit(1)
api_names_to_inspect = []
for api_name in api_names:
if not args.api or self.wildcard_match(args.api, api_name):
api_names_to_inspect.append(api_name)
if len(api_names_to_inspect) != 1 and args.follow:
sys.stderr.write(
f"The --follow option is only available when following a single API.\n"
f"Please specify the --api option to select a single API to follow.\n"
)
sys.exit(1)
try:
for api_name in api_names_to_inspect:
if not args.api or self.wildcard_match(args.api, api_name):
logger.debug(f"Inspecting {api_name}")
block(self.search_in_api(args, api_name, bus))
else:
logger.debug(
f"API {api_name} did not match {args.api}. Skipping")
except KeyboardInterrupt:
logger.info("Stopped by user")
def test_block_outside_event_loop():
# Should not raise an error
async def co():
return 1
resuult = block(co())
assert resuult == 1
def _handle(self, args, config, plugin_registry: PluginRegistry):
self.setup_logging(override=getattr(args, "log_level", None),
config=config)
bus_module, bus = self.import_bus(args)
# TODO: Move to lightbus.create()?
if args.schema:
if args.schema == "-":
# if '-' read from stdin
source = None
else:
source = args.schema
bus.schema.load_local(source)
restart_signals = (signal.SIGINT, signal.SIGTERM)
# Handle incoming signals
async def signal_handler():
# Stop handling signals now. If we receive the signal again
# let the process quit naturally
for signal_ in restart_signals:
asyncio.get_event_loop().remove_signal_handler(signal_)
logger.debug("Caught signal. Stopping main thread event loop")
bus.client.shutdown_server(exit_code=0)
for signal_ in restart_signals:
asyncio.get_event_loop().add_signal_handler(
signal_, lambda: asyncio.ensure_future(signal_handler()))
try:
block(plugin_registry.execute_hook("receive_args", args=args),
timeout=5)
if args.events_only:
bus.client.run_forever(consume_rpcs=False)
else:
bus.client.run_forever()
finally:
# Cleanup signal handlers
for signal_ in restart_signals:
asyncio.get_event_loop().remove_signal_handler(signal_)
if bus.client.exit_code:
sys.exit(bus.client.exit_code)
def call(self, *args, bus_options: dict = None, **kwargs):
"""Call this BusPath node as an RPC"
In contrast to __call__(), this method provides the ability to call
with the additional `bus_options` argument.
"""
# Use a larger value of `rpc_timeout` because call_rpc_remote() should
# handle timeout
rpc_timeout = self.client.config.api(self.api_name).rpc_timeout * 1.5
return block(self.call_async(*args, **kwargs, bus_options=bus_options), timeout=rpc_timeout)
def listen(self,
listener,
*,
listener_name: str,
bus_options: dict = None):
return block(
self.listen_async(listener,
listener_name=listener_name,
bus_options=bus_options),
timeout=self.client.config.api(
self.api_name).event_listener_setup_timeout,
)
def start_server(self, consume_rpcs=True):
"""Server startup procedure
Must be called from within the main thread
"""
# Ensure an event loop exists
get_event_loop()
self._server_shutdown_queue = janus.Queue()
self._server_tasks = set()
async def server_shutdown_monitor():
exit_code = await self._server_shutdown_queue.async_q.get()
self.exit_code = exit_code
self.loop.stop()
self._server_shutdown_queue.async_q.task_done()
shutdown_monitor_task = asyncio.ensure_future(server_shutdown_monitor())
shutdown_monitor_task.add_done_callback(make_exception_checker(self, die=True))
self._shutdown_monitor_task = shutdown_monitor_task
block(self._start_server_inner())
def handle(self,
args,
config,
plugin_registry: PluginRegistry,
fake_it=False):
command_utilities.setup_logging(args.log_level or "warning", config)
try:
# pylint: disable=unused-import,cyclic-import,import-outside-toplevel
import bpython
from bpython.curtsies import main as bpython_main
except ImportError: # pragma: no cover
print(
"Lightbus shell requires bpython. Run `pip install bpython` to install bpython."
)
sys.exit(1)
return # noqa
lightbus_logger = logging.getLogger("lightbus")
lightbus_logger.setLevel(logging.WARNING)
bus_module, bus = command_utilities.import_bus(args)
block(bus.client.lazy_load_now())
objects = {k: v for k, v in lightbus.__dict__.items() if isclass(v)}
objects.update(bus=bus)
block(plugin_registry.execute_hook("receive_args", args=args),
timeout=5)
# Ability to not start up the repl is useful for testing
if not fake_it:
bpython_main(
args=["-i", "-q"],
locals_=objects,
welcome_message=
"Welcome to the Lightbus shell. Use `bus` to access your bus.",
)
def create(*args, **kwargs) -> BusPath:
"""
Create a new bus instance which can be used to access the bus.
Typically this will be used as follows:
import lightbus
bus = lightbus.create()
See Also: This function is a wrapper around `create_async()`, see `create_async()`
for a list of arguments
"""
return block(create_async(*args, **kwargs), timeout=5)
def run_forever(self):
block(self.start_worker())
self._actually_run_forever()
logger.debug("Main thread event loop was stopped")
# Close down the worker
logger.debug("Stopping worker")
block(self.stop_worker())
# Close down the client
logger.debug("Closing bus")
block(self.close_async())
return self.exit_code
def stop_server(self):
block(cancel(self._shutdown_monitor_task))
block(self._stop_server_inner())
def two():
# Blocks!
logger.debug("--> Two started")
block(three())
logger.debug("--> Two finished")
def close(self):
logger.debug("Bus is closing")
self._call_queue.sync_q.join()
block(cancel(self._perform_calls_task))
def worker(self, bus_client, after_shutdown: Callable = None):
"""
A note about error handling in the worker thread:
There are two scenarios in which the worker thread my encounter an error.
1. The bus is being used as a client. A bus method is called by the client code,
and this call raises an exception. This exception is propagated to the client
code for it to deal with.1
2. The bus is being used as a server and has various coroutines running at any one
time. In this case, if a coroutine encounters an error then it should cause the
lightbus server to exit.
In response to either of these cases the bus needs to shut itself down. Therefore,
the worker needs to keep on running for a while in order to handle the various shutdown tasks.
In case 1 above, we assume the developer will take responsibility for closing the bus
correctly when they are done with it.
In case 2 above, the worker needs to signal the main lightbus run process to tell it to begin the
shutdown procedure
"""
logger.debug(f"Bus thread {self._thread.name} initialising")
# Start a new event loop for this new thread
asyncio.set_event_loop(asyncio.new_event_loop())
self._call_queue = janus.Queue()
self._worker_shutdown_queue = janus.Queue()
async def worker_shutdown_monitor():
await self._worker_shutdown_queue.async_q.get()
asyncio.get_event_loop().stop()
self._worker_shutdown_queue.async_q.task_done()
shutdown_monitor_task = asyncio.ensure_future(worker_shutdown_monitor())
shutdown_monitor_task.add_done_callback(make_exception_checker(bus_client, die=True))
perform_calls_task = asyncio.ensure_future(self.perform_calls())
perform_calls_task.add_done_callback(make_exception_checker(bus_client, die=True))
self._ready.set()
asyncio.get_event_loop().run_forever()
logging.debug(f"Event loop stopped in bus worker thread {self._thread.name}. Closing down.")
self._ready.clear()
if after_shutdown:
after_shutdown()
logger.debug("Canceling worker tasks")
block(cancel(perform_calls_task, shutdown_monitor_task))
logger.debug("Closing the call queue")
self._call_queue.close()
block(self._call_queue.wait_closed())
logger.debug("Closing the worker shutdown queue")
self._worker_shutdown_queue.close()
block(self._worker_shutdown_queue.wait_closed())
logger.debug("Worker shutdown complete")
def register_api(self, api: Api):
block(self.register_api_async(api), timeout=5)
def fire(self, *args, bus_options: dict = None, **kwargs):
"""Fire an event for this BusPath node"""
return block(
self.fire_async(*args, **kwargs, bus_options=bus_options),
timeout=self.client.config.api(self.api_name).event_fire_timeout,
)
def handle(self, args, config, plugin_registry: PluginRegistry):
try:
self._handle(args, config, plugin_registry)
except Exception as e:
block(plugin_registry.execute_hook("exception", e=e), timeout=5)
raise
def output(self, args, transport: EventTransport, message: EventMessage,
bus: BusPath):
"""Print out the given message"""
serialized = transport.serializer(message)
if args.format in ("json", "pretty"):
if args.format == "pretty":
dumped = json.dumps(serialized, indent=4)
else:
dumped = json.dumps(serialized)
sys.stdout.write(dumped)
sys.stdout.write("\n")
sys.stdout.flush()
elif args.format == "human":
print(Colors.BGreen, end="")
print(f" {message.api_name}.{message.event_name} ".center(80, "="))
if hasattr(message, "datetime"):
print(f" {message.datetime.strftime('%c')} ".center(80, " "))
print(Colors.Reset, end="")
print(f"\n{Colors.BWhite}Metadata:{Colors.Reset}")
for k, v in message.get_metadata().items():
print(f" {str(k).ljust(20)}: {v}")
print(f"\n{Colors.BWhite}Data:{Colors.Reset}")
for k, v in message.get_kwargs().items():
if isinstance(v, (dict, list)):
v = json.dumps(v, indent=4)
pad = " " * 24
v = "".join(pad + v
for v in v.splitlines(keepends=True)).lstrip()
print(f" {str(k).ljust(20)}: {v}")
if args.validate or args.show_casting:
print(f"\n{Colors.BWhite}Extra:{Colors.Reset}")
if args.validate:
try:
bus.client.schema.validate_parameters(
message.api_name, message.event_name, message.kwargs)
except ValidationError as e:
validation_message = f"{Colors.Red}{e}{Colors.Reset}"
else:
validation_message = f"{Colors.Green}Passed{Colors.Reset}"
print(f" Validation: {validation_message}")
if args.show_casting:
block(bus.client.hook_registry.execute("before_worker_start"))
for listener in bus.client.event_client._event_listeners:
if (message.api_name,
message.event_name) not in listener.events:
continue
hints = get_type_hints(listener.callable)
casted = cast_to_signature(parameters=message.kwargs,
callable=listener.callable)
print(
f"\n {Colors.BWhite}Casting for listener: {listener.name}{Colors.Reset}"
)
for key, value in message.kwargs.items():
was = type(value)
via = hints[key]
now = type(casted[key])
color = Colors.Green if via == now else Colors.Red
print(" "
f"{color}{str(key).ljust(20)}: "
f"Received a '{was.__name__}', "
f"casted to a '{via.__name__}', "
f"result was a '{now.__name__}'"
f"{Colors.Reset}")
print("\n")
else:
sys.stderr.write(f"Unknown output format '{args.format}'\n")
sys.exit(1)
def setup(self, plugins: dict = None):
block(self.setup_async(plugins), timeout=5)
def create_client(
config: Union[dict, RootConfig] = None,
*,
config_file: str = None,
service_name: str = None,
process_name: str = None,
features: List[Union[Feature, str]] = ALL_FEATURES,
client_class: Type[BusClient] = BusClient,
node_class: Type[BusPath] = BusPath,
plugins=None,
flask: bool = False,
hook_registry: Optional[HookRegistry] = None,
api_registry: Optional[ApiRegistry] = None,
**kwargs,
) -> BusClient:
"""
Create a new bus instance which can be used to access the bus.
Typically this will be used as follows:
import lightbus
bus = lightbus.create()
This will be a `BusPath` instance. If you wish to access the lower
level `BusClient` you can do so via `bus.client`.
Args:
config (dict, Config): The config object or dictionary to load
config_file (str): The path to a config file to load (should end in .json or .yaml)
service_name (str): The name of this service - will be used when creating event consumer groups
process_name (str): The unique name of this process - used when retrieving unprocessed events following a crash
client_class (Type[BusClient]): The class from which the bus client will be instantiated
node_class (BusPath): The class from which the bus path will be instantiated
plugins (list): A list of plugin instances to load
flask (bool): Are we using flask? If so we will make sure we don't start lightbus in the reloader process
**kwargs (): Any additional instantiation arguments to be passed to `client_class`.
Returns: BusPath
"""
if flask:
in_flask_server = sys.argv[0].endswith("flask") and "run" in sys.argv
if in_flask_server and os.environ.get("WERKZEUG_RUN_MAIN",
"").lower() != "true":
# Flask has a reloader process that shouldn't start a lightbus client
return
# Ensure an event loop exists, as creating InternalQueue
# objects requires that we have one.
get_event_loop()
# If were are running via the Lightbus CLI then we may have
# some command line arguments we need to apply.
# pylint: disable=cyclic-import,import-outside-toplevel
from lightbus.commands import COMMAND_PARSED_ARGS
config_file = COMMAND_PARSED_ARGS.get("config_file", None) or config_file
service_name = COMMAND_PARSED_ARGS.get("service_name",
None) or service_name
process_name = COMMAND_PARSED_ARGS.get("process_name",
None) or process_name
if config is None:
config = load_config(from_file=config_file,
service_name=service_name,
process_name=process_name)
if isinstance(config, Mapping):
config = Config.load_dict(config or {})
elif isinstance(config, RootConfig):
config = Config(config)
transport_registry = kwargs.pop(
"transport_registry", None) or TransportRegistry().load_config(config)
schema = Schema(
schema_transport=transport_registry.get_schema_transport(),
max_age_seconds=config.bus().schema.ttl,
human_readable=config.bus().schema.human_readable,
)
error_queue: ErrorQueueType = InternalQueue()
# Plugin registry
plugin_registry = PluginRegistry()
if plugins is None:
logger.debug("Auto-loading any installed Lightbus plugins...")
plugin_registry.autoload_plugins(config)
else:
logger.debug("Loading explicitly specified Lightbus plugins....")
plugin_registry.set_plugins(plugins)
# Hook registry
if not hook_registry:
hook_registry = HookRegistry(
error_queue=error_queue,
execute_plugin_hooks=plugin_registry.execute_hook)
# API registry
if not api_registry:
api_registry = ApiRegistry()
api_registry.add(LightbusStateApi())
api_registry.add(LightbusMetricsApi())
events_queue_client_to_dock = InternalQueue()
events_queue_dock_to_client = InternalQueue()
event_client = EventClient(
api_registry=api_registry,
hook_registry=hook_registry,
config=config,
schema=schema,
error_queue=error_queue,
consume_from=events_queue_dock_to_client,
produce_to=events_queue_client_to_dock,
)
event_dock = EventDock(
transport_registry=transport_registry,
api_registry=api_registry,
config=config,
error_queue=error_queue,
consume_from=events_queue_client_to_dock,
produce_to=events_queue_dock_to_client,
)
rpcs_queue_client_to_dock = InternalQueue()
rpcs_queue_dock_to_client = InternalQueue()
rpc_result_client = RpcResultClient(
api_registry=api_registry,
hook_registry=hook_registry,
config=config,
schema=schema,
error_queue=error_queue,
consume_from=rpcs_queue_dock_to_client,
produce_to=rpcs_queue_client_to_dock,
)
rpc_result_dock = RpcResultDock(
transport_registry=transport_registry,
api_registry=api_registry,
config=config,
error_queue=error_queue,
consume_from=rpcs_queue_client_to_dock,
produce_to=rpcs_queue_dock_to_client,
)
client = client_class(
config=config,
hook_registry=hook_registry,
plugin_registry=plugin_registry,
features=features,
schema=schema,
api_registry=api_registry,
event_client=event_client,
rpc_result_client=rpc_result_client,
error_queue=error_queue,
transport_registry=transport_registry,
**kwargs,
)
# Pass the client to any hooks
# (use a weakref to prevent circular references)
hook_registry.set_extra_parameter("client", weakref.proxy(client))
# We don't do this normally as the docks do not need to be
# accessed directly, but this is useful in testing
# TODO: Testing flag removed, but these are only needed in testing.
# Perhaps wrap them up in a way that makes this obvious
client.event_dock = event_dock
client.rpc_result_dock = rpc_result_dock
log_welcome_message(
logger=logger,
transport_registry=transport_registry,
schema=schema,
plugin_registry=plugin_registry,
config=config,
)
# Wait until the docks are ready to go
if not get_event_loop().is_running():
# If the event loop is running then we can assume that this setup
# will happen momentarily anyway. Plus, if an event loop is running
# then we cannot block. This is mainly to make sure everything
# starts up in a sensible fashion when we are in non-worker-mode.
block(event_dock.wait_until_ready(), timeout=2)
block(rpc_result_dock.wait_until_ready(), timeout=2)
return client
def _handle(self, args, config, plugin_registry: PluginRegistry):
command_utilities.setup_logging(override=getattr(
args, "log_level", None),
config=config)
bus_module, bus = command_utilities.import_bus(args)
# Convert only & skip into a list of features to enable
if args.only or args.skip:
if args.only:
features = args.only
else:
features = self.all_features
for skip_feature in args.skip or []:
if skip_feature in features:
features.remove(skip_feature)
elif os.environ.get("LIGHTBUS_FEATURES"):
features = csv_type(os.environ.get("LIGHTBUS_FEATURES"))
else:
features = ALL_FEATURES
bus.client.set_features(features)
# TODO: Move to lightbus.create()?
if args.schema:
if args.schema == "-":
# if '-' read from stdin
source = None
else:
source = args.schema
bus.schema.load_local(source)
restart_signals = (signal.SIGINT, signal.SIGTERM)
# Handle incoming signals
async def signal_handler():
# Stop handling signals now. If we receive the signal again
# let the process quit naturally
for signal_ in restart_signals:
asyncio.get_event_loop().remove_signal_handler(signal_)
logger.debug("Caught signal. Stopping main thread event loop")
bus.client.shutdown_server(exit_code=0)
for signal_ in restart_signals:
asyncio.get_event_loop().add_signal_handler(
signal_, lambda: asyncio.ensure_future(signal_handler()))
try:
block(plugin_registry.execute_hook("receive_args", args=args),
timeout=5)
bus.client.run_forever()
finally:
# Cleanup signal handlers
for signal_ in restart_signals:
asyncio.get_event_loop().remove_signal_handler(signal_)
if bus.client.exit_code:
sys.exit(bus.client.exit_code)
def call(self, *, bus_options: dict = None, **kwargs):
# Use a larger value of `rpc_timeout` because call_rpc_remote() should
# handle timeout
rpc_timeout = self.client.config.api(self.api_name).rpc_timeout * 1.5
return block(self.call_async(**kwargs, bus_options=bus_options),
timeout=rpc_timeout)
def close(self):
"""Close the bus client
This will cancel all tasks and close all transports/connections
"""
block(self.close_async())
def fire(self, *, bus_options: dict = None, **kwargs):
return block(
self.fire_async(**kwargs, bus_options=bus_options),
timeout=self.client.config.api(self.api_name).event_fire_timeout,
)
