def handle(self, args, config, fake_it=False):
self.setup_logging(args.log_level or "warning", config)
try:
import bpython
from bpython.curtsies import main as bpython_main
except ImportError: # pragma: no cover
print(
"Lightbus shell requires bpython. Run `pip install bpython` to install bpython."
)
exit(1)
return # noqa
logger = logging.getLogger("lightbus")
logger.setLevel(logging.WARNING)
bus_module, bus = self.import_bus(args)
objects = {k: v for k, v in lightbus.__dict__.items() if isclass(v)}
objects.update(bus=bus)
block(plugin_hook("receive_args", args=args), timeout=5)
# Ability to not start up the repl is useful for testing
if not fake_it:
bpython_main(
args=["-i", "-q"],
locals_=objects,
welcome_message=
"Welcome to the Lightbus shell. Use `bus` to access your bus.",
)
def _run_forever(self, consume_rpcs):
# Setup RPC consumption
consume_rpc_task = None
if consume_rpcs and registry.all():
consume_rpc_task = asyncio.ensure_future(self.consume_rpcs())
# Setup schema monitoring
monitor_task = asyncio.ensure_future(self.schema.monitor())
self.loop.add_signal_handler(signal.SIGINT, self.loop.stop)
self.loop.add_signal_handler(signal.SIGTERM, self.loop.stop)
try:
self._actually_run_forever()
logger.warning("Interrupt received. Shutting down...")
except KeyboardInterrupt:
logger.warning("Keyboard interrupt. Shutting down...")
# The loop has stopped, so we're shutting down
# Remove the signal handlers
self.loop.remove_signal_handler(signal.SIGINT)
self.loop.remove_signal_handler(signal.SIGTERM)
# Cancel the tasks we created above
block(cancel(consume_rpc_task, monitor_task), timeout=1)
def transaction_transport(aiopg_connection, aiopg_cursor, loop):
transport = TransactionalEventTransport(DebugEventTransport())
block(transport.set_connection(aiopg_connection, aiopg_cursor),
loop=loop,
timeout=1)
block(transport.database.migrate(), loop=loop, timeout=1)
return transport
def redis_config_file(loop, server, redis_client):
config = REDIS_BUS_CONFIG.format(host=server.tcp_address.host, port=server.tcp_address.port)
with NamedTemporaryFile() as f:
f.write(config.encode("utf8"))
f.flush()
yield f.name
block(redis_client.execute(b"CLIENT", b"KILL", b"TYPE", b"NORMAL"), loop=loop, timeout=1)
def _handle(self, args, config):
self.setup_logging(override=getattr(args, "log_level", None),
config=config)
bus_module, bus = self.import_bus(args)
# TODO: Move to lightbus.create()?
if args.schema:
if args.schema == "-":
# if '-' read from stdin
source = None
else:
source = args.schema
bus.schema.load_local(source)
before_server_start = getattr(bus_module, "before_server_start", None)
if before_server_start:
logger.debug("Calling {}.before_server_start() callback".format(
bus_module.__name__))
co = before_server_start(bus)
if iscoroutine(co):
block(co, asyncio.get_event_loop(), timeout=10)
block(plugin_hook("receive_args", args=args),
asyncio.get_event_loop(),
timeout=5)
if args.events_only:
bus.run_forever(consume_rpcs=False)
else:
bus.run_forever()
def handle(self, args, config):
try:
self._handle(args, config)
except Exception as e:
block(plugin_hook("exception", e=e),
asyncio.get_event_loop(),
timeout=5)
raise
def parse_args(args=None):
parser = argparse.ArgumentParser(
description='Lightbus management command.')
subparsers = parser.add_subparsers(help='Commands', dest='subcommand')
subparsers.required = True
parser_run = subparsers.add_parser(
'run',
help='Run Lightbus',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser_run_action_group = parser_run.add_mutually_exclusive_group()
parser_run_action_group.add_argument(
'--events-only',
help='Run Lightbus, but only listen for and handle events',
action='store_true')
parser_run_action_group.add_argument(
'--rpcs-only',
help='Run Lightbus, but only consume and handle RPCs',
action='store_true')
parser_run_action_group.add_argument(
'--import',
dest='imprt',
help='The Python module to import initially. If omited ')
parser_run.set_defaults(func=command_run)
parser_run_transport_group = parser_run.add_argument_group(
title='Transport options')
parser_run_transport_group.add_argument(
'--rpc-transport',
help='RPC transport class to use',
default='lightbus.RedisRpcTransport')
parser_run_transport_group.add_argument(
'--result-transport',
help='Result transport class to use',
default='lightbus.RedisResultTransport')
parser_run_transport_group.add_argument(
'--event-transport',
help='Event transport class to use',
default='lightbus.RedisEventTransport')
autoload_plugins()
block(plugin_hook('before_parse_args',
parser=parser,
subparsers=subparsers),
timeout=5)
# parser_run_connection_group = parser_run.add_argument_group(title='Connection options')
# parser_run_connection_group.add_argument(
# '--redis-url', help='URL to Redis server when using Redis-based transports', default='redis://localhost:6379/0'
# )
args = parser.parse_args(sys.argv[1:] if args is None else args)
block(plugin_hook('after_parse_args', args=args), timeout=5)
return args
def run_forever(self, *, loop=None, consume_rpcs=True, consume_events=True, plugins=None):
logger.info(LBullets(
"Lightbus getting ready to run. Brokers in use",
items={
"RPC transport": L(
'{}.{}',
self.rpc_transport.__module__, Bold(self.rpc_transport.__class__.__name__)
),
"Result transport": L(
'{}.{}', self.result_transport.__module__,
Bold(self.result_transport.__class__.__name__)
),
"Event transport": L(
'{}.{}', self.event_transport.__module__,
Bold(self.event_transport.__class__.__name__)
),
}
))
self.setup(plugins=plugins)
registry.add(LightbusStateApi())
registry.add(LightbusMetricsApi())
if consume_rpcs:
if registry.public():
logger.info(LBullets(
"APIs in registry ({})".format(len(registry.all())),
items=registry.all()
))
else:
if consume_events:
logger.warning(
"No APIs have been registered, lightbus may still receive events "
"but Lightbus will not handle any incoming RPCs"
)
else:
logger.error(
"No APIs have been registered, yet Lightbus has been configured to only "
"handle RPCs. There is therefore nothing for lightbus to do. Exiting."
)
return
block(handle_aio_exceptions(
plugin_hook('before_server_start', bus_client=self, loop=loop)
), timeout=5)
loop = loop or asyncio.get_event_loop()
self._run_forever(loop, consume_rpcs, consume_events)
loop.run_until_complete(handle_aio_exceptions(
plugin_hook('after_server_stopped', bus_client=self, loop=loop)
))
def parse_args(args=None):
parser = argparse.ArgumentParser(
description="Lightbus management command.")
parser.add_argument(
"--service-name",
"-s",
help="Name of service in which this process resides. YOU SHOULD "
"LIKELY SET THIS IN PRODUCTION. Can also be set using the "
"LIGHTBUS_SERVICE_NAME environment. Will default to a random string.",
)
parser.add_argument(
"--process-name",
"-p",
help=
"A unique name of this process within the service. Can also be set using the "
"LIGHTBUS_PROCESS_NAME environment. Will default to a random string.",
)
parser.add_argument("--config",
dest="config_file",
help="Config file to load, JSON or YAML",
metavar="FILE")
parser.add_argument(
"--log-level",
help="Set the log level. Overrides any value set in config. "
"One of debug, info, warning, critical, exception.",
metavar="LOG_LEVEL",
)
subparsers = parser.add_subparsers(help="Commands", dest="subcommand")
subparsers.required = True
lightbus.commands.run.Command().setup(parser, subparsers)
lightbus.commands.shell.Command().setup(parser, subparsers)
lightbus.commands.dump_schema.Command().setup(parser, subparsers)
lightbus.commands.dump_schema.Command().setup(parser, subparsers)
lightbus.commands.dump_config_schema.Command().setup(parser, subparsers)
autoload_plugins(config=Config.load_dict({}))
loop = get_event_loop()
block(plugin_hook("before_parse_args",
parser=parser,
subparsers=subparsers),
loop,
timeout=5)
args = parser.parse_args(sys.argv[1:] if args is None else args)
# Note that we don't have an after_parse_args plugin hook. Instead we use the receive_args
# hook which is called once we have instantiated our plugins
return args
def aiopg_cursor(aiopg_connection, loop, cursor_factory):
cursor = block(aiopg_connection.cursor(cursor_factory=cursor_factory),
loop=loop,
timeout=1)
block(cursor.execute("BEGIN -- aiopg_cursor"), loop=loop, timeout=1)
block(cursor.execute("DROP TABLE IF EXISTS lightbus_processed_events"),
loop=loop,
timeout=1)
block(cursor.execute("DROP TABLE IF EXISTS lightbus_event_outbox"),
loop=loop,
timeout=1)
block(cursor.execute("COMMIT -- aiopg_cursor"), loop=loop, timeout=1)
return cursor
def aiopg_connection(pg_kwargs, loop):
import aiopg
connection = block(aiopg.connect(loop=loop, **pg_kwargs),
loop=loop,
timeout=2)
yield connection
connection.close()
def run_forever(self, *, consume_rpcs=True):
registry.add(LightbusStateApi())
registry.add(LightbusMetricsApi())
if consume_rpcs:
logger.info(
LBullets("APIs in registry ({})".format(len(registry.all())),
items=registry.names()))
block(self._plugin_hook("before_server_start"), timeout=5)
self._run_forever(consume_rpcs)
self.loop.run_until_complete(self._plugin_hook("after_server_stopped"))
# Close the bus (which will in turn close the transports)
self.close()
# See if we've set the exit code on the event loop
if hasattr(self.loop, "lightbus_exit_code"):
raise SystemExit(self.loop.lightbus_exit_code)
def __call__(self, request):
connection = connections["default"]
if connection.in_atomic_block:
start_transaction = False
elif connection.autocommit:
start_transaction = True
else:
start_transaction = None
lightbus_transaction_context = lightbus_set_database(
self.bus, connection)
block(lightbus_transaction_context.__aenter__(), self.loop, timeout=5)
response = self.get_response(request)
if 500 <= response.status_code < 600:
block(lightbus_transaction_context.__aexit__(True, True, True),
self.loop,
timeout=5)
else:
block(lightbus_transaction_context.__aexit__(None, None, None),
self.loop,
timeout=5)
return response
def create(*args, **kwargs) -> BusPath:
"""
Create a new bus instance which can be used to access the bus.
Typically this will be used as follows:
import lightbus
bus = lightbus.create()
See Also: This function is a wrapper around `create_async()`, see `create_async()`
for a list of arguments
"""
return block(create_async(*args, **kwargs), timeout=5)
def call(self, *, bus_options=None, **kwargs):
# Use a larger value of `rpc_timeout` because call_rpc_remote() should
# handle timeout
rpc_timeout = self.client.config.api(self.api_name).rpc_timeout * 1.5
return block(self.call_async(**kwargs, bus_options=bus_options),
timeout=rpc_timeout)
def __enter__(self):
loop = self.loop or get_event_loop()
block(self.__aenter__(), loop, timeout=5)
def test_table(aiopg_cursor, loop):
block(aiopg_cursor.execute("BEGIN -- test_table (setup)"),
loop=loop,
timeout=1)
block(aiopg_cursor.execute("DROP TABLE IF EXISTS test_table"),
loop=loop,
timeout=1)
block(aiopg_cursor.execute("CREATE TABLE test_table (pk VARCHAR(100))"),
loop=loop,
timeout=1)
block(aiopg_cursor.execute("COMMIT -- test_table (setup)"),
loop=loop,
timeout=1)
class TestTable(object):
async def total_rows(self):
await aiopg_cursor.execute("SELECT COUNT(*) FROM test_table")
return (await aiopg_cursor.fetchone())[0]
yield TestTable()
block(aiopg_cursor.execute("BEGIN -- test_table (tear down)"),
loop=loop,
timeout=1)
block(aiopg_cursor.execute("DROP TABLE test_table"), loop=loop, timeout=1)
block(aiopg_cursor.execute("COMMIT -- test_table (tear down)"),
loop=loop,
timeout=1)
def __exit__(self, exc_type, exc_val, exc_tb):
block(self.__aexit__(exc_type, exc_val, exc_tb), timeout=5)
def migrate(self):
# TODO: This needs to be a core lightbus feature somehow
with connections["default"].cursor() as cursor:
block(DbApiConnection(connections["default"], cursor).migrate(),
timeout=5)
def close(self):
block(self.close_async(), timeout=5)
def __exit__(self, exc_type, exc_val, exc_tb):
loop = self.loop or get_event_loop()
block(self.__aexit__(exc_type, exc_val, exc_tb), loop, timeout=5)
def fire(self, *, bus_options: dict=None, **kwargs):
return block(self.fire_async(**kwargs, bus_options=bus_options), timeout=5)
def close(self):
block(self.close_async(), loop=self.loop, timeout=5)
def __enter__(self):
block(self.__aenter__(), timeout=5)
def call(self, *, bus_options=None, **kwargs):
return block(self.call_async(**kwargs, bus_options=bus_options), timeout=1)
def listen(self, listener, *, bus_options: dict = None):
return block(
self.listen_async(listener, bus_options=bus_options),
timeout=self.client.config.api(
self.api_name).event_listener_setup_timeout,
)
def handle(self, args, config):
try:
self._handle(args, config)
except Exception as e:
block(plugin_hook("exception", e=e), timeout=5)
raise
def setup(self, plugins: dict = None):
block(self.setup_async(plugins), timeout=5)
def fire(self, *, bus_options: dict = None, **kwargs):
return block(
self.fire_async(**kwargs, bus_options=bus_options),
timeout=self.client.config.api(self.api_name).event_fire_timeout,
)
def listen(self, listener, *, bus_options: dict=None):
return block(self.listen_async(listener, bus_options=bus_options), timeout=5)
