def __enter__(self):
"""Start of a context where all the bus' transports have been replaced with mocks"""
# Mutable structures which we will use this to get the mocked data into
# the transports
# RPC
mock_responses = {}
# Events
mock_events = set()
# Create our transport classes, into which we inject our mutable structures
TestRpcTransport = make_test_rpc_transport()
TestResultTransport = make_test_result_transport(mock_responses)
TestEventTransport = make_test_event_transport(mock_events)
TestSchemaTransport = make_test_schema_transport()
new_registry = TransportRegistry()
new_registry.set_schema_transport(TestSchemaTransport,
TestSchemaTransport.Config(),
self.bus.client.config)
self.old_transport_registry = self.bus.client.transport_registry
for api_name, entry in self.old_transport_registry._registry.items():
new_registry.set_rpc_transport(api_name, TestRpcTransport,
TestRpcTransport.Config(),
self.bus.client.config)
new_registry.set_result_transport(
api_name,
TestResultTransport,
TestResultTransport.Config(
require_mocking=self.require_mocking),
self.bus.client.config,
)
new_registry.set_event_transport(
api_name,
TestEventTransport,
TestEventTransport.Config(
require_mocking=self.require_mocking),
self.bus.client.config,
)
# The docs are only available on the bus client during testing
self.bus.client.event_dock.transport_registry = new_registry
self.bus.client.rpc_result_dock.transport_registry = new_registry
queue_mocker = BusQueueMockerContext(client=self.bus.client)
bus_with_mocked_queues = queue_mocker.__enter__()
self.stack.append(queue_mocker)
return MockResult(bus_with_mocked_queues,
mock_responses=mock_responses,
mock_events=mock_events)
def log_welcome_message(
logger: Logger,
transport_registry: TransportRegistry,
schema: "Schema",
plugin_registry: PluginRegistry,
config: Config,
):
"""Show the worker-startup welcome message
"""
logger.info(
LBullets(
"Lightbus is setting up",
items={
"service_name (set with -s or LIGHTBUS_SERVICE_NAME)":
Bold(config.service_name),
"process_name (with with -p or LIGHTBUS_PROCESS_NAME)":
Bold(config.process_name),
},
))
# Log the transport information
rpc_transport = transport_registry.get_rpc_transport("default",
default=None)
result_transport = transport_registry.get_result_transport("default",
default=None)
event_transport = transport_registry.get_event_transport("default",
default=None)
log_transport_information(rpc_transport, result_transport, event_transport,
schema.schema_transport, logger)
if plugin_registry._plugins:
logger.info(
LBullets(
"Loaded the following plugins ({})".format(
len(plugin_registry._plugins)),
items=plugin_registry._plugins,
))
else:
logger.info("No plugins loaded")
def test_transport_registry_get_does_not_exist_default():
registry = TransportRegistry()
with pytest.raises(TransportNotFound):
assert not registry.get_rpc_transport("default")
with pytest.raises(TransportNotFound):
assert not registry.get_result_transport("default")
with pytest.raises(TransportNotFound):
assert not registry.get_event_transport("default")
with pytest.raises(TransportNotFound):
assert not registry.get_schema_transport()
def test_transport_registry_load_config(redis_default_config):
registry = TransportRegistry().load_config(redis_default_config)
assert registry.get_rpc_transport(
"default").transport_class == RedisRpcTransport
assert registry.get_result_transport(
"default").transport_class == RedisResultTransport
assert registry.get_event_transport(
"default").transport_class == RedisEventTransport
assert registry.get_schema_transport(
"default").transport_class == RedisSchemaTransport
def test_transport_registry_get_specific_api(redis_other_config):
registry = TransportRegistry().load_config(redis_other_config)
assert registry.get_rpc_transport(
"other").transport_class == RedisRpcTransport
assert registry.get_result_transport(
"other").transport_class == RedisResultTransport
assert registry.get_event_transport(
"other").transport_class == RedisEventTransport
assert registry.get_schema_transport(
"other").transport_class == RedisSchemaTransport
def test_get_all_transports(redis_default_config):
registry = TransportRegistry().load_config(redis_default_config)
# Is equal to the default transport, so will not be included
# in the get_all_transports() return value
registry.set_event_transport("another", RedisEventTransport,
RedisEventTransport.Config(),
Config.default())
registry.set_event_transport("foo", DebugEventTransport,
DebugEventTransport.Config(),
Config.default())
# redis rpc + redis result + redis event + foo debug transport (above) = 4
assert len(registry.get_all_transports()) == 4
def test_has_schema_transport(redis_no_default_config):
registry = TransportRegistry().load_config(redis_no_default_config)
assert registry.has_schema_transport()
registry.schema_transport = None
assert not registry.has_schema_transport()
def test_has_event_transport(redis_no_default_config):
registry = TransportRegistry().load_config(redis_no_default_config)
assert registry.has_event_transport("other")
assert not registry.has_event_transport("foo")
def test_transport_registry_get_event_transports(redis_default_config):
registry = TransportRegistry().load_config(redis_default_config)
# See comment in test_transport_registry_get_rpc_transports
registry.set_event_transport("redis1", RedisEventTransport,
RedisEventTransport.Config(batch_size=99),
Config.default())
registry.set_event_transport("redis2", RedisEventTransport,
RedisEventTransport.Config(batch_size=99),
Config.default())
registry.set_event_transport("debug1", DebugEventTransport,
DebugEventTransport.Config(),
Config.default())
registry.set_event_transport("debug2", DebugEventTransport,
DebugEventTransport.Config(),
Config.default())
transports = registry.get_event_transports(
["default", "foo", "bar", "redis1", "redis2", "debug1", "debug2"])
default_transport = registry.get_event_transport("default")
redis_transport = registry.get_event_transport("redis1")
debug_transport = registry.get_event_transport("debug1")
assert set(transports[default_transport]) == {"default", "foo", "bar"}
assert set(transports[debug_transport]) == {"debug1", "debug2"}
assert set(transports[redis_transport]) == {"redis1", "redis2"}
def test_transport_registry_get_rpc_transports(redis_default_config):
registry = TransportRegistry().load_config(redis_default_config)
# Note how we set a config value below. We do this because
# we need this transport to appear different from the default
# transport for the purposes of this test. Also note that these
# are not actual transports, but transport pools which wrap transports.
# We are therefore actually comparing transport pools. Transport
# pools are considered equal if they have the same transport class
# and config. It is for this reason we modify the config below.
registry.set_rpc_transport("redis1", RedisRpcTransport,
RedisRpcTransport.Config(rpc_timeout=99),
Config.default())
registry.set_rpc_transport("redis2", RedisRpcTransport,
RedisRpcTransport.Config(rpc_timeout=99),
Config.default())
registry.set_rpc_transport("debug1", DebugRpcTransport,
DebugRpcTransport.Config(), Config.default())
registry.set_rpc_transport("debug2", DebugRpcTransport,
DebugRpcTransport.Config(), Config.default())
transport_pools = registry.get_rpc_transports(
["default", "foo", "bar", "redis1", "redis2", "debug1", "debug2"])
default_transport = registry.get_rpc_transport("default")
redis_transport = registry.get_rpc_transport("redis1")
debug_transport = registry.get_rpc_transport("debug1")
assert set(transport_pools[default_transport]) == {"default", "foo", "bar"}
assert set(transport_pools[debug_transport]) == {"debug1", "debug2"}
assert set(transport_pools[redis_transport]) == {"redis1", "redis2"}
def test_transport_registry_get_does_not_exist_default_default_value():
registry = TransportRegistry()
assert registry.get_rpc_transport("default", default=None) is None
assert registry.get_result_transport("default", default=None) is None
assert registry.get_event_transport("default", default=None) is None
assert registry.get_schema_transport(default=None) is None
def create(
config: Union[dict, RootConfig] = None,
*,
config_file: str = None,
service_name: str = None,
process_name: str = None,
features: List[Union[Feature, str]] = ALL_FEATURES,
client_class: Type[BusClient] = BusClient,
node_class: Type[BusPath] = BusPath,
plugins=None,
flask: bool = False,
**kwargs,
) -> BusPath:
"""
Create a new bus instance which can be used to access the bus.
Typically this will be used as follows:
import lightbus
bus = lightbus.create()
This will be a `BusPath` instance. If you wish to access the lower
level `BusClient` you can do so via `bus.client`.
Args:
config (dict, Config): The config object or dictionary to load
config_file (str): The path to a config file to load (should end in .json or .yaml)
service_name (str): The name of this service - will be used when creating event consumer groups
process_name (str): The unique name of this process - used when retrieving unprocessed events following a crash
client_class (Type[BusClient]): The class from which the bus client will be instantiated
node_class (BusPath): The class from which the bus path will be instantiated
plugins (list): A list of plugin instances to load
flask (bool): Are we using flask? If so we will make sure we don't start lightbus in the reloader process
**kwargs (): Any additional instantiation arguments to be passed to `client_class`.
Returns: BusPath
"""
if flask:
in_flask_server = sys.argv[0].endswith("flask") and "run" in sys.argv
if in_flask_server and os.environ.get("WERKZEUG_RUN_MAIN", "").lower() != "true":
# Flask has a reloader process that shouldn't start a lightbus client
return
# Ensure an event loop exists, as creating InternalQueue
# objects requires that we have one.
get_event_loop()
# If were are running via the Lightbus CLI then we may have
# some command line arguments we need to apply.
# pylint: disable=cyclic-import,import-outside-toplevel
from lightbus.commands import COMMAND_PARSED_ARGS
config_file = COMMAND_PARSED_ARGS.get("config_file", None) or config_file
service_name = COMMAND_PARSED_ARGS.get("service_name", None) or service_name
process_name = COMMAND_PARSED_ARGS.get("process_name", None) or process_name
if config is None:
config = load_config(
from_file=config_file, service_name=service_name, process_name=process_name
)
if isinstance(config, Mapping):
config = Config.load_dict(config or {})
elif isinstance(config, RootConfig):
config = Config(config)
transport_registry = kwargs.pop("transport_registry", None) or TransportRegistry().load_config(
config
)
schema = Schema(
schema_transport=transport_registry.get_schema_transport(),
max_age_seconds=config.bus().schema.ttl,
human_readable=config.bus().schema.human_readable,
)
error_queue: ErrorQueueType = InternalQueue()
# Plugin registry
plugin_registry = PluginRegistry()
if plugins is None:
logger.debug("Auto-loading any installed Lightbus plugins...")
plugin_registry.autoload_plugins(config)
else:
logger.debug("Loading explicitly specified Lightbus plugins....")
plugin_registry.set_plugins(plugins)
# Hook registry
hook_registry = HookRegistry(
error_queue=error_queue, execute_plugin_hooks=plugin_registry.execute_hook
)
# API registry
api_registry = ApiRegistry()
api_registry.add(LightbusStateApi())
api_registry.add(LightbusMetricsApi())
events_queue_client_to_dock = InternalQueue()
events_queue_dock_to_client = InternalQueue()
event_client = EventClient(
api_registry=api_registry,
hook_registry=hook_registry,
config=config,
schema=schema,
error_queue=error_queue,
consume_from=events_queue_dock_to_client,
produce_to=events_queue_client_to_dock,
)
event_dock = EventDock(
transport_registry=transport_registry,
api_registry=api_registry,
config=config,
error_queue=error_queue,
consume_from=events_queue_client_to_dock,
produce_to=events_queue_dock_to_client,
)
rpcs_queue_client_to_dock = InternalQueue()
rpcs_queue_dock_to_client = InternalQueue()
rpc_result_client = RpcResultClient(
api_registry=api_registry,
hook_registry=hook_registry,
config=config,
schema=schema,
error_queue=error_queue,
consume_from=rpcs_queue_dock_to_client,
produce_to=rpcs_queue_client_to_dock,
)
rpc_result_dock = RpcResultDock(
transport_registry=transport_registry,
api_registry=api_registry,
config=config,
error_queue=error_queue,
consume_from=rpcs_queue_client_to_dock,
produce_to=rpcs_queue_dock_to_client,
)
client = client_class(
config=config,
hook_registry=hook_registry,
plugin_registry=plugin_registry,
features=features,
schema=schema,
api_registry=api_registry,
event_client=event_client,
rpc_result_client=rpc_result_client,
error_queue=error_queue,
transport_registry=transport_registry,
**kwargs,
)
# Pass the client to any hooks
# (use a weakref to prevent circular references)
hook_registry.set_extra_parameter("client", weakref.proxy(client))
# We don't do this normally as the docks do not need to be
# accessed directly, but this is useful in testing
# TODO: Testing flag removed, but these are only needed in testing.
# Perhaps wrap them up in a way that makes this obvious
client.event_dock = event_dock
client.rpc_result_dock = rpc_result_dock
log_welcome_message(
logger=logger,
transport_registry=transport_registry,
schema=schema,
plugin_registry=plugin_registry,
config=config,
)
return node_class(name="", parent=None, client=client)
