def make_custom_object_schema(type_, property_names=None):
"""Convert a named tuple into a JSON schema
While a lot of conversations happen in `python_type_to_json_schemas()`, conversion of
annotated classes is a little more involved. It therefore gets its own function.
While this is mainly used for named tuples within lightbus, it can also be used for
general classes.
"""
if property_names is None:
property_names = [
p for p in set(list(type_.__annotations__.keys()) + dir(type_)) if p[0] != "_"
]
properties = {}
required = []
for property_name in property_names:
default = empty
if callable(getattr(type_, property_name, None)):
# is a method
continue
if issubclass(type_, tuple):
# namedtuple
if hasattr(type_, "_field_defaults"):
default = type_._field_defaults.get(property_name, empty)
else:
default = getattr(type_, property_name, empty)
if callable(default):
default = empty
if hasattr(type_, "__annotations__"):
properties[property_name] = wrap_with_one_of(
python_type_to_json_schemas(type_.__annotations__.get(property_name, None))
)
elif default is not empty:
properties[property_name] = wrap_with_one_of(python_type_to_json_schemas(type(default)))
else:
properties[property_name] = {}
if default is empty:
required.append(property_name)
else:
properties[property_name]["default"] = deform_to_bus(default)
schema = {
"type": "object",
"title": type_.__name__,
"properties": properties,
"required": required,
"additionalProperties": False,
}
# required key should not be present if it is empty
if not schema["required"]:
schema.pop("required")
return schema
async def _consume_rpcs_with_transport(self,
rpc_transport: RpcTransport,
apis: List[Api] = None):
rpc_messages = await rpc_transport.consume_rpcs(apis)
for rpc_message in rpc_messages:
self._validate(rpc_message, "incoming")
await self._plugin_hook("before_rpc_execution",
rpc_message=rpc_message)
try:
result = await self.call_rpc_local(
api_name=rpc_message.api_name,
name=rpc_message.procedure_name,
kwargs=rpc_message.kwargs,
)
except SuddenDeathException:
# Used to simulate message failure for testing
pass
else:
result = deform_to_bus(result)
result_message = ResultMessage(result=result,
rpc_message_id=rpc_message.id)
await self._plugin_hook("after_rpc_execution",
rpc_message=rpc_message,
result_message=result_message)
self._validate(
result_message,
"outgoing",
api_name=rpc_message.api_name,
procedure_name=rpc_message.procedure_name,
)
await self.send_result(rpc_message=rpc_message,
result_message=result_message)
def test_deform_to_bus(test_input, expected):
value_before = copy(test_input)
actual = deform_to_bus(test_input)
assert actual == expected
assert type(actual) == type(expected)
# Test input value has not been mutated
assert value_before == test_input
def make_custom_object_schema(type_, property_names=None):
"""Convert a named tuple into a JSON schema
While a lot of conversations happen in `python_type_to_json_schemas()`, conversion of
annotated classes is a little more involved. It therefore gets its own function.
While this is mainly used for named tuples within lightbus, it can also be used for
general classes.
"""
if property_names is None:
# Use typing.get_type_hints() rather than `__annotations__`, as this will resolve
# forward references
property_names = [
p for p in set(list(get_type_hints(type_).keys()) + dir(type_))
if p[0] != "_"
]
properties = {}
required = []
for property_name in property_names:
property_value = getattr(type_, property_name, None)
if callable(property_value):
# is a method or dynamic property
continue
default = get_property_default(type_, property_name)
type_hints = get_type_hints(type_)
if hasattr(type_, "__annotations__") and property_name in type_hints:
# Use typing.get_type_hints() rather than `__annotations__`, as this will resolve
# forward references
properties[property_name] = wrap_with_any_of(
annotation_to_json_schemas(
annotation=type_hints[property_name], default=default))
elif default is not empty:
properties[property_name] = wrap_with_any_of(
python_type_to_json_schemas(type(default)))
else:
properties[property_name] = {}
if default is empty:
required.append(property_name)
else:
properties[property_name]["default"] = deform_to_bus(default)
schema = {
"type": "object",
"title": type_.__name__,
"properties": properties,
"required": required,
"additionalProperties": False,
}
# required key should not be present if it is empty
if not schema["required"]:
schema.pop("required")
return schema
async def before_rpc_call(self, *, rpc_message: RpcMessage,
client: "lightbus.client.BusClient"):
await self.send_event(
client,
"rpc_call_sent",
id=rpc_message.id,
api_name=rpc_message.api_name,
procedure_name=rpc_message.procedure_name,
kwargs=deform_to_bus(rpc_message.kwargs),
)
async def after_event_sent(self, *, event_message: EventMessage,
client: "lightbus.client.BusClient"):
await self.send_event(
client,
"event_fired",
event_id="event_id",
api_name=event_message.api_name,
event_name=event_message.event_name,
kwargs=deform_to_bus(event_message.kwargs),
)
async def before_event_execution(self, *, event_message: EventMessage,
client: "lightbus.client.BusClient"):
await self.send_event(
client,
"event_received",
event_id="event_id",
api_name=event_message.api_name,
event_name=event_message.event_name,
kwargs=deform_to_bus(event_message.kwargs),
)
def send_event(self, client: "lightbus.client.BusClient", event_name_, **kwargs) -> Coroutine:
"""Send an event to the bus
"""
kwargs.setdefault("timestamp", datetime.utcnow().timestamp())
kwargs.setdefault("service_name", self.service_name)
kwargs.setdefault("process_name", self.process_name)
kwargs = deform_to_bus(kwargs)
return client.fire_event(
api_name="internal.metrics", name=event_name_, kwargs=kwargs, options={}
)
def config_as_json_schema() -> dict:
"""Get the configuration structure as a json schema"""
from .structure import RootConfig
schema, = python_type_to_json_schemas(RootConfig)
# Some of the default values will still be python types,
# so let's use deform_to_bus to turn them into something
# that'll be json safe
schema = deform_to_bus(schema)
schema["$schema"] = SCHEMA_URI
return schema
async def fire_event(self,
api_name,
name,
kwargs: dict = None,
options: dict = None):
kwargs = kwargs or {}
try:
api = registry.get(api_name)
except UnknownApi:
raise UnknownApi(
"Lightbus tried to fire the event {api_name}.{name}, but could not find API {api_name} in the "
"registry. An API being in the registry implies you are an authority on that API. Therefore, "
"Lightbus requires the API to be in the registry as it is a bad idea to fire "
"events on behalf of remote APIs. However, this could also be caused by a typo in the "
"API name or event name, or be because the API class has not been "
"imported. ".format(**locals()))
self._validate_name(api_name, "event", name)
try:
event = api.get_event(name)
except EventNotFound:
raise EventNotFound(
"Lightbus tried to fire the event {api_name}.{name}, but the API {api_name} does not "
"seem to contain an event named {name}. You may need to define the event, you "
"may also be using the incorrect API. Also check for typos.".
format(**locals()))
if set(kwargs.keys()) != _parameter_names(event.parameters):
raise InvalidEventArguments(
"Invalid event arguments supplied when firing event. Attempted to fire event with "
"{} arguments: {}. Event expected {}: {}".format(
len(kwargs),
sorted(kwargs.keys()),
len(event.parameters),
sorted(_parameter_names(event.parameters)),
))
kwargs = deform_to_bus(kwargs)
event_message = EventMessage(api_name=api.meta.name,
event_name=name,
kwargs=kwargs)
self._validate(event_message, "outgoing")
event_transport = self.transport_registry.get_event_transport(api_name)
await self._plugin_hook("before_event_sent",
event_message=event_message)
logger.info(
L("π€ Sending event {}.{}".format(Bold(api_name), Bold(name))))
await event_transport.send_event(event_message, options=options)
await self._plugin_hook("after_event_sent",
event_message=event_message)
async def fire_event(self, api_name, name, kwargs: dict = None, options: dict = None):
kwargs = kwargs or {}
try:
api = self.api_registry.get(api_name)
except UnknownApi:
raise UnknownApi(
"Lightbus tried to fire the event {api_name}.{name}, but no API named {api_name} was found in the "
"registry. An API being in the registry implies you are an authority on that API. Therefore, "
"Lightbus requires the API to be in the registry as it is a bad idea to fire "
"events on behalf of remote APIs. However, this could also be caused by a typo in the "
"API name or event name, or be because the API class has not been "
"registered using bus.client.register_api(). ".format(**locals())
)
validate_event_or_rpc_name(api_name, "event", name)
try:
event = api.get_event(name)
except EventNotFound:
raise EventNotFound(
"Lightbus tried to fire the event {api_name}.{name}, but the API {api_name} does not "
"seem to contain an event named {name}. You may need to define the event, you "
"may also be using the incorrect API. Also check for typos.".format(**locals())
)
parameter_names = {p.name if isinstance(p, Parameter) else p for p in event.parameters}
if set(kwargs.keys()) != parameter_names:
raise InvalidEventArguments(
"Invalid event arguments supplied when firing event. Attempted to fire event with "
"{} arguments: {}. Event expected {}: {}".format(
len(kwargs),
sorted(kwargs.keys()),
len(event.parameters),
sorted(parameter_names),
)
)
kwargs = deform_to_bus(kwargs)
event_message = EventMessage(
api_name=api.meta.name, event_name=name, kwargs=kwargs, version=api.meta.version
)
validate_outgoing(self.config, self.schema, event_message)
await self.hook_registry.execute("before_event_sent", event_message=event_message)
logger.info(L("Γ°ΕΈβΒ€ Sending event {}.{}".format(Bold(api_name), Bold(name))))
await self.producer.send(SendEventCommand(message=event_message, options=options)).wait()
await self.hook_registry.execute("after_event_sent", event_message=event_message)
async def after_event_execution(
self, *, event_message: EventMessage, client: "lightbus.client.BusClient"
):
if event_message.api_name == "internal.metrics":
return
await self.send_event(
client,
"event_processed",
event_id="event_id",
api_name=event_message.api_name,
event_name=event_message.event_name,
kwargs=deform_to_bus(event_message.kwargs),
)
async def after_rpc_execution(
self,
*,
rpc_message: RpcMessage,
result_message: ResultMessage,
client: "lightbus.client.BusClient",
):
await self.send_event(
client,
"rpc_response_sent",
id=rpc_message.id,
api_name=rpc_message.api_name,
procedure_name=rpc_message.procedure_name,
result=deform_to_bus(result_message.result),
)
def send_event(self, client, event_name_, **kwargs) -> Coroutine:
"""Send an event to the bus
Note that we bypass using BusClient directly, otherwise we would trigger this
plugin again thereby causing an infinite loop.
"""
kwargs.setdefault("timestamp", datetime.utcnow().timestamp())
kwargs.setdefault("service_name", self.service_name)
kwargs.setdefault("process_name", self.process_name)
kwargs = deform_to_bus(kwargs)
event_transport = client.transport_registry.get_event_transport("internal.metrics")
return event_transport.send_event(
EventMessage(api_name="internal.metrics", event_name=event_name_, kwargs=kwargs),
options={},
bus_client=client,
)
async def _consume_rpcs_with_transport(
self, rpc_transport: RpcTransport, apis: List[Api] = None
):
while True:
try:
rpc_messages = await rpc_transport.consume_rpcs(apis, bus_client=self)
except TransportIsClosed:
return
for rpc_message in rpc_messages:
self._validate(rpc_message, "incoming")
await self._execute_hook("before_rpc_execution", rpc_message=rpc_message)
try:
result = await self.call_rpc_local(
api_name=rpc_message.api_name,
name=rpc_message.procedure_name,
kwargs=rpc_message.kwargs,
)
except SuddenDeathException:
# Used to simulate message failure for testing
return
except CancelledError:
raise
except Exception as e:
result = e
else:
result = deform_to_bus(result)
result_message = ResultMessage(result=result, rpc_message_id=rpc_message.id)
await self._execute_hook(
"after_rpc_execution", rpc_message=rpc_message, result_message=result_message
)
self._validate(
result_message,
"outgoing",
api_name=rpc_message.api_name,
procedure_name=rpc_message.procedure_name,
)
await self.send_result(rpc_message=rpc_message, result_message=result_message)
async def handle_execute_rpc(self, command: commands.ExecuteRpcCommand):
await self.schema.ensure_loaded_from_bus()
validate_incoming(self.config, self.schema, command.message)
await self.hook_registry.execute("before_rpc_execution", rpc_message=command.message)
try:
result = await self._call_rpc_local(
api_name=command.message.api_name,
name=command.message.procedure_name,
kwargs=command.message.kwargs,
)
except SuddenDeathException:
# Used to simulate message failure for testing
return
except asyncio.CancelledError:
raise
except Exception as e:
result = e
else:
result = deform_to_bus(result)
result_message = ResultMessage(
result=result,
rpc_message_id=command.message.id,
api_name=command.message.api_name,
procedure_name=command.message.procedure_name,
)
await self.hook_registry.execute(
"after_rpc_execution", rpc_message=command.message, result_message=result_message
)
if not result_message.error:
validate_outgoing(self.config, self.schema, result_message)
await self.producer.send(
commands.SendResultCommand(message=result_message, rpc_message=command.message)
).wait()
def test_deform_to_bus_custom_object():
obj = CustomClass()
with pytest.raises(DeformError):
assert deform_to_bus(obj) == obj
def test_deform_to_bus(test_input, expected):
actual = deform_to_bus(test_input)
assert actual == expected
assert type(actual) == type(expected)
async def call_rpc_remote(
self, api_name: str, name: str, kwargs: dict = frozendict(), options: dict = frozendict()
):
""" Perform an RPC call
Call an RPC and return the result.
"""
kwargs = deform_to_bus(kwargs)
rpc_message = RpcMessage(api_name=api_name, procedure_name=name, kwargs=kwargs)
validate_event_or_rpc_name(api_name, "rpc", name)
logger.info("π Calling remote RPC {}.{}".format(Bold(api_name), Bold(name)))
start_time = time.time()
validate_outgoing(self.config, self.schema, rpc_message)
await self.hook_registry.execute("before_rpc_call", rpc_message=rpc_message)
result_queue = InternalQueue()
# Send the RPC
await self.producer.send(
commands.CallRpcCommand(message=rpc_message, options=options)
).wait()
# Start a listener which will wait for results
await self.producer.send(
commands.ReceiveResultCommand(
message=rpc_message, destination_queue=result_queue, options=options
)
).wait()
# Wait for the result from the listener we started.
# The RpcResultDock will handle timeouts
result = await bail_on_error(self.error_queue, result_queue.get())
call_time = time.time() - start_time
try:
if isinstance(result, Exception):
raise result
except asyncio.TimeoutError:
raise LightbusTimeout(
f"Timeout when calling RPC {rpc_message.canonical_name} after waiting for {human_time(call_time)}. "
f"It is possible no Lightbus process is serving this API, or perhaps it is taking "
f"too long to process the request. In which case consider raising the 'rpc_timeout' "
f"config option."
) from None
else:
assert isinstance(result, ResultMessage)
result_message = result
await self.hook_registry.execute(
"after_rpc_call", rpc_message=rpc_message, result_message=result_message
)
if not result_message.error:
logger.info(
L(
"π Remote call of {} completed in {}",
Bold(rpc_message.canonical_name),
human_time(call_time),
)
)
else:
logger.warning(
L(
"β‘ Error during remote call of RPC {}. Took {}: {}",
Bold(rpc_message.canonical_name),
human_time(call_time),
result_message.result,
)
)
raise LightbusWorkerError(
"Error while calling {}: {}\nRemote stack trace:\n{}".format(
rpc_message.canonical_name, result_message.result, result_message.trace
)
)
validate_incoming(self.config, self.schema, result_message)
return result_message.result
async def call_rpc_remote(self,
api_name: str,
name: str,
kwargs: dict = frozendict(),
options: dict = frozendict()):
rpc_transport = self.transport_registry.get_rpc_transport(api_name)
result_transport = self.transport_registry.get_result_transport(
api_name)
kwargs = deform_to_bus(kwargs)
rpc_message = RpcMessage(api_name=api_name,
procedure_name=name,
kwargs=kwargs)
return_path = result_transport.get_return_path(rpc_message)
rpc_message.return_path = return_path
options = options or {}
timeout = options.get("timeout", self.config.api(api_name).rpc_timeout)
self._validate_name(api_name, "rpc", name)
logger.info("π Calling remote RPC {}.{}".format(
Bold(api_name), Bold(name)))
start_time = time.time()
# TODO: It is possible that the RPC will be called before we start waiting for the response. This is bad.
self._validate(rpc_message, "outgoing")
future = asyncio.gather(
self.receive_result(rpc_message, return_path, options=options),
rpc_transport.call_rpc(rpc_message, options=options),
)
await self._plugin_hook("before_rpc_call", rpc_message=rpc_message)
try:
result_message, _ = await asyncio.wait_for(future, timeout=timeout)
future.result()
except asyncio.TimeoutError:
# Allow the future to finish, as per https://bugs.python.org/issue29432
try:
await future
future.result()
except CancelledError:
pass
# TODO: Remove RPC from queue. Perhaps add a RpcBackend.cancel() method. Optional,
# as not all backends will support it. No point processing calls which have timed out.
raise LightbusTimeout(
f"Timeout when calling RPC {rpc_message.canonical_name} after {timeout} seconds. "
f"It is possible no Lightbus process is serving this API, or perhaps it is taking "
f"too long to process the request. In which case consider raising the 'rpc_timeout' "
f"config option.") from None
await self._plugin_hook("after_rpc_call",
rpc_message=rpc_message,
result_message=result_message)
if not result_message.error:
logger.info(
L(
"π Remote call of {} completed in {}",
Bold(rpc_message.canonical_name),
human_time(time.time() - start_time),
))
else:
logger.warning(
L(
"β‘ Server error during remote call of {}. Took {}: {}",
Bold(rpc_message.canonical_name),
human_time(time.time() - start_time),
result_message.result,
))
raise LightbusServerError(
"Error while calling {}: {}\nRemote stack trace:\n{}".format(
rpc_message.canonical_name, result_message.result,
result_message.trace))
self._validate(result_message,
"incoming",
api_name,
procedure_name=name)
return result_message.result
