async def call_rpc_remote(self,
api_name: str,
name: str,
kwargs: dict = frozendict(),
options: dict = frozendict()):
""" Perform an RPC call
Call an RPC and return the result.
"""
await self.lazy_load_now()
return await self.rpc_result_client.call_rpc_remote(api_name=api_name,
name=name,
kwargs=kwargs,
options=options)
def __init__(
self,
redis_pool=None,
*,
consumer_group_prefix: str,
consumer_name: str,
url=None,
serializer=ByFieldMessageSerializer(),
deserializer=ByFieldMessageDeserializer(EventMessage),
connection_parameters: Mapping = frozendict(maxsize=100),
batch_size=10,
reclaim_batch_size: int = None,
acknowledgement_timeout: float = 60,
max_stream_length: Optional[int] = 100000,
stream_use: StreamUse = StreamUse.PER_API,
consumption_restart_delay: int = 5,
):
self.set_redis_pool(redis_pool, url, connection_parameters)
self.serializer = serializer
self.deserializer = deserializer
self.batch_size = batch_size
self.reclaim_batch_size = reclaim_batch_size if reclaim_batch_size else batch_size * 10
self.consumer_group_prefix = consumer_group_prefix
self.consumer_name = consumer_name
self.acknowledgement_timeout = acknowledgement_timeout
self.max_stream_length = max_stream_length
self.stream_use = stream_use
self.consumption_restart_delay = consumption_restart_delay
self._task = None
self._reload = False
def from_config(
cls,
config,
url: str = "redis://127.0.0.1:6379/0",
connection_parameters: Mapping = frozendict(),
):
return cls(url=url, connection_parameters=connection_parameters)
async def call_rpc_local(self,
api_name: str,
name: str,
kwargs: dict = frozendict()):
api = registry.get(api_name)
self._validate_name(api_name, "rpc", name)
start_time = time.time()
try:
method = getattr(api, name)
if self.config.api(api_name).cast_values:
kwargs = cast_to_signature(kwargs, method)
result = method(**kwargs)
result = await await_if_necessary(result)
except (CancelledError, SuddenDeathException):
raise
except Exception as e:
logger.warning(
L(
"⚡ Error while executing {}.{}. Took {}",
Bold(api_name),
Bold(name),
human_time(time.time() - start_time),
))
return e
else:
logger.info(
L(
"⚡ Executed {}.{} in {}",
Bold(api_name),
Bold(name),
human_time(time.time() - start_time),
))
return result
async def _call_rpc_local(self, api_name: str, name: str, kwargs: dict = frozendict()):
api = self.api_registry.get(api_name)
validate_event_or_rpc_name(api_name, "rpc", name)
start_time = time.time()
try:
method = getattr(api, name)
if self.config.api(api_name).cast_values:
kwargs = cast_to_signature(kwargs, method)
result = await run_user_provided_callable(method, args=[], kwargs=kwargs)
except (asyncio.CancelledError, SuddenDeathException):
raise
except Exception as e:
logging.exception(e)
logger.warning(
L(
"⚡ Error while executing {}.{}. Took {}",
Bold(api_name),
Bold(name),
human_time(time.time() - start_time),
)
)
raise
else:
logger.info(
L(
"⚡ Executed {}.{} in {}",
Bold(api_name),
Bold(name),
human_time(time.time() - start_time),
)
)
return result
def __init__(
self,
*,
redis_pool=None,
url: str = "redis://127.0.0.1:6379/0",
connection_parameters: Mapping = frozendict(),
):
self.set_redis_pool(redis_pool, url, connection_parameters)
self._latest_ids = {}
def set_redis_pool(
self,
redis_pool: Optional[Redis],
url: str = None,
connection_parameters: Mapping = frozendict(),
):
self._redis_pool = None
self._closed = False
if not redis_pool:
# Connect lazily using the provided parameters
self.connection_parameters = self.connection_parameters.copy()
self.connection_parameters.update(connection_parameters)
if url:
self.connection_parameters["address"] = url
else:
# Use the provided connection
if isinstance(redis_pool, (ConnectionsPool,)):
# If they've passed a raw pool then wrap it up in a Redis object.
# aioredis.create_redis_pool() normally does this for us.
redis_pool = Redis(redis_pool)
if not isinstance(redis_pool, (Redis,)):
raise InvalidRedisPool(
"Invalid Redis connection provided: {}. If unsure, use aioredis.create_redis_pool() to "
"create your redis connection.".format(redis_pool)
)
if not isinstance(redis_pool._pool_or_conn, (ConnectionsPool,)):
raise InvalidRedisPool(
"The provided redis connection is backed by a single connection, rather than a "
"pool of connections. This will lead to lightbus deadlocks and is unsupported. "
"If unsure, use aioredis.create_redis_pool() to create your redis connection."
)
# Determine the connection parameters from the given pool
# (we will need these in other to create new pools for other threads)
self.connection_parameters = dict(
address=redis_pool.address,
db=redis_pool.db,
password=redis_pool._pool_or_conn._password,
encoding=redis_pool.encoding,
minsize=redis_pool._pool_or_conn.minsize,
maxsize=redis_pool._pool_or_conn.maxsize,
ssl=redis_pool._pool_or_conn._ssl,
parser=redis_pool._pool_or_conn._parser_class,
timeout=redis_pool._pool_or_conn._create_connection_timeout,
connection_cls=redis_pool._pool_or_conn._connection_cls,
)
self._redis_pool = redis_pool
def from_config(
cls,
config: "Config",
service_name: str = None,
consumer_name: str = None,
url: str = "redis://127.0.0.1:6379/0",
connection_parameters: Mapping = frozendict(maxsize=100),
batch_size: int = 10,
reclaim_batch_size: int = None,
serializer: str = "lightbus.serializers.ByFieldMessageSerializer",
deserializer: str = "lightbus.serializers.ByFieldMessageDeserializer",
acknowledgement_timeout: float = 60,
max_stream_length: Optional[int] = 100_000,
stream_use: StreamUse = StreamUse.PER_API,
def __init__(
self,
redis_pool=None,
*,
service_name: str,
consumer_name: str,
url=None,
serializer=ByFieldMessageSerializer(),
deserializer=ByFieldMessageDeserializer(RedisEventMessage),
connection_parameters: Mapping = frozendict(maxsize=100),
batch_size=10,
reclaim_batch_size: int = None,
acknowledgement_timeout: float = 60,
max_stream_length: Optional[int] = 100_000,
stream_use: StreamUse = StreamUse.PER_API,
def __init__(
self,
*,
redis_pool=None,
url=None,
serializer=BlobMessageSerializer(),
deserializer=BlobMessageDeserializer(ResultMessage),
connection_parameters: Mapping = frozendict(maxsize=100),
result_ttl=60,
rpc_timeout=5,
):
# NOTE: We use the blob message_serializer here, as the results come back as single values in a redis list
self.set_redis_pool(redis_pool, url, connection_parameters)
self.serializer = serializer
self.deserializer = deserializer
self.result_ttl = result_ttl
self.rpc_timeout = rpc_timeout
def __init__(
self,
*,
redis_pool=None,
url=None,
serializer=BlobMessageSerializer(),
deserializer=BlobMessageDeserializer(RpcMessage),
connection_parameters: Mapping = frozendict(maxsize=100),
batch_size=10,
rpc_timeout=5,
consumption_restart_delay=5,
):
self.set_redis_pool(redis_pool, url, connection_parameters)
self._latest_ids = {}
self.serializer = serializer
self.deserializer = deserializer
self.batch_size = batch_size
self.rpc_timeout = rpc_timeout
self.consumption_restart_delay = consumption_restart_delay
def from_config(
cls,
config: "Config",
url: str = "redis://127.0.0.1:6379/0",
serializer: str = "lightbus.serializers.BlobMessageSerializer",
deserializer: str = "lightbus.serializers.BlobMessageDeserializer",
connection_parameters: Mapping = frozendict(maxsize=100),
result_ttl=60,
rpc_timeout=5,
):
serializer = import_from_string(serializer)()
deserializer = import_from_string(deserializer)(ResultMessage)
return cls(
url=url,
serializer=serializer,
deserializer=deserializer,
connection_parameters=connection_parameters,
result_ttl=result_ttl,
rpc_timeout=rpc_timeout,
)
def from_config(
cls,
config: "Config",
consumer_group_prefix: str = None,
consumer_name: str = None,
url: str = "redis://127.0.0.1:6379/0",
connection_parameters: Mapping = frozendict(maxsize=100),
batch_size: int = 10,
reclaim_batch_size: int = None,
serializer: str = "lightbus.serializers.ByFieldMessageSerializer",
deserializer: str = "lightbus.serializers.ByFieldMessageDeserializer",
acknowledgement_timeout: float = 60,
max_stream_length: Optional[int] = 100000,
stream_use: StreamUse = StreamUse.PER_API,
consumption_restart_delay: int = 5,
):
serializer = import_from_string(serializer)()
deserializer = import_from_string(deserializer)(EventMessage)
consumer_group_prefix = consumer_group_prefix or config.service_name
consumer_name = consumer_name or config.process_name
if isinstance(stream_use, str):
stream_use = StreamUse[stream_use.upper()]
return cls(
redis_pool=None,
consumer_group_prefix=consumer_group_prefix,
consumer_name=consumer_name,
url=url,
connection_parameters=connection_parameters,
batch_size=batch_size,
reclaim_batch_size=reclaim_batch_size,
serializer=serializer,
deserializer=deserializer,
acknowledgement_timeout=acknowledgement_timeout,
max_stream_length=max_stream_length,
stream_use=stream_use,
consumption_restart_delay=consumption_restart_delay,
)
def from_config(
cls,
config: "Config",
url: str = "redis://127.0.0.1:6379/0",
connection_parameters: Mapping = frozendict(maxsize=100),
batch_size: int = 10,
serializer: str = "lightbus.serializers.BlobMessageSerializer",
deserializer: str = "lightbus.serializers.BlobMessageDeserializer",
rpc_timeout: int = 5,
consumption_restart_delay: int = 5,
):
serializer = import_from_string(serializer)()
deserializer = import_from_string(deserializer)(RpcMessage)
return cls(
url=url,
serializer=serializer,
deserializer=deserializer,
connection_parameters=connection_parameters,
batch_size=batch_size,
rpc_timeout=rpc_timeout,
consumption_restart_delay=consumption_restart_delay,
)
def set_redis_pool(
self,
redis_pool: Optional[Redis],
url: str = None,
connection_parameters: Mapping = frozendict(),
):
self._local = threading.local()
self._closed = False
if not redis_pool:
# Connect lazily using the provided parameters
self.connection_parameters = self.connection_parameters.copy()
self.connection_parameters.update(connection_parameters)
if url:
self.connection_parameters["address"] = url
else:
# Use the provided connection
self.connection_parameters = None
if isinstance(redis_pool, (ConnectionsPool,)):
# If they've passed a raw pool then wrap it up in a Redis object.
# aioredis.create_redis_pool() normally does this for us.
redis_pool = Redis(redis_pool)
if not isinstance(redis_pool, (Redis,)):
raise InvalidRedisPool(
"Invalid Redis connection provided: {}. If unsure, use aioredis.create_redis_pool() to "
"create your redis connection.".format(redis_pool)
)
if not isinstance(redis_pool._pool_or_conn, (ConnectionsPool,)):
raise InvalidRedisPool(
"The provided redis connection is backed by a single connection, rather than a "
"pool of connections. This will lead to lightbus deadlocks and is unsupported. "
"If unsure, use aioredis.create_redis_pool() to create your redis connection."
)
self._local.redis_pool = redis_pool
async def call_rpc_remote(
self, api_name: str, name: str, kwargs: dict = frozendict(), options: dict = frozendict()
):
""" Perform an RPC call
Call an RPC and return the result.
"""
kwargs = deform_to_bus(kwargs)
rpc_message = RpcMessage(api_name=api_name, procedure_name=name, kwargs=kwargs)
validate_event_or_rpc_name(api_name, "rpc", name)
logger.info("📞 Calling remote RPC {}.{}".format(Bold(api_name), Bold(name)))
start_time = time.time()
validate_outgoing(self.config, self.schema, rpc_message)
await self.hook_registry.execute("before_rpc_call", rpc_message=rpc_message)
result_queue = InternalQueue()
# Send the RPC
await self.producer.send(
commands.CallRpcCommand(message=rpc_message, options=options)
).wait()
# Start a listener which will wait for results
await self.producer.send(
commands.ReceiveResultCommand(
message=rpc_message, destination_queue=result_queue, options=options
)
).wait()
# Wait for the result from the listener we started.
# The RpcResultDock will handle timeouts
result = await bail_on_error(self.error_queue, result_queue.get())
call_time = time.time() - start_time
try:
if isinstance(result, Exception):
raise result
except asyncio.TimeoutError:
raise LightbusTimeout(
f"Timeout when calling RPC {rpc_message.canonical_name} after waiting for {human_time(call_time)}. "
f"It is possible no Lightbus process is serving this API, or perhaps it is taking "
f"too long to process the request. In which case consider raising the 'rpc_timeout' "
f"config option."
) from None
else:
assert isinstance(result, ResultMessage)
result_message = result
await self.hook_registry.execute(
"after_rpc_call", rpc_message=rpc_message, result_message=result_message
)
if not result_message.error:
logger.info(
L(
"🏁 Remote call of {} completed in {}",
Bold(rpc_message.canonical_name),
human_time(call_time),
)
)
else:
logger.warning(
L(
"⚡ Error during remote call of RPC {}. Took {}: {}",
Bold(rpc_message.canonical_name),
human_time(call_time),
result_message.result,
)
)
raise LightbusWorkerError(
"Error while calling {}: {}\nRemote stack trace:\n{}".format(
rpc_message.canonical_name, result_message.result, result_message.trace
)
)
validate_incoming(self.config, self.schema, result_message)
return result_message.result
(1, int, 1),
("1", int, 1),
(1.23, float, 1.23),
("1.23", float, 1.23),
(True, bool, True),
("1", bool, True),
("", bool, False),
(None, int, None),
("a", str, "a"),
("YWJjAA==", bytes, b"abc\0"),
(1, str, "1"),
("1.23", Decimal, Decimal("1.23")),
("(1+2j)", complex, complex("(1+2j)")),
({
"a": 1
}, frozendict, frozendict(a=1)),
(
"abf4ddeb-fb9c-44c5-b865-012ba7787469",
UUID,
UUID("abf4ddeb-fb9c-44c5-b865-012ba7787469"),
),
({
"a": 1,
"b": "2"
}, SimpleNamedTuple, SimpleNamedTuple(a="1", b=2)),
({
"a": 1,
"b": "2"
}, SimpleDataclass, SimpleDataclass(a="1", b=2)),
({
"a": 1,
"test_input,hint,expected",
[
(1, int, 1),
("1", int, 1),
(1.23, float, 1.23),
("1.23", float, 1.23),
(True, bool, True),
("1", bool, True),
("", bool, False),
(None, int, None),
("a", str, "a"),
("YWJjAA==", bytes, b"abc\0"),
(1, str, "1"),
("1.23", Decimal, Decimal("1.23")),
("(1+2j)", complex, complex("(1+2j)")),
({"a": 1}, frozendict, frozendict(a=1)),
(
"abf4ddeb-fb9c-44c5-b865-012ba7787469",
UUID,
UUID("abf4ddeb-fb9c-44c5-b865-012ba7787469"),
),
({"a": 1, "b": "2"}, SimpleNamedTuple, SimpleNamedTuple(a="1", b=2)),
({"a": 1, "b": "2"}, SimpleDataclass, SimpleDataclass(a="1", b=2)),
({"a": 1, "b": "2"}, DataclassWithMethod, DataclassWithMethod(a="1", b=2)),
(
{"val": {"a": 1, "b": "2"}},
ComplexNamedTuple,
ComplexNamedTuple(val=SimpleNamedTuple(a="1", b=2)),
),
(
{"val": {"a": 1, "b": "2"}},
async def call_rpc_remote(self,
api_name: str,
name: str,
kwargs: dict = frozendict(),
options: dict = frozendict()):
rpc_transport = self.transport_registry.get_rpc_transport(api_name)
result_transport = self.transport_registry.get_result_transport(
api_name)
kwargs = deform_to_bus(kwargs)
rpc_message = RpcMessage(api_name=api_name,
procedure_name=name,
kwargs=kwargs)
return_path = result_transport.get_return_path(rpc_message)
rpc_message.return_path = return_path
options = options or {}
timeout = options.get("timeout", self.config.api(api_name).rpc_timeout)
self._validate_name(api_name, "rpc", name)
logger.info("📞 Calling remote RPC {}.{}".format(
Bold(api_name), Bold(name)))
start_time = time.time()
# TODO: It is possible that the RPC will be called before we start waiting for the response. This is bad.
self._validate(rpc_message, "outgoing")
future = asyncio.gather(
self.receive_result(rpc_message, return_path, options=options),
rpc_transport.call_rpc(rpc_message, options=options),
)
await self._plugin_hook("before_rpc_call", rpc_message=rpc_message)
try:
result_message, _ = await asyncio.wait_for(future, timeout=timeout)
future.result()
except asyncio.TimeoutError:
# Allow the future to finish, as per https://bugs.python.org/issue29432
try:
await future
future.result()
except CancelledError:
pass
# TODO: Remove RPC from queue. Perhaps add a RpcBackend.cancel() method. Optional,
# as not all backends will support it. No point processing calls which have timed out.
raise LightbusTimeout(
f"Timeout when calling RPC {rpc_message.canonical_name} after {timeout} seconds. "
f"It is possible no Lightbus process is serving this API, or perhaps it is taking "
f"too long to process the request. In which case consider raising the 'rpc_timeout' "
f"config option.") from None
await self._plugin_hook("after_rpc_call",
rpc_message=rpc_message,
result_message=result_message)
if not result_message.error:
logger.info(
L(
"🏁 Remote call of {} completed in {}",
Bold(rpc_message.canonical_name),
human_time(time.time() - start_time),
))
else:
logger.warning(
L(
"⚡ Server error during remote call of {}. Took {}: {}",
Bold(rpc_message.canonical_name),
human_time(time.time() - start_time),
result_message.result,
))
raise LightbusServerError(
"Error while calling {}: {}\nRemote stack trace:\n{}".format(
rpc_message.canonical_name, result_message.result,
result_message.trace))
self._validate(result_message,
"incoming",
api_name,
procedure_name=name)
return result_message.result
def test_frozendict_get(d):
assert hash(d) == hash(d.copy())
assert hash(d) != hash(frozendict({}))
def d():
return frozendict(a=1, b=2)
"b": 1
}),
(
datetime(2018, 6, 5, 10, 48, 12, 792_937, tzinfo=timezone.utc),
"2018-06-05T10:48:12.792937+00:00",
),
(date(2018, 6, 5), "2018-06-05"),
(CustomClassWithMagicMethod(), {
"a": 1
}),
(CustomClassWithMagicMethodNeedsEncoding(), {
"a": "(1+2j)"
}),
(Decimal("1.23"), "1.23"),
(complex(1, 2), "(1+2j)"),
(frozendict(a=1), {
"a": 1
}),
(UUID("abf4ddeb-fb9c-44c5-b865-012ba7787469"),
"abf4ddeb-fb9c-44c5-b865-012ba7787469"),
({
"a": 1
}, {
"a": 1
}),
({
"val": ExampleEnum.foo
}, {
"val": "a"
}),
(OrderedDict({"val": ExampleEnum.foo}), {
datetime(2018, 6, 5, 10, 48, 12, 792_937, tzinfo=timezone.utc),
"2018-06-05T10:48:12.792937+00:00",
),
"date": (date(2018, 6, 5), "2018-06-05"),
"custom_class_magic_method": (CustomClassWithMagicMethod(), {
"a": 1
}),
"custom_class_magic_method_needs_encoding": (
CustomClassWithMagicMethodNeedsEncoding(),
{
"a": "(1+2j)"
},
),
"decimal": (Decimal("1.23"), "1.23"),
"complex": (complex(1, 2), "(1+2j)"),
"frozendict": (frozendict(a=1), {
"a": 1
}),
"uuid": (UUID("abf4ddeb-fb9c-44c5-b865-012ba7787469"),
"abf4ddeb-fb9c-44c5-b865-012ba7787469"),
"dict": ({
"a": 1
}, {
"a": 1
}),
"dict_with_types": ({
"val": ExampleEnum.foo
}, {
"val": "a"
}),
"orderd_dict_with_types": (OrderedDict({"val": ExampleEnum.foo}), {
