async def call_rpc_remote(self, api_name: str, name: str, kwargs: dict = frozendict(), options: dict = frozendict()): """ Perform an RPC call Call an RPC and return the result. """ await self.lazy_load_now() return await self.rpc_result_client.call_rpc_remote(api_name=api_name, name=name, kwargs=kwargs, options=options)
def __init__( self, redis_pool=None, *, consumer_group_prefix: str, consumer_name: str, url=None, serializer=ByFieldMessageSerializer(), deserializer=ByFieldMessageDeserializer(EventMessage), connection_parameters: Mapping = frozendict(maxsize=100), batch_size=10, reclaim_batch_size: int = None, acknowledgement_timeout: float = 60, max_stream_length: Optional[int] = 100000, stream_use: StreamUse = StreamUse.PER_API, consumption_restart_delay: int = 5, ): self.set_redis_pool(redis_pool, url, connection_parameters) self.serializer = serializer self.deserializer = deserializer self.batch_size = batch_size self.reclaim_batch_size = reclaim_batch_size if reclaim_batch_size else batch_size * 10 self.consumer_group_prefix = consumer_group_prefix self.consumer_name = consumer_name self.acknowledgement_timeout = acknowledgement_timeout self.max_stream_length = max_stream_length self.stream_use = stream_use self.consumption_restart_delay = consumption_restart_delay self._task = None self._reload = False
def from_config( cls, config, url: str = "redis://127.0.0.1:6379/0", connection_parameters: Mapping = frozendict(), ): return cls(url=url, connection_parameters=connection_parameters)
async def call_rpc_local(self, api_name: str, name: str, kwargs: dict = frozendict()): api = registry.get(api_name) self._validate_name(api_name, "rpc", name) start_time = time.time() try: method = getattr(api, name) if self.config.api(api_name).cast_values: kwargs = cast_to_signature(kwargs, method) result = method(**kwargs) result = await await_if_necessary(result) except (CancelledError, SuddenDeathException): raise except Exception as e: logger.warning( L( "⚡ Error while executing {}.{}. Took {}", Bold(api_name), Bold(name), human_time(time.time() - start_time), )) return e else: logger.info( L( "⚡ Executed {}.{} in {}", Bold(api_name), Bold(name), human_time(time.time() - start_time), )) return result
async def _call_rpc_local(self, api_name: str, name: str, kwargs: dict = frozendict()): api = self.api_registry.get(api_name) validate_event_or_rpc_name(api_name, "rpc", name) start_time = time.time() try: method = getattr(api, name) if self.config.api(api_name).cast_values: kwargs = cast_to_signature(kwargs, method) result = await run_user_provided_callable(method, args=[], kwargs=kwargs) except (asyncio.CancelledError, SuddenDeathException): raise except Exception as e: logging.exception(e) logger.warning( L( "⚡ Error while executing {}.{}. Took {}", Bold(api_name), Bold(name), human_time(time.time() - start_time), ) ) raise else: logger.info( L( "⚡ Executed {}.{} in {}", Bold(api_name), Bold(name), human_time(time.time() - start_time), ) ) return result
def __init__( self, *, redis_pool=None, url: str = "redis://127.0.0.1:6379/0", connection_parameters: Mapping = frozendict(), ): self.set_redis_pool(redis_pool, url, connection_parameters) self._latest_ids = {}
def set_redis_pool( self, redis_pool: Optional[Redis], url: str = None, connection_parameters: Mapping = frozendict(), ): self._redis_pool = None self._closed = False if not redis_pool: # Connect lazily using the provided parameters self.connection_parameters = self.connection_parameters.copy() self.connection_parameters.update(connection_parameters) if url: self.connection_parameters["address"] = url else: # Use the provided connection if isinstance(redis_pool, (ConnectionsPool,)): # If they've passed a raw pool then wrap it up in a Redis object. # aioredis.create_redis_pool() normally does this for us. redis_pool = Redis(redis_pool) if not isinstance(redis_pool, (Redis,)): raise InvalidRedisPool( "Invalid Redis connection provided: {}. If unsure, use aioredis.create_redis_pool() to " "create your redis connection.".format(redis_pool) ) if not isinstance(redis_pool._pool_or_conn, (ConnectionsPool,)): raise InvalidRedisPool( "The provided redis connection is backed by a single connection, rather than a " "pool of connections. This will lead to lightbus deadlocks and is unsupported. " "If unsure, use aioredis.create_redis_pool() to create your redis connection." ) # Determine the connection parameters from the given pool # (we will need these in other to create new pools for other threads) self.connection_parameters = dict( address=redis_pool.address, db=redis_pool.db, password=redis_pool._pool_or_conn._password, encoding=redis_pool.encoding, minsize=redis_pool._pool_or_conn.minsize, maxsize=redis_pool._pool_or_conn.maxsize, ssl=redis_pool._pool_or_conn._ssl, parser=redis_pool._pool_or_conn._parser_class, timeout=redis_pool._pool_or_conn._create_connection_timeout, connection_cls=redis_pool._pool_or_conn._connection_cls, ) self._redis_pool = redis_pool
def from_config( cls, config: "Config", service_name: str = None, consumer_name: str = None, url: str = "redis://127.0.0.1:6379/0", connection_parameters: Mapping = frozendict(maxsize=100), batch_size: int = 10, reclaim_batch_size: int = None, serializer: str = "lightbus.serializers.ByFieldMessageSerializer", deserializer: str = "lightbus.serializers.ByFieldMessageDeserializer", acknowledgement_timeout: float = 60, max_stream_length: Optional[int] = 100_000, stream_use: StreamUse = StreamUse.PER_API,
def __init__( self, redis_pool=None, *, service_name: str, consumer_name: str, url=None, serializer=ByFieldMessageSerializer(), deserializer=ByFieldMessageDeserializer(RedisEventMessage), connection_parameters: Mapping = frozendict(maxsize=100), batch_size=10, reclaim_batch_size: int = None, acknowledgement_timeout: float = 60, max_stream_length: Optional[int] = 100_000, stream_use: StreamUse = StreamUse.PER_API,
def __init__( self, *, redis_pool=None, url=None, serializer=BlobMessageSerializer(), deserializer=BlobMessageDeserializer(ResultMessage), connection_parameters: Mapping = frozendict(maxsize=100), result_ttl=60, rpc_timeout=5, ): # NOTE: We use the blob message_serializer here, as the results come back as single values in a redis list self.set_redis_pool(redis_pool, url, connection_parameters) self.serializer = serializer self.deserializer = deserializer self.result_ttl = result_ttl self.rpc_timeout = rpc_timeout
def __init__( self, *, redis_pool=None, url=None, serializer=BlobMessageSerializer(), deserializer=BlobMessageDeserializer(RpcMessage), connection_parameters: Mapping = frozendict(maxsize=100), batch_size=10, rpc_timeout=5, consumption_restart_delay=5, ): self.set_redis_pool(redis_pool, url, connection_parameters) self._latest_ids = {} self.serializer = serializer self.deserializer = deserializer self.batch_size = batch_size self.rpc_timeout = rpc_timeout self.consumption_restart_delay = consumption_restart_delay
def from_config( cls, config: "Config", url: str = "redis://127.0.0.1:6379/0", serializer: str = "lightbus.serializers.BlobMessageSerializer", deserializer: str = "lightbus.serializers.BlobMessageDeserializer", connection_parameters: Mapping = frozendict(maxsize=100), result_ttl=60, rpc_timeout=5, ): serializer = import_from_string(serializer)() deserializer = import_from_string(deserializer)(ResultMessage) return cls( url=url, serializer=serializer, deserializer=deserializer, connection_parameters=connection_parameters, result_ttl=result_ttl, rpc_timeout=rpc_timeout, )
def from_config( cls, config: "Config", consumer_group_prefix: str = None, consumer_name: str = None, url: str = "redis://127.0.0.1:6379/0", connection_parameters: Mapping = frozendict(maxsize=100), batch_size: int = 10, reclaim_batch_size: int = None, serializer: str = "lightbus.serializers.ByFieldMessageSerializer", deserializer: str = "lightbus.serializers.ByFieldMessageDeserializer", acknowledgement_timeout: float = 60, max_stream_length: Optional[int] = 100000, stream_use: StreamUse = StreamUse.PER_API, consumption_restart_delay: int = 5, ): serializer = import_from_string(serializer)() deserializer = import_from_string(deserializer)(EventMessage) consumer_group_prefix = consumer_group_prefix or config.service_name consumer_name = consumer_name or config.process_name if isinstance(stream_use, str): stream_use = StreamUse[stream_use.upper()] return cls( redis_pool=None, consumer_group_prefix=consumer_group_prefix, consumer_name=consumer_name, url=url, connection_parameters=connection_parameters, batch_size=batch_size, reclaim_batch_size=reclaim_batch_size, serializer=serializer, deserializer=deserializer, acknowledgement_timeout=acknowledgement_timeout, max_stream_length=max_stream_length, stream_use=stream_use, consumption_restart_delay=consumption_restart_delay, )
def from_config( cls, config: "Config", url: str = "redis://127.0.0.1:6379/0", connection_parameters: Mapping = frozendict(maxsize=100), batch_size: int = 10, serializer: str = "lightbus.serializers.BlobMessageSerializer", deserializer: str = "lightbus.serializers.BlobMessageDeserializer", rpc_timeout: int = 5, consumption_restart_delay: int = 5, ): serializer = import_from_string(serializer)() deserializer = import_from_string(deserializer)(RpcMessage) return cls( url=url, serializer=serializer, deserializer=deserializer, connection_parameters=connection_parameters, batch_size=batch_size, rpc_timeout=rpc_timeout, consumption_restart_delay=consumption_restart_delay, )
def set_redis_pool( self, redis_pool: Optional[Redis], url: str = None, connection_parameters: Mapping = frozendict(), ): self._local = threading.local() self._closed = False if not redis_pool: # Connect lazily using the provided parameters self.connection_parameters = self.connection_parameters.copy() self.connection_parameters.update(connection_parameters) if url: self.connection_parameters["address"] = url else: # Use the provided connection self.connection_parameters = None if isinstance(redis_pool, (ConnectionsPool,)): # If they've passed a raw pool then wrap it up in a Redis object. # aioredis.create_redis_pool() normally does this for us. redis_pool = Redis(redis_pool) if not isinstance(redis_pool, (Redis,)): raise InvalidRedisPool( "Invalid Redis connection provided: {}. If unsure, use aioredis.create_redis_pool() to " "create your redis connection.".format(redis_pool) ) if not isinstance(redis_pool._pool_or_conn, (ConnectionsPool,)): raise InvalidRedisPool( "The provided redis connection is backed by a single connection, rather than a " "pool of connections. This will lead to lightbus deadlocks and is unsupported. " "If unsure, use aioredis.create_redis_pool() to create your redis connection." ) self._local.redis_pool = redis_pool
async def call_rpc_remote( self, api_name: str, name: str, kwargs: dict = frozendict(), options: dict = frozendict() ): """ Perform an RPC call Call an RPC and return the result. """ kwargs = deform_to_bus(kwargs) rpc_message = RpcMessage(api_name=api_name, procedure_name=name, kwargs=kwargs) validate_event_or_rpc_name(api_name, "rpc", name) logger.info("📞 Calling remote RPC {}.{}".format(Bold(api_name), Bold(name))) start_time = time.time() validate_outgoing(self.config, self.schema, rpc_message) await self.hook_registry.execute("before_rpc_call", rpc_message=rpc_message) result_queue = InternalQueue() # Send the RPC await self.producer.send( commands.CallRpcCommand(message=rpc_message, options=options) ).wait() # Start a listener which will wait for results await self.producer.send( commands.ReceiveResultCommand( message=rpc_message, destination_queue=result_queue, options=options ) ).wait() # Wait for the result from the listener we started. # The RpcResultDock will handle timeouts result = await bail_on_error(self.error_queue, result_queue.get()) call_time = time.time() - start_time try: if isinstance(result, Exception): raise result except asyncio.TimeoutError: raise LightbusTimeout( f"Timeout when calling RPC {rpc_message.canonical_name} after waiting for {human_time(call_time)}. " f"It is possible no Lightbus process is serving this API, or perhaps it is taking " f"too long to process the request. In which case consider raising the 'rpc_timeout' " f"config option." ) from None else: assert isinstance(result, ResultMessage) result_message = result await self.hook_registry.execute( "after_rpc_call", rpc_message=rpc_message, result_message=result_message ) if not result_message.error: logger.info( L( "🏁 Remote call of {} completed in {}", Bold(rpc_message.canonical_name), human_time(call_time), ) ) else: logger.warning( L( "⚡ Error during remote call of RPC {}. Took {}: {}", Bold(rpc_message.canonical_name), human_time(call_time), result_message.result, ) ) raise LightbusWorkerError( "Error while calling {}: {}\nRemote stack trace:\n{}".format( rpc_message.canonical_name, result_message.result, result_message.trace ) ) validate_incoming(self.config, self.schema, result_message) return result_message.result
(1, int, 1), ("1", int, 1), (1.23, float, 1.23), ("1.23", float, 1.23), (True, bool, True), ("1", bool, True), ("", bool, False), (None, int, None), ("a", str, "a"), ("YWJjAA==", bytes, b"abc\0"), (1, str, "1"), ("1.23", Decimal, Decimal("1.23")), ("(1+2j)", complex, complex("(1+2j)")), ({ "a": 1 }, frozendict, frozendict(a=1)), ( "abf4ddeb-fb9c-44c5-b865-012ba7787469", UUID, UUID("abf4ddeb-fb9c-44c5-b865-012ba7787469"), ), ({ "a": 1, "b": "2" }, SimpleNamedTuple, SimpleNamedTuple(a="1", b=2)), ({ "a": 1, "b": "2" }, SimpleDataclass, SimpleDataclass(a="1", b=2)), ({ "a": 1,
"test_input,hint,expected", [ (1, int, 1), ("1", int, 1), (1.23, float, 1.23), ("1.23", float, 1.23), (True, bool, True), ("1", bool, True), ("", bool, False), (None, int, None), ("a", str, "a"), ("YWJjAA==", bytes, b"abc\0"), (1, str, "1"), ("1.23", Decimal, Decimal("1.23")), ("(1+2j)", complex, complex("(1+2j)")), ({"a": 1}, frozendict, frozendict(a=1)), ( "abf4ddeb-fb9c-44c5-b865-012ba7787469", UUID, UUID("abf4ddeb-fb9c-44c5-b865-012ba7787469"), ), ({"a": 1, "b": "2"}, SimpleNamedTuple, SimpleNamedTuple(a="1", b=2)), ({"a": 1, "b": "2"}, SimpleDataclass, SimpleDataclass(a="1", b=2)), ({"a": 1, "b": "2"}, DataclassWithMethod, DataclassWithMethod(a="1", b=2)), ( {"val": {"a": 1, "b": "2"}}, ComplexNamedTuple, ComplexNamedTuple(val=SimpleNamedTuple(a="1", b=2)), ), ( {"val": {"a": 1, "b": "2"}},
async def call_rpc_remote(self, api_name: str, name: str, kwargs: dict = frozendict(), options: dict = frozendict()): rpc_transport = self.transport_registry.get_rpc_transport(api_name) result_transport = self.transport_registry.get_result_transport( api_name) kwargs = deform_to_bus(kwargs) rpc_message = RpcMessage(api_name=api_name, procedure_name=name, kwargs=kwargs) return_path = result_transport.get_return_path(rpc_message) rpc_message.return_path = return_path options = options or {} timeout = options.get("timeout", self.config.api(api_name).rpc_timeout) self._validate_name(api_name, "rpc", name) logger.info("📞 Calling remote RPC {}.{}".format( Bold(api_name), Bold(name))) start_time = time.time() # TODO: It is possible that the RPC will be called before we start waiting for the response. This is bad. self._validate(rpc_message, "outgoing") future = asyncio.gather( self.receive_result(rpc_message, return_path, options=options), rpc_transport.call_rpc(rpc_message, options=options), ) await self._plugin_hook("before_rpc_call", rpc_message=rpc_message) try: result_message, _ = await asyncio.wait_for(future, timeout=timeout) future.result() except asyncio.TimeoutError: # Allow the future to finish, as per https://bugs.python.org/issue29432 try: await future future.result() except CancelledError: pass # TODO: Remove RPC from queue. Perhaps add a RpcBackend.cancel() method. Optional, # as not all backends will support it. No point processing calls which have timed out. raise LightbusTimeout( f"Timeout when calling RPC {rpc_message.canonical_name} after {timeout} seconds. " f"It is possible no Lightbus process is serving this API, or perhaps it is taking " f"too long to process the request. In which case consider raising the 'rpc_timeout' " f"config option.") from None await self._plugin_hook("after_rpc_call", rpc_message=rpc_message, result_message=result_message) if not result_message.error: logger.info( L( "🏁 Remote call of {} completed in {}", Bold(rpc_message.canonical_name), human_time(time.time() - start_time), )) else: logger.warning( L( "⚡ Server error during remote call of {}. Took {}: {}", Bold(rpc_message.canonical_name), human_time(time.time() - start_time), result_message.result, )) raise LightbusServerError( "Error while calling {}: {}\nRemote stack trace:\n{}".format( rpc_message.canonical_name, result_message.result, result_message.trace)) self._validate(result_message, "incoming", api_name, procedure_name=name) return result_message.result
def test_frozendict_get(d): assert hash(d) == hash(d.copy()) assert hash(d) != hash(frozendict({}))
def d(): return frozendict(a=1, b=2)
"b": 1 }), ( datetime(2018, 6, 5, 10, 48, 12, 792_937, tzinfo=timezone.utc), "2018-06-05T10:48:12.792937+00:00", ), (date(2018, 6, 5), "2018-06-05"), (CustomClassWithMagicMethod(), { "a": 1 }), (CustomClassWithMagicMethodNeedsEncoding(), { "a": "(1+2j)" }), (Decimal("1.23"), "1.23"), (complex(1, 2), "(1+2j)"), (frozendict(a=1), { "a": 1 }), (UUID("abf4ddeb-fb9c-44c5-b865-012ba7787469"), "abf4ddeb-fb9c-44c5-b865-012ba7787469"), ({ "a": 1 }, { "a": 1 }), ({ "val": ExampleEnum.foo }, { "val": "a" }), (OrderedDict({"val": ExampleEnum.foo}), {
datetime(2018, 6, 5, 10, 48, 12, 792_937, tzinfo=timezone.utc), "2018-06-05T10:48:12.792937+00:00", ), "date": (date(2018, 6, 5), "2018-06-05"), "custom_class_magic_method": (CustomClassWithMagicMethod(), { "a": 1 }), "custom_class_magic_method_needs_encoding": ( CustomClassWithMagicMethodNeedsEncoding(), { "a": "(1+2j)" }, ), "decimal": (Decimal("1.23"), "1.23"), "complex": (complex(1, 2), "(1+2j)"), "frozendict": (frozendict(a=1), { "a": 1 }), "uuid": (UUID("abf4ddeb-fb9c-44c5-b865-012ba7787469"), "abf4ddeb-fb9c-44c5-b865-012ba7787469"), "dict": ({ "a": 1 }, { "a": 1 }), "dict_with_types": ({ "val": ExampleEnum.foo }, { "val": "a" }), "orderd_dict_with_types": (OrderedDict({"val": ExampleEnum.foo}), {