def to_model(self,
typ: Optional[str],
value: str,
serializer: CodecArg) -> Any:
"""Convert command-line argument to model.
Generic version of :meth:`to_key`/:meth:`to_value`.
Arguments:
typ: The name of the model to create.
key: The string json of the data to populate it with.
serializer: The argument setting it apart from to_key/to_value
enables you to specify a custom serializer not mandated
by :attr:`key_serializer`, and :attr:`value_serializer`.
Notes:
Uses :attr:`value_serializer` to set the :term:`codec`
for the value (e.g. ``"json"``), as set by the
:option:`--value-serializer <faust send --value-serializer>`
option.
"""
if typ:
model: ModelT = self.import_relative_to_app(typ)
return model.loads(want_bytes(value), serializer=serializer)
return want_bytes(value)
def _create_req(
self,
key: K = None,
value: V = None,
reply_to: ReplyToArg = None,
correlation_id: str = None,
headers: HeadersArg = None,
) -> Tuple[V, Optional[HeadersArg]]:
if reply_to is None:
raise TypeError("Missing reply_to argument")
topic_name = self._get_strtopic(reply_to)
correlation_id = correlation_id or str(uuid4())
open_headers = prepare_headers(headers or {})
if self.use_reply_headers:
merge_headers(
open_headers,
{
"Faust-Ag-ReplyTo": want_bytes(topic_name),
"Faust-Ag-CorrelationId": want_bytes(correlation_id),
},
)
return value, open_headers
else:
# wrap value in envelope
req = self._request_class(value)(
value=value,
reply_to=topic_name,
correlation_id=correlation_id,
)
return req, open_headers
async def _set(self,
key: str,
value: bytes,
timeout: float = None) -> None:
if timeout is not None:
self.storage.setex(key, timeout, want_bytes(value))
else:
self.storage.set(key, want_bytes(value))
def build_key(self, request: Request, method: str, prefix: str,
headers: Mapping[str, str]) -> str:
"""Build cache key from web request and environment."""
context = hashlib.md5(b''.join(
want_bytes(k) + want_bytes(v)
for k, v in headers.items())).hexdigest()
url = hashlib.md5(iri_to_uri(str(
request.url)).encode('ascii')).hexdigest()
return f'{self.ident}.{prefix}.{method}.{url}.{context}'
def dumps_key(self,
typ: Optional[ModelArg],
key: K,
*,
serializer: CodecArg = None,
skip: IsInstanceArg = (bytes,)) -> Optional[bytes]:
"""Serialize key.
Arguments:
typ: Model hint (can also be :class:`str`/:class:`bytes`).
When ``typ=str`` or :class:`bytes`, raw serializer is assumed.
key: The key value to serializer.
serializer: Codec to use for this key, if it is not a model type.
If not set the default will be used (:attr:`key_serializer`).
"""
is_model = False
if isinstance(key, ModelT):
is_model = True
key = cast(ModelT, key)
serializer = key._options.serializer or serializer
serializer = self._serializer(typ, serializer, self.key_serializer)
if serializer and not isinstance(key, skip):
if is_model:
return cast(ModelT, key).dumps(serializer=serializer)
return dumps(serializer, key)
return want_bytes(cast(bytes, key)) if key is not None else None
async def test_send(key, topic_name, expected_topic, key_serializer, app):
topic = app.topic(topic_name)
event = Value(amount=0.0)
await app.send(topic, key, event, key_serializer=key_serializer)
# do it twice so producer_started is also True
await app.send(topic, key, event, key_serializer=key_serializer)
expected_sender = app.producer.send
if key is not None:
if isinstance(key, str):
# Default serializer is raw, and str should be serialized
# (note that a bytes key will be left alone.
key_serializer = 'raw'
if isinstance(key, ModelT):
expected_key = key.dumps(serializer='raw')
elif key_serializer:
expected_key = codecs.dumps(key_serializer, key)
else:
expected_key = want_bytes(key)
else:
expected_key = None
expected_sender.assert_called_with(
expected_topic,
expected_key,
event.dumps(),
partition=None,
timestamp=None,
headers={},
)
def merge_headers(target: OpenHeadersArg, source: Mapping[str, Any]) -> None:
# XXX modify in-place
if source:
source = {want_str(k): want_bytes(v) for k, v in source.items()}
if isinstance(target, Mapping):
target = cast(MutableMapping, target)
target.update({k: v for k, v in source.items()})
elif isinstance(target, list):
target.extend((h for h in source.items()))
def test_on_produce_attach_test_headers(self, *, livecheck, app, execution):
headers = [("Foo-Bar-Baz", b"moo")]
original_headers = list(headers)
with current_test_stack.push(execution):
livecheck.on_produce_attach_test_headers(
sender=app,
key=b"k",
value=b"v",
partition=3,
headers=headers,
)
kafka_headers = {
k: want_bytes(v) for k, v in execution.as_headers().items()
}
assert headers == (original_headers + list(kafka_headers.items()))
def on_produce_attach_test_headers(self,
sender: AppT,
key: bytes = None,
value: bytes = None,
partition: int = None,
timestamp: float = None,
headers: List[Tuple[str, bytes]] = None,
**kwargs: Any) -> None:
"""Attach test headers to Kafka produce requests."""
test = current_test()
if test is not None:
if headers is None:
raise TypeError('Produce request missing headers list')
headers.extend([(k, want_bytes(v))
for k, v in test.as_headers().items()])
def _prepare_payload(self, typ: Optional[ModelArg], value: Any) -> Any:
if typ is None: # (autodetect)
return self.Model._maybe_reconstruct(value)
elif typ is int:
return int(want_str(value))
elif typ is float:
return float(want_str(value))
elif typ is Decimal:
return Decimal(want_str(value))
elif typ is str:
return want_str(value)
elif typ is bytes:
return want_bytes(value)
else:
# type set to Model
model = cast(ModelT, typ)
return model.from_data(value, preferred_type=model)
def merge_headers(target: HeadersArg, source: Mapping[str, Any]) -> HeadersArg:
# XXX may modify in-place, but always use return value.
if target is None:
target = []
if source:
source = {want_str(k): want_bytes(v) for k, v in source.items()}
if isinstance(target, MutableMapping):
target.update({k: v for k, v in source.items()})
elif isinstance(target, Mapping):
target = dict(target)
target.update({k: v for k, v in source.items()})
elif isinstance(target, list):
target.extend((h for h in source.items()))
elif isinstance(target, tuple):
target += tuple(h for h in source.items())
elif isinstance(target, Iterable):
target = list(cast(Iterable[Tuple[str, bytes]], target))
target.extend((h for h in source.items()))
return target
def _dumps(self, s: bytes) -> bytes:
return want_bytes(s)
def _loads(self, s: bytes) -> bytes:
return want_bytes(s)
def _dumps(self, s: bytes) -> bytes:
return b64encode(want_bytes(s))
def _dumps(self, s: Any) -> bytes:
if _yaml is None:
raise ImproperlyConfigured('Missing yaml: pip install PyYAML')
return want_bytes(_yaml.safe_dump(s))
def _dumps(self, s: Any) -> bytes:
return want_bytes(_json.dumps(s))
async def _set(self, key: str, value: bytes, timeout: float) -> None:
self._expires[key] = timeout
self._data[key] = want_bytes(value)
self._time_index[key] = monotonic()
async def _set(self, key: str, value: bytes, timeout: float) -> None:
self.storage.setex(key, timeout, want_bytes(value))
def test_want_bytes(input, expected):
assert want_bytes(input) == expected
async def _get(self, key: str) -> Optional[bytes]:
value: Optional[bytes] = await self.client.get(key)
if value is not None:
return want_bytes(value)
return None