def test_schema__overriding(self, *, app): schema = faust.Schema( key_type="bytes", value_type="bytes", key_serializer="msgpack", value_serializer="msgpack", ) topic = app.topic( "foo", schema=schema, key_type="str", value_type="str", key_serializer="captnproto", value_serializer="captnproto", ) assert topic.key_type == "str" assert topic.value_type == "str" assert topic.key_serializer == "captnproto" assert topic.value_serializer == "captnproto" assert topic.schema.key_type == "str" assert topic.schema.value_type == "str" assert topic.schema.key_serializer == "captnproto" assert topic.schema.value_serializer == "captnproto" assert repr(topic.schema)
def test_schema__overriding(self, *, app): schema = faust.Schema( key_type='bytes', value_type='bytes', key_serializer='msgpack', value_serializer='msgpack', ) topic = app.topic( 'foo', schema=schema, key_type='str', value_type='str', key_serializer='captnproto', value_serializer='captnproto', ) assert topic.key_type == 'str' assert topic.value_type == 'str' assert topic.key_serializer == 'captnproto' assert topic.value_serializer == 'captnproto' assert topic.schema.key_type == 'str' assert topic.schema.value_type == 'str' assert topic.schema.key_serializer == 'captnproto' assert topic.schema.value_serializer == 'captnproto' assert repr(topic.schema)
def test_schema__overriding(*, app): schema = faust.Schema( key_type='bytes', value_type='bytes', key_serializer='msgpack', value_serializer='msgpack', ) channel = app.channel(schema=schema, key_type='str', value_type='str') assert channel.key_type == 'str' assert channel.value_type == 'str' assert channel.schema.key_type == 'str' assert channel.schema.value_type == 'str' assert channel.schema.key_serializer == 'msgpack' assert channel.schema.value_serializer == 'msgpack'
def test_schema__overriding(*, app): schema = faust.Schema( key_type="bytes", value_type="bytes", key_serializer="msgpack", value_serializer="msgpack", ) channel = app.channel(schema=schema, key_type="str", value_type="str") assert channel.key_type == "str" assert channel.value_type == "str" assert channel.schema.key_type == "str" assert channel.schema.value_type == "str" assert channel.schema.key_serializer == "msgpack" assert channel.schema.value_serializer == "msgpack"
def test_schema__from_schema(*, app): schema = faust.Schema( key_type='bytes', value_type='bytes', key_serializer='msgpack', value_serializer='msgpack', ) channel = app.channel(schema=schema) assert channel.key_type == 'bytes' assert channel.value_type == 'bytes' assert channel.schema is schema assert channel.schema.key_type == channel.key_type assert channel.schema.value_type == channel.value_type assert channel.schema.key_serializer == 'msgpack' assert channel.schema.value_serializer == 'msgpack'
def test_schema__from_schema(*, app): schema = faust.Schema( key_type="bytes", value_type="bytes", key_serializer="msgpack", value_serializer="msgpack", ) channel = app.channel(schema=schema) assert channel.key_type == "bytes" assert channel.value_type == "bytes" assert channel.schema is schema assert channel.schema.key_type == channel.key_type assert channel.schema.value_type == channel.value_type assert channel.schema.key_serializer == "msgpack" assert channel.schema.value_serializer == "msgpack"
def test_schema__from_schema(self, *, app): schema = faust.Schema( key_type="bytes", value_type="bytes", key_serializer="msgpack", value_serializer="msgpack", ) topic = app.topic("foo", schema=schema) assert topic.key_type == "bytes" assert topic.value_type == "bytes" assert topic.key_serializer == "msgpack" assert topic.value_serializer == "msgpack" assert topic.schema is schema assert topic.schema.key_type == topic.key_type assert topic.schema.value_type == topic.value_type assert topic.schema.key_serializer == "msgpack" assert topic.schema.value_serializer == "msgpack" assert repr(topic.schema)
def test_schema__from_schema(self, *, app): schema = faust.Schema( key_type='bytes', value_type='bytes', key_serializer='msgpack', value_serializer='msgpack', ) topic = app.topic('foo', schema=schema) assert topic.key_type == 'bytes' assert topic.value_type == 'bytes' assert topic.key_serializer == 'msgpack' assert topic.value_serializer == 'msgpack' assert topic.schema is schema assert topic.schema.key_type == topic.key_type assert topic.schema.value_type == topic.value_type assert topic.schema.key_serializer == 'msgpack' assert topic.schema.value_serializer == 'msgpack' assert repr(topic.schema)
# fetcher/models.py import faust from pipeline.app import app class RawUser(faust.Record): id: dict gender: str name: dict location: dict email: str login: dict dob: dict registered: dict phone: str cell: str picture: dict nat: str output_schema = faust.Schema( key_type=str, value_type=RawUser, key_serializer="json", value_serializer="json", ) output_topic = app.topic("fetcher-0.0.1", schema=output_schema)
account_id: str product_id: str amount: int price: float app = faust.App('hello-app', broker='kafka://localhost') orders_kafka_topic = app.topic('orders', key_type=Order, value_type=Order) orders_by_id_topic = app.topic('orders-by-id', key_type=str, value_type=Order) # our table is sharded amongst worker instances, and replicated # with standby copies to take over if one of the nodes fail. order_count_by_account = app.Table('order_count', default=int, key_type=str, schema=faust.Schema(key_type=str, key_serializer="raw")) @app.agent(orders_kafka_topic) async def process(orders: faust.Stream[Order]) -> None: async for order in orders.group_by(Order.account_id, topic=orders_by_id_topic): order_count_by_account[order.account_id] += 1 for k, v in order_count_by_account.items(): print(f"key{k} -> {v}") @app.page('/count/') @app.table_route(table=order_count_by_account, query_param='i') async def get_count(web, request): i = request.query['i']
def test_init_schema_and_channel(self, *, app): with pytest.raises(AssertionError): @app.agent(app.topic('foo'), schema=faust.Schema(key_type=bytes)) async def foo(): ...
KAFKA_CONSUMER_RETRY_TOPIC = os.getenv('KAFKA_CONSUMER_RETRY_TOPIC', 'fano_speech_segmentation.retry') KAFAK_CONSUMER_DLQ_TOPIC = os.getenv('KAFAK_CONSUMER_DLQ_TOPIC', 'fano_speech_segmentation.dlq') KAFKA_CONSUMER_FORWARD_TOPIC = os.getenv('KAFKA_CONSUMER_FORWARD_TOPIC', 'fano_speech_segmentation_2') KAFAK_CONSUMER_RETRY_LIMIT = os.getenv('KAFAK_CONSUMER_RETRY_LIMIT', 3) KAFKA_CONSUMER_MAX_FETCH_SIZE = int(os.getenv('KAFKA_CONSUMER_MAX_FETCH_SIZE', 1073741824)) # The model describes the data sent to our agent, # We will use a JSON serialized dictionary # with two str fields: input, and output. class SegmentMessage(faust.Record): input: str output: str response_schema = faust.Schema( key_type=SegmentMessage, value_type=SegmentMessage, key_serializer='json', value_serializer='json', ) # Next, we create the Faust application object that # configures our environment. app = faust.App( id=KAFKA_CLIENT_ID, broker=KAFKA_BROKERS, processing_guarantee='at_least_once', topic_replication_factor=KAFKA_CONSUMER_TOPIC_REPLICATION_FACTOR, topic_partitions=KAFKA_CONSUMER_TOPIC_PARTITIONS, broker_request_timeout=KAFKA_BROKER_REQUEST_TIMEOUT, broker_session_timeout=KAFKA_BROKER_SESSION_TIMEOUT, broker_heartbeat_interval=KAFKA_BROKER_HEARTBEAT_INTERVAL, broker_max_poll_interval=KAFKA_BROKER_MAX_POLL_INTERVAL,
class QTableEntryUpdateRecord(faust.Record, serializer='json'): vehicle_id: str x_node_id: str y_node_id: str dest_node_id: str min_y_qvalue: float vehicle_time: float ######################################### # topic schemas ######################################### qtable_entry_init_schema = faust.Schema( key_type=str, value_type=InitQTableEntryRecord, key_serializer='json', value_serializer='json' ) vehicle_entry_init_schema = faust.Schema( key_type=str, value_type=InitVehicleEntryRecord, key_serializer='json', value_serializer='json' ) vehicle_status_schema = faust.Schema( key_type=str, value_type=VehicleStatusRecord, key_serializer='json', value_serializer='json'
from faust.serializers import codecs from faust_s3_backed_serializer import S3BackedSerializer import faust broker = "kafka://localhost:9094" input_topic = "texts" app = faust.App("faust-s3-backed-demo", broker=broker) value_serializer = "s3_backed_str_serializer" str_serializer = codecs.get_codec("raw") s3_serializer = S3BackedSerializer() codecs.register(value_serializer, str_serializer | s3_serializer) schema = faust.Schema(key_type=str, key_serializer="raw", value_type=str, value_serializer=value_serializer) texts_topic = app.topic(input_topic, schema=schema) @app.agent(texts_topic) async def print_length(texts): async for key, text in texts.items(): print("{} has {} characters".format(key, len(text))) if __name__ == '__main__': app.main()