def test_schema__overriding(self, *, app):
schema = faust.Schema(
key_type="bytes",
value_type="bytes",
key_serializer="msgpack",
value_serializer="msgpack",
)
topic = app.topic(
"foo",
schema=schema,
key_type="str",
value_type="str",
key_serializer="captnproto",
value_serializer="captnproto",
)
assert topic.key_type == "str"
assert topic.value_type == "str"
assert topic.key_serializer == "captnproto"
assert topic.value_serializer == "captnproto"
assert topic.schema.key_type == "str"
assert topic.schema.value_type == "str"
assert topic.schema.key_serializer == "captnproto"
assert topic.schema.value_serializer == "captnproto"
assert repr(topic.schema)
def test_schema__overriding(self, *, app):
schema = faust.Schema(
key_type='bytes',
value_type='bytes',
key_serializer='msgpack',
value_serializer='msgpack',
)
topic = app.topic(
'foo',
schema=schema,
key_type='str',
value_type='str',
key_serializer='captnproto',
value_serializer='captnproto',
)
assert topic.key_type == 'str'
assert topic.value_type == 'str'
assert topic.key_serializer == 'captnproto'
assert topic.value_serializer == 'captnproto'
assert topic.schema.key_type == 'str'
assert topic.schema.value_type == 'str'
assert topic.schema.key_serializer == 'captnproto'
assert topic.schema.value_serializer == 'captnproto'
assert repr(topic.schema)
def test_schema__overriding(*, app):
schema = faust.Schema(
key_type='bytes',
value_type='bytes',
key_serializer='msgpack',
value_serializer='msgpack',
)
channel = app.channel(schema=schema, key_type='str', value_type='str')
assert channel.key_type == 'str'
assert channel.value_type == 'str'
assert channel.schema.key_type == 'str'
assert channel.schema.value_type == 'str'
assert channel.schema.key_serializer == 'msgpack'
assert channel.schema.value_serializer == 'msgpack'
def test_schema__overriding(*, app):
schema = faust.Schema(
key_type="bytes",
value_type="bytes",
key_serializer="msgpack",
value_serializer="msgpack",
)
channel = app.channel(schema=schema, key_type="str", value_type="str")
assert channel.key_type == "str"
assert channel.value_type == "str"
assert channel.schema.key_type == "str"
assert channel.schema.value_type == "str"
assert channel.schema.key_serializer == "msgpack"
assert channel.schema.value_serializer == "msgpack"
def test_schema__from_schema(*, app):
schema = faust.Schema(
key_type='bytes',
value_type='bytes',
key_serializer='msgpack',
value_serializer='msgpack',
)
channel = app.channel(schema=schema)
assert channel.key_type == 'bytes'
assert channel.value_type == 'bytes'
assert channel.schema is schema
assert channel.schema.key_type == channel.key_type
assert channel.schema.value_type == channel.value_type
assert channel.schema.key_serializer == 'msgpack'
assert channel.schema.value_serializer == 'msgpack'
def test_schema__from_schema(*, app):
schema = faust.Schema(
key_type="bytes",
value_type="bytes",
key_serializer="msgpack",
value_serializer="msgpack",
)
channel = app.channel(schema=schema)
assert channel.key_type == "bytes"
assert channel.value_type == "bytes"
assert channel.schema is schema
assert channel.schema.key_type == channel.key_type
assert channel.schema.value_type == channel.value_type
assert channel.schema.key_serializer == "msgpack"
assert channel.schema.value_serializer == "msgpack"
def test_schema__from_schema(self, *, app):
schema = faust.Schema(
key_type="bytes",
value_type="bytes",
key_serializer="msgpack",
value_serializer="msgpack",
)
topic = app.topic("foo", schema=schema)
assert topic.key_type == "bytes"
assert topic.value_type == "bytes"
assert topic.key_serializer == "msgpack"
assert topic.value_serializer == "msgpack"
assert topic.schema is schema
assert topic.schema.key_type == topic.key_type
assert topic.schema.value_type == topic.value_type
assert topic.schema.key_serializer == "msgpack"
assert topic.schema.value_serializer == "msgpack"
assert repr(topic.schema)
def test_schema__from_schema(self, *, app):
schema = faust.Schema(
key_type='bytes',
value_type='bytes',
key_serializer='msgpack',
value_serializer='msgpack',
)
topic = app.topic('foo', schema=schema)
assert topic.key_type == 'bytes'
assert topic.value_type == 'bytes'
assert topic.key_serializer == 'msgpack'
assert topic.value_serializer == 'msgpack'
assert topic.schema is schema
assert topic.schema.key_type == topic.key_type
assert topic.schema.value_type == topic.value_type
assert topic.schema.key_serializer == 'msgpack'
assert topic.schema.value_serializer == 'msgpack'
assert repr(topic.schema)
# fetcher/models.py
import faust
from pipeline.app import app
class RawUser(faust.Record):
id: dict
gender: str
name: dict
location: dict
email: str
login: dict
dob: dict
registered: dict
phone: str
cell: str
picture: dict
nat: str
output_schema = faust.Schema(
key_type=str,
value_type=RawUser,
key_serializer="json",
value_serializer="json",
)
output_topic = app.topic("fetcher-0.0.1", schema=output_schema)
account_id: str
product_id: str
amount: int
price: float
app = faust.App('hello-app', broker='kafka://localhost')
orders_kafka_topic = app.topic('orders', key_type=Order, value_type=Order)
orders_by_id_topic = app.topic('orders-by-id', key_type=str, value_type=Order)
# our table is sharded amongst worker instances, and replicated
# with standby copies to take over if one of the nodes fail.
order_count_by_account = app.Table('order_count',
default=int,
key_type=str,
schema=faust.Schema(key_type=str,
key_serializer="raw"))
@app.agent(orders_kafka_topic)
async def process(orders: faust.Stream[Order]) -> None:
async for order in orders.group_by(Order.account_id,
topic=orders_by_id_topic):
order_count_by_account[order.account_id] += 1
for k, v in order_count_by_account.items():
print(f"key{k} -> {v}")
@app.page('/count/')
@app.table_route(table=order_count_by_account, query_param='i')
async def get_count(web, request):
i = request.query['i']
def test_init_schema_and_channel(self, *, app):
with pytest.raises(AssertionError):
@app.agent(app.topic('foo'), schema=faust.Schema(key_type=bytes))
async def foo():
...
KAFKA_CONSUMER_RETRY_TOPIC = os.getenv('KAFKA_CONSUMER_RETRY_TOPIC', 'fano_speech_segmentation.retry')
KAFAK_CONSUMER_DLQ_TOPIC = os.getenv('KAFAK_CONSUMER_DLQ_TOPIC', 'fano_speech_segmentation.dlq')
KAFKA_CONSUMER_FORWARD_TOPIC = os.getenv('KAFKA_CONSUMER_FORWARD_TOPIC', 'fano_speech_segmentation_2')
KAFAK_CONSUMER_RETRY_LIMIT = os.getenv('KAFAK_CONSUMER_RETRY_LIMIT', 3)
KAFKA_CONSUMER_MAX_FETCH_SIZE = int(os.getenv('KAFKA_CONSUMER_MAX_FETCH_SIZE', 1073741824))
# The model describes the data sent to our agent,
# We will use a JSON serialized dictionary
# with two str fields: input, and output.
class SegmentMessage(faust.Record):
input: str
output: str
response_schema = faust.Schema(
key_type=SegmentMessage,
value_type=SegmentMessage,
key_serializer='json',
value_serializer='json',
)
# Next, we create the Faust application object that
# configures our environment.
app = faust.App(
id=KAFKA_CLIENT_ID,
broker=KAFKA_BROKERS,
processing_guarantee='at_least_once',
topic_replication_factor=KAFKA_CONSUMER_TOPIC_REPLICATION_FACTOR,
topic_partitions=KAFKA_CONSUMER_TOPIC_PARTITIONS,
broker_request_timeout=KAFKA_BROKER_REQUEST_TIMEOUT,
broker_session_timeout=KAFKA_BROKER_SESSION_TIMEOUT,
broker_heartbeat_interval=KAFKA_BROKER_HEARTBEAT_INTERVAL,
broker_max_poll_interval=KAFKA_BROKER_MAX_POLL_INTERVAL,
class QTableEntryUpdateRecord(faust.Record, serializer='json'):
vehicle_id: str
x_node_id: str
y_node_id: str
dest_node_id: str
min_y_qvalue: float
vehicle_time: float
#########################################
# topic schemas
#########################################
qtable_entry_init_schema = faust.Schema(
key_type=str,
value_type=InitQTableEntryRecord,
key_serializer='json',
value_serializer='json'
)
vehicle_entry_init_schema = faust.Schema(
key_type=str,
value_type=InitVehicleEntryRecord,
key_serializer='json',
value_serializer='json'
)
vehicle_status_schema = faust.Schema(
key_type=str,
value_type=VehicleStatusRecord,
key_serializer='json',
value_serializer='json'
from faust.serializers import codecs
from faust_s3_backed_serializer import S3BackedSerializer
import faust
broker = "kafka://localhost:9094"
input_topic = "texts"
app = faust.App("faust-s3-backed-demo", broker=broker)
value_serializer = "s3_backed_str_serializer"
str_serializer = codecs.get_codec("raw")
s3_serializer = S3BackedSerializer()
codecs.register(value_serializer, str_serializer | s3_serializer)
schema = faust.Schema(key_type=str,
key_serializer="raw",
value_type=str,
value_serializer=value_serializer)
texts_topic = app.topic(input_topic, schema=schema)
@app.agent(texts_topic)
async def print_length(texts):
async for key, text in texts.items():
print("{} has {} characters".format(key, len(text)))
if __name__ == '__main__':
app.main()
