#!/usr/bin/env python
import random
import faust
app = faust.App(
'leader-example',
broker='kafka://localhost:9092',
value_serializer='raw',
)
@app.timer(2.0, on_leader=True)
async def publish_greetings():
print('PUBLISHING ON LEADER!')
await say.send(value=str(random.random()))
@app.agent()
async def say(greetings):
async for greeting in greetings:
print(greeting)
class Purchase(faust.Record):
username: str
currency: str
amount: int
fraud_score: float = 0.0
#
# TODO: Define a function which adds a fraud score to incoming records
#
def add_fraud_score(value):
value.fraud_score = random.random()
return value
app = faust.App("hello-faust", broker="kafka://localhost:9092")
purchases_topic = app.topic("com.udacity.streams.purchases", value_type=Purchase)
#
# TODO: Define a stream
# TODO: Assign processors to incoming records
#
purchases_stream = app.stream(...)
#
# TODO: Enumerate the purchases stream
#
async for purchase in purchases_stream:
print(json.dumps(asdict(purchase), indent=2))
blue: bool
green: bool
# Faust will produce records to Kafka in this format
class TransformedStation(faust.Record):
station_id: int
station_name: str
order: int
line: str
# TODO: Define a Faust Stream that ingests data from the Kafka Connect stations topic and
# places it into a new topic with only the necessary information.
app = faust.App("stations-stream",
broker="kafka://localhost:9092",
store="memory://")
# TODO: Define the input Kafka Topic. Hint: What topic did Kafka Connect output to?
topic = app.topic("org.chicago.cta.stations", value_type=Station)
# TODO: Define the output Kafka Topic
out_topic = app.topic("org.chicago.cta.stations.table.v1", partitions=1)
# TODO: Define a Faust Table
table = app.Table(
"org.chicago.cta.stations.table.v1",
default=TransformedStation,
partitions=1,
changelog_topic=out_topic,
)
#
#######################################################################
# REQ: Create Kafka stream processing in python to read message from
# topic and to make external API call to send out notification
#
# TODO: Add logic to parse AVRO messages
#
# SEE: https://medium.com/swlh/how-to-deserialize-avro-messages-in-python-faust-400118843447
# https://www.python.org/dev/peps/pep-0350/
#
# STAT: Inprogress
#######################################################################
# NOTE: To debug set consumer name to "'weather-alert-notify-' + str(time.time())"
app = faust.App('weather-alert-notify',
broker='kafka://broker:9092',
value_serializer='raw')
json_topic = app.topic('STM_WEATHER_ALERT_APP_9000_NOTIFY',
value_type=bytes,
key_type=str)
# Faust stream processing
@app.agent(json_topic)
async def processor(records):
async for record in records:
des_data = json.loads(record)
print(des_data)
# Account details from twilio.com
import faust
from numpy.random import default_rng
rng = default_rng(12345)
app = faust.App("sample", broker="kafka://localhost")
class Record(faust.Record, serializer="json"):
id_string: str
value: float
topic = app.topic("sample-topic", value_type=Record)
@app.agent(topic)
async def process_record(records):
async for record in records:
print(f"Got {record.id_string}: {record.value}")
@app.timer(interval=1.0)
async def producer1(app):
await app.send("sample-topic",
value=Record(id_string="producer 1",
value=rng.uniform(0, 2)))
@app.timer(interval=2.0)
async def producer2(app):
import faust
app = faust.App(
'proj',
origin='proj',
autodiscover=True,
)
#!/usr/bin/env python
import faust
app = faust.App(
'hello-world',
broker='kafka://localhost:9092',
)
greetings_topic = app.topic('greetings', value_type=str)
@app.agent(greetings_topic)
async def print_greetings(greetings):
async for greeting in greetings:
print(greeting)
@app.timer(5)
async def produce():
for i in range(100):
await print_greetings.send(value=f'hello {i}')
if __name__ == '__main__':
app.main()
import faust
from typing import List
import requests
import json
app = faust.App(
'flight_prediction_app',
broker='kafka://localhost:9092',
value_serializer='raw',
)
kafka_topic = app.topic('flight')
@app.agent(kafka_topic)
async def process(flights):
async for value in flights:
result = requests.post('http://127.0.0.1:5000/invocations',
json=json.loads(value))
print('Input data: ' + str(value))
print('Flight arrival delay result: ' + str(result.json()))
if __name__ == '__main__':
# run the consumer
app.main()
#!/usr/bin/env python
import asyncio
import faust
WORDS = ['the', 'quick', 'brown', 'fox']
app = faust.App(
'word-counts',
broker='kafka://localhost:9092',
store='rocksdb://',
version=1,
topic_partitions=8,
)
posts_topic = app.topic('posts', value_type=str)
word_counts = app.Table('word_counts',
default=int,
help='Keep count of words (str to int).')
@app.agent(posts_topic)
async def shuffle_words(posts):
async for post in posts:
for word in post.split():
await count_words.send(key=word, value=word)
@app.agent(value_type=str)
async def count_words(words):
"""Count words from blog post article body."""
async for word in words:
#faust -A f_getlive worker -l info --web-port 6066
import faust, json
import asyncpg, asyncio
loop = asyncio.get_event_loop()
async def init_app():
pool = await asyncpg.create_pool(
'postgres://*****:*****@postgres:5432/data')
print('sec')
return pool
pool = loop.run_until_complete(init_app())
app = faust.App('tester',
key_serializer='raw',
value_serializer='raw',
broker='kafka://broker:29092')
topic = app.topic('tester', value_type=bytes)
@app.agent(topic)
async def processor(stream):
async for e in stream.events():
# async for payload in stream:
print(e.key.decode('UTF-8'))
print(e.message.timestamp)
data = json.loads(e.value)
print(data)
def create_app():
return faust.App(
"proj323",
origin="proj323",
autodiscover=True,
)
3. Subscribe to the topic contained cleaned data and calculate
some data statistics over a window of time.
"""
import os
import faust
from datetime import timedelta
from data_cleaner import perform_full_clean
KAFKA_BROKER_URL = os.environ.get('KAFKA_BROKER_URL', 'broker:9092')
USER_TOPIC = os.environ.get('USER_TOPIC')
CLEANED_USER_TOPIC = os.environ.get('CLEANED_USER_TOPIC')
app = faust.App('vodka',
broker=KAFKA_BROKER_URL,
value_serializer='json',
consumer_auto_offset_reset='latest')
# raw data topic
user_topic = app.topic(USER_TOPIC)
# topic contains cleaned data
cleaned_user_topic = app.topic(CLEANED_USER_TOPIC, internal=True, partitions=1)
# Faust windowed table, serve as storage for
# statistics calculation
# refer to https://faust.readthedocs.io/en/latest/userguide/tables.html
counts = app.Table('user_activity_count',
partitions=1,
key_type=str,
default=int).tumbling(timedelta(minutes=1),
expires=timedelta(minutes=5),
import faust
import asyncio
import json
# from ..elastic_search import elasticsearch_consumer
# from ..elastic_search import elastic_twitter_producer
# from ..elastic_search import TwitterProducer
# from ..elastic_search.elastic_twitter_producer import TwitterProducer
# from ..elastic_search.elastic_twitter_producer import TwitterProducer
class StreamFilterTweets(faust.Record):
text: str
followers: int
app = faust.App('myapp', broker='kafka://localhost')
topic = app.topic('twitter_topics', key_type='bytes', partitions=3)
@app.agent(
value_type=StreamFilterTweets, )
async def confirm_followers(tweets, serializer='json'):
print('NOW INSIDE AGENT!!', app.topic)
async for tweet in tweets:
print('this is the tweet, ladies and gentlemen>>>>>>>>>>', tweet)
the_tweet = json.loads(tweet)
if the_tweet['followers'] > 2000:
print('WE FOUND SOMEONE!!!!!!!')
if __name__ == '__main__':
import json
import settings
class State(faust.Record, serializer='json'):
end: bool
good: int
reject: int
total: int
optimum: int
runtime: int
availability: float
id: str
app = faust.App(settings.PERFORMANCE_TOPIC, broker='kafka://master.cluster2:9092')
app_topic = app.topic(settings.PERFORMANCE_TOPIC, value_type=State)
performance = app.Table(settings.PERFORMANCE_TABLE, default=lambda: 1.0)
kafka_producer = utils.connect_kafka_producer()
@app.agent(app_topic)
async def consume(states):
with open('data/performance.msgpack', 'rb') as f:
data = msgpack.unpackb(f.read())
async for state in states.group_by(State.id):
try:
if not state.end:
performance[state.id] -= 1 / settings.BATCH_SIZE * state.total / state.optimum
else:
performance[state.id] = 1
import faust
app = faust.App(
'word_count',
broker='kafka://kafka1:9092',
value_serializer='raw',
)
test_topic = app.topic('test_topic')
word_count_topic = app.Table('word_count_table', default=int)
print('Starting')
@app.agent(test_topic)
async def process(stream):
async for event in stream:
# split sentences into words
for word in event.split():
# word_event = ???
yield word.forward(word_count_topic)
print('Going out the end')
order: int
red: bool
blue: bool
green: bool
# Faust will produce records to Kafka in this format
class TransformedStation(faust.Record):
station_id: int
station_name: str
order: int
line: str
# Faust Stream that ingests data from the Kafka Connect stations topic and places it into a new topic with only the necessary information.
app = faust.App("stations-stream", broker="kafka://localhost:9092")
# Input Topic
topic = app.topic("jdbc.stations", value_type=Station)
# Output Topics
out_topic = app.topic("faust.stations.transformed",
partitions=1,
value_type=TransformedStation)
table = app.Table(
"stations.transformation.table",
default=int,
partitions=1,
changelog_topic=out_topic,
)
# seconds of duration we expect different the following results per method
# called in `WindowWrapper`:
# - now(): Gets the closest value to current local time. It will always be
# between 1 and 3.
# - current(): Gets the value relative to the event's timestamp. It will
# always be between 1 and 3.
# - value(): Gets the value relative to default relative option. It will
# always be between 1 and 3.
# - delta(30): Gets the value of window 30 secs before the current event. For
# the first 20 seconds it will be 0 and after second 30 it will always be 5.
from random import random
from datetime import timedelta
import faust
app = faust.App('windowing', broker='kafka://localhost:9092')
class Model(faust.Record, serializer='json'):
random: float
TOPIC = 'hopping_topic'
hopping_topic = app.topic(TOPIC, value_type=Model)
hopping_table = app.Table(
'hopping_table',
default=int).hopping(10, 5, expires=timedelta(minutes=10))
@app.agent(hopping_topic)
import json
import utils
import json
import settings
class State(faust.Record, serializer='json'):
end: bool
good: int
reject: int
total: int
optimum: int
id: str
app = faust.App(settings.AVAILABILITY_TOPIC,
broker='kafka://master.cluster2:9092')
app_topic = app.topic(settings.AVAILABILITY_TOPIC, value_type=State)
n_downtime = app.Table(settings.N_DOWNTIME_TABLE, default=int)
availability = app.Table(settings.AVAILABILITY_TABLE, default=lambda: 1.0)
runtime = app.Table(settings.RUNTIME_TABLE,
default=lambda: settings.TOTAL_TIME)
kafka_producer = utils.connect_kafka_producer()
@app.agent(app_topic)
async def consume(states):
with open('data/availability.msgpack', 'rb') as f:
data = msgpack.unpackb(f.read())
async for state in states.group_by(State.id):
try:
from typing import Any, Mapping
import faust
from confluent_kafka.admin import AdminClient
from faust.types import TP
from faust.types.tables import TableT
from gs_framework.faust_utilities import FaustUtilities
from gs_framework.state_stream import ObjectStateStream
from gs_framework.topic_channel_wrapper import TopicWrapper
from gs_framework.utilities import bytes_2_object, get_random_str, object_2_bytes
topic2_define = faust.types.TP("martin-gs_framework_test-guess-number-3", 1)
app_listen = faust.App("martin-gs_framework_test-app",
broker="kafka://gftoffice.sedns.cn:31090",
store="rocksdb://",
datadir="/opt/gsfaust")
# FaustUtilities.Admin.check_and_create_topic(topic2_define)
topic2 = app_listen.topic(topic2_define.topic,
key_type=str,
value_type=str,
partitions=topic2_define.partition)
async def agent_function(stream):
async for event in stream.events():
message = event.message
print(f"key={message.key}, value={message.value}")
import aiohttp
from typing import Union
import json
import deviceController
import asyncio
# Variaveis de ambiente
TB_HOST = 'thingsboard'
KAFKA_HOST = 'kafka'
APP_NAME = 'conectorKafktaThingsboard'
TOPIC_NAME = 'weatherstation.sensor'
# Instância do faust
app = faust.App(
APP_NAME,
broker='kafka://'+KAFKA_HOST+':29092'
)
# Modelo da mensagens do Kafka
class MessageSensorModel(faust.Record, serializer='json'):
timestamp: int
stationCode: str
temperature: float
humidity: float
latitude: str
longitude: str
class MessageHeatModel(faust.Record, serializer='json'):
timestamp: int
from typing import List, Optional, AsyncIterable
import random
import uuid
import faust
from kafka import KafkaProducer
app = faust.App("demo")
producer = KafkaProducer()
class CustomerRecord(faust.Record, serializer="json"):
msg_id: uuid.UUID
customer: str
amount: int
number_of_purchases: int
note: Optional[str]
customer_topic = app.topic("customer_topic", value_type=CustomerRecord)
completed_customer_topic = app.topic("completed_customer_topic",
value_type=CustomerRecord)
def ring_up_customer(customer):
customer.number_of_purchases += 1
purchase_price = random.randint(1, 100)
customer.amount += purchase_price
print(f"{customer.customer} spent {purchase_price}")
from itertools import count
import faust
from faust.cli import option
class Withdrawal(faust.Record, isodates=True, serializer='json'):
user: str
country: str
amount: float
date: datetime = None
app = faust.App(
'faust-withdrawals',
broker='kafka://127.0.0.1:9092',
store='rocksdb://',
origin='examples.withdrawals',
topic_partitions=8,
)
app.producer_only = True
withdrawals_topic = app.topic('withdrawals', value_type=Withdrawal)
user_to_total = app.Table(
'user_to_total',
default=int,
).tumbling(3600).relative_to_stream()
country_to_total = app.Table(
'country_to_total',
default=int,
).tumbling(10.0, expires=10.0).relative_to_stream()
import faust
import sys
sys.path.insert(1, '/Users/zharas/Desktop/ttask/shared_functions')
from funtion_a import Products
app = faust.App('hit_counter', broker="kafka://localhost:29092")
class productCount(faust.Record, validation=True):
products: int
productId: int
hit_topic = app.topic("hit_count", value_type=productCount)
extra_topic = app.topic('extra_topic', internal=True, value_type=productCount)
count_table = app.Table("major-count",
key_type=str,
value_type=int,
partitions=1,
default=int)
@app.agent(hit_topic)
async def count_hits(counts):
async for count in counts:
print(f"Data recieved is {count}")
if Products.function(count.products):
await extra_topic.send(value=count)
import socket
import sys
import uuid
from datetime import timedelta
from enum import Enum
from pathlib import Path
from typing import Any, Dict, IO, Iterator, NamedTuple, Set
import aiohttp
import faust
import faust.transport.utils
import mode
from yarl import URL
SETTINGS: Path = Path('docs/userguide/settings.rst')
app = faust.App('verify_defaults')
ignore_settings: Set[str] = {'id', 'tabledir', 'reply_to'}
builtin_locals: Dict[str, Any] = {
'aiohttp': aiohttp,
'app': app,
'datetime': datetime,
'datadir': app.conf.datadir,
'faust': faust,
'logging': logging,
'mode': mode,
'socket': socket,
'timedelta': timedelta,
'web_host': socket.gethostname(),
'web_port': 6066,
import faust
from simple_settings import settings
from logging.config import dictConfig
app = faust.App(
version=1, # fmt: off
autodiscover=True,
origin="replay_output_experiment",
id="1",
broker=settings.KAFKA_BOOTSTRAP_SERVER,
logging_config=dictConfig(settings.LOGGING),
)
def main() -> None:
app.main()
import faust
app = faust.App(
'page_views',
broker='kafka://localhost:9092',
topic_partitions=4,
)
class PageView(faust.Record):
id: str
user: str
page_view_topic = app.topic('page_views', value_type=PageView)
page_views = app.Table('page_views', default=int)
@app.agent(page_view_topic)
async def count_page_views(views):
async for view in views.group_by(PageView.id):
page_views[view.id] += 100
# Please complete the TODO items in the code
from dataclasses import asdict, dataclass
import json
import faust
@dataclass
class ClickEvent(faust.Record):
email: str
timestamp: str
number: int
app = faust.App("exercise2", broker="kafka://localhost:9092")
clickevents_topic = app.topic(
"com.udacity.streams.clickevents",
key_type=str,
value_type=ClickEvent,
)
@app.agent(clickevents_topic)
async def clickevent(clickevents):
async for ce in clickevents:
print(json.dumps(asdict(ce), indent=2))
if __name__ == "__main__":
app.main()
"""
import faust
from dataclasses import dataclass, asdict
import json
""" A stream is an infinite async iterable, consuming messages from a channel/topic """
@dataclass
class Purchase(faust.Record):
username: str
currency: str
amount: int
app = faust.App("FaustStreams", broker="localhost:9092")
topic = app.topic("com.udacity.streams.purchases",
key_type=str,
value_type=Purchase)
out_topic = app.topic("com.udacity.streams.purchases.high_value_international",
key_type=str,
value_type=Purchase,
value_serializer="json")
@app.agent(topic)
async def purchase(purchases):
async for purchase in purchases.filter(
lambda x: x.currency != "USD" and x.amount > 100000):
await out_topic.send(key=purchase.username, value=purchase)
#: Positional arguments to the task.
arguments: Sequence
#: Keyword arguments to the task.
keyword_arguments: Mapping
async def __call__(self) -> Any:
return await self.handler(*self.arguments, **self.keyword_arguments)
@property
def handler(self) -> Callable[..., Awaitable]:
return task_registry[self.name]
app = faust.App('faust-celery', broker='kafka://localhost')
task_queue_topic = app.topic('tasks', value_type=Request)
task_registry: MutableMapping[str, Callable[..., Awaitable]]
task_registry = {}
@app.agent(task_queue_topic)
async def process_task(tasks: faust.Stream[Request]) -> None:
"""A "worker" stream processor that executes tasks."""
async for task in tasks:
print(f'Processing task: {task!r}')
result = await task()
print(f'Result of {task.id} is: {result!r}')
continue
# 도배 글 쓴 사람이 같다면 -> ip 와 id 가 같은 사람.. 지수 + 1, id를 바꿔서 도배.. 지수 + 100
if document['ip'] == ip and document['id'] == user_name:
spam_cnt += 0.01
elif document['ip'] == ip and document['id'] != user_name:
spam_cnt += 1
# print('예전 글 작성 시간 : ' + document['date'])
# print('방금 들어온 글 작성 시간 : ' + write_time)
# print(spam_title)
update_user_list(ip, user_name, spam_cnt)
app = faust.App('nf-worker-2',
broker='kafka://49.50.174.75:9092',
broker_max_poll_record=3)
def get_spam_cnt(ip, user_name, spam_cnt):
res = es.search(index="naver.finance.board",
body={
"aggs": {
"same_ip_group": {
"terms": {
"field": ip,
"size": 100
},
"aggs": {
"same_body_group": {
