def install_produce_command(app) -> None:
@app.command(
option(
"--max-latency",
type=float,
default=0.5,
envvar="PRODUCE_LATENCY",
help="Add delay of (at most) n seconds between publishing.",
),
option(
"--max-messages",
type=int,
default=None,
help="Send at most N messages or 0 for infinity.",
),
)
async def produce(self, max_latency: float, max_messages: int):
"""Produce example events."""
prods = {aiter(p(max_messages)) for p in app.stress_producers}
i = 0
while not app.should_stop:
to_remove: Set[Any] = set()
for producer in prods:
i += 1
try:
await anext(producer)
except StopAsyncIteration:
to_remove.add(producer)
if not max_latency:
# no latency, print every 10,000 messages
if not i % 10000:
self.say(f"+SEND {i}")
else:
# with latency, print every 10 messages
if not i % 10:
self.say(f"+SEND {i}")
if not prods:
await asyncio.sleep(1.0)
if max_latency:
await asyncio.sleep(random.uniform(0, max_latency))
for producer in to_remove:
prods.discard(producer)
print("No more producers - exiting", file=sys.stderr)
# 2) wait for order to be sent to Kafka
await self.order_sent_to_kafka.wait(timeout=30.0)
# 3) wait for redis index to be updated.
await self.order_cache_in_redis.wait(timeout=30.0)
# make sure it's now actually in redis
assert await livecheck.cache.client.sismember(
f"order.{order.user_id}.orders", order.id)
# 4) wait for execution agent to execute the order.
await self.order_executed.wait(timeout=30.0)
async def make_fake_request(self) -> None:
await self.get_url("http://localhost:6066/order/init/sell/?fake=1")
@app.command(
cli.option("--side", default="sell", help="Order side: buy, sell"),
cli.option("--base-url", default="http://localhost:6066"),
)
async def post_order(self: cli.AppCommand, side: str, base_url: str) -> None:
path = self.app.web.url_for("orders:init", side=side)
url = "".join([base_url.rstrip("/"), path])
async with self.app.http_client.get(url) as response:
response.raise_for_status()
print(await response.read())
if __name__ == "__main__":
app.main()
@app.agent(withdrawals_topic)
async def track_user_withdrawal(withdrawals):
async for withdrawal in withdrawals:
user_to_total[withdrawal.user] += withdrawal.amount
@app.agent()
async def track_country_withdrawal(withdrawals):
async for withdrawal in withdrawals.group_by(Withdrawal.country):
country_to_total[withdrawal.country] += withdrawal.amount
@app.command(
option('--max-latency',
type=float,
default=0.5,
envvar='PRODUCE_LATENCY',
help='Add delay of (at most) n seconds between publishing.'),
option('--max-messages',
type=int,
default=None,
help='Send at most N messages or 0 for infinity.'),
)
async def produce(self, max_latency: float, max_messages: int):
"""Produce example Withdrawal events."""
for i, withdrawal in enumerate(generate_withdrawals(max_messages)):
await withdrawals_topic.send(key=withdrawal.user, value=withdrawal)
if not i % 10000:
self.say(f'+SEND {i}')
if max_latency:
await asyncio.sleep(random.uniform(0, max_latency))
class Benchmark:
_agent = None
_produce = None
produce_options = [
option('--max-latency',
type=float,
default=0.0,
envvar='PRODUCE_LATENCY',
help='Add delay of (at most) n seconds between publishing.'),
option('--max-messages',
type=int,
default=None,
help='Send at most N messages or 0 for infinity.'),
]
def __init__(self, app, topic, *, n=TIME_EVERY, consume_topic=None):
self.app = app
self.topic = topic
if consume_topic is None:
consume_topic = topic
self.consume_topic = consume_topic
self.n = n
self.app.finalize()
self.app.conf.producer_acks = ACKS
self.app.conf.producer_max_batch_size = BSIZE
self.app.conf.producer_linger = LINGER
def install(self, main_name):
self.create_benchmark_agent()
self.create_produce_command()
if main_name == '__main__':
# we use this for cProfile
bench_args = {
'worker': ['worker', '-l', 'info'],
'produce': ['produce'],
}
if len(sys.argv) < 2:
sys.argv.extend(bench_args[BENCH_TYPE])
self.app.main()
def create_benchmark_agent(self):
self._agent = self.app.agent(self.consume_topic)(self.process)
async def process(self, stream):
time_last = None
async for i, value in stream.enumerate():
if not i:
time_last = monotonic()
await self.process_value(value)
if i and not i % self.n: # noqa: S001
now = monotonic()
runtime, time_last = now - time_last, now
print(f'RECV {i} in {runtime}s')
async def process_value(self, value):
# put worker benchmark-specific code in here
...
def create_produce_command(self):
self._produce = self.app.command(*self.produce_options)(self.produce)
async def produce(self, max_latency: float, max_messages: int, **kwargs):
i = 0
time_start = None
app = self.app
topic = self.topic
for i, (key, value) in enumerate(self.generate_values(max_messages)):
callback = None
if not i:
time_start = monotonic()
time_1st = monotonic()
def on_published(meta):
print(f'1ST OK: {meta} AFTER {monotonic() - time_1st}s')
callback = on_published
await topic.send(key=key, value=value, callback=callback)
if i and not i % self.n: # noqa: S001
print(f'+SEND {i} in {monotonic() - time_start}s')
time_start = monotonic()
if max_latency:
await asyncio.sleep(random.uniform(0, max_latency))
await asyncio.sleep(10)
await app.producer.stop()
print(f'Time spent on {i} messages: {monotonic() - time_start}s')
def generate_values(self, max_messages: int):
# override this to generate other types of values
return (
# convert int to bytes, cannot use bytes(int) as that gives
# character code in Python3.
(None, str(i).encode())
for i in (range(max_messages) if max_messages else count()))
import logging
from faust.cli import option
from app import app
logger = logging.getLogger(__name__)
@app.command(option("--answer", type=str, help="Send your question to the server", required=False))
async def answer(self, track: str):
""" Do something here """
logger.info("Command invoked ")
import uuid
from typing import Tuple
from faust.cli import AppCommand, option
from visit_aggregator.app import app
from visit_aggregator.models import Visit
logger: logging.Logger = logging.getLogger("visit_aggregator")
@app.command(
option(
"--frequency",
type=float,
default=1,
help="The frequency in which messages are produced per second.",
show_default=True,
),
option(
"--max-messages",
type=int,
default=300,
help="The maximum number of messages to produce.",
show_default=True,
),
option(
"--account-id",
type=int,
default=(352, ),
multiple=True,
from faust import cli
from ..app import app
@app.command(
cli.option('--side', default='sell', help='Order side: buy, sell'),
cli.option('--base-url', default='http://localhost:6066'),
)
async def post_order(self: cli.AppCommand, side: str, base_url: str) -> None:
path = self.app.web.url_for('orders:init', side=side)
url = ''.join([base_url.rstrip('/'), path])
async with self.app.http_client.get(url) as response:
response.raise_for_status()
self.say(await response.read())
package_index.url,
)
except Exception as exc:
_LOGGER.exception(
f"Failed to do release notification for {package_name} ({package_version} "
f"from {package_index.url}), error is not fatal: {str(exc)}"
)
return async_tasks, package_releases_messages_sent
@app.command(
cli.option(
"-v",
"--verbose",
is_flag=True,
envvar="THOTH_PACKAGE_RELEASES_DEBUG",
help="Be verbose about what's going on.",
),
cli.option(
"--version",
is_flag=True,
is_eager=True,
callback=_print_version,
expose_value=False,
help="Print version and exit.",
),
cli.option(
"--monitoring-config",
"-m",
type=str,
from app import app, events_topic
from random import randint
from faust.cli import option
@app.command(option('--max-events', type=int, required=True))
async def send_events(self, max_events:int):
print(f'Sending {max_events} events...')
half_events = max_events // 2
for i in range(max_events):
await events_topic.send(key=str(randint(1, 1500000)), value=randint(0, 1000))
if i == half_events:
print(f'Half amount of events sent: {i}')
import logging
from typing import List
import peony
from faust.cli import option
from peony import PeonyClient
from src.app import app, hashtags_topic
from src.settings import ACCESS_TOKEN, ACCESS_TOKEN_SECRET, CONSUMER_KEY, CONSUMER_SECRET
logger = logging.getLogger(__name__)
@app.command(option("--track", type=str, help="List of words in CSV format to filter stream of tweets", required=True))
async def hashtags_events_generator(self, track: str):
"""
Generate Kafka events for each hashtag found in tweets matching with the track used to filter the stream.
"""
logger.info(f"Searching tweets with track: {track}")
client = _get_twitter_client()
req = client.stream.statuses.filter.post(track=track)
async with req as stream:
async for tweet in stream:
tweet_hashtags = _get_hashtags_from_tweet(tweet=tweet)
await _produce_events_for_hashtags(hashtags=tweet_hashtags)
def _get_twitter_client() -> PeonyClient:
creds = dict(
)
from kafkaaggregator.generator import AgentGenerator
logger = logging.getLogger("kafkaaggregator")
def main() -> None:
"""Entrypoint for Faust CLI."""
app.main()
@app.command(
option(
"--frequency",
type=float,
default=config.frequency,
help="The frequency in Hz in wich messages are produced.",
show_default=True,
),
option(
"--max-messages",
type=int,
default=config.max_messages,
help="The maximum number of messages to produce.",
show_default=True,
),
)
async def produce(self: AppCommand, frequency: float,
max_messages: int) -> None:
"""Produce messages for the aggregation example."""
example = AggregationExample()
app = MessageBase().app
init_logging()
_LOGGER = logging.getLogger("thoth.messaging")
__service_version__ = f"{__version__}+common.{__common__version__}"
_LOGGER.info("This is Thoth Messaging CLI v%s", __service_version__)
## create cli
@app.command(
cli.option(
"--topic-name",
"-n",
envvar="THOTH_MESSAGING_TOPIC_NAME",
type=str,
help="Name of topic to send message to.",
required=False,
),
cli.option(
"--create-if-not-exist",
envvar="THOTH_MESSAGING_CREATE_IF_NOT_EXIST",
default=False,
help="If topic doesn't already exist on Kafka then create it.",
flag_value=True,
),
cli.option(
"--message-contents",
"-m",
envvar="THOTH_MESSAGING_MESSAGE_CONTENTS",
type=str,
