def install_produce_command(app) -> None: @app.command( option( "--max-latency", type=float, default=0.5, envvar="PRODUCE_LATENCY", help="Add delay of (at most) n seconds between publishing.", ), option( "--max-messages", type=int, default=None, help="Send at most N messages or 0 for infinity.", ), ) async def produce(self, max_latency: float, max_messages: int): """Produce example events.""" prods = {aiter(p(max_messages)) for p in app.stress_producers} i = 0 while not app.should_stop: to_remove: Set[Any] = set() for producer in prods: i += 1 try: await anext(producer) except StopAsyncIteration: to_remove.add(producer) if not max_latency: # no latency, print every 10,000 messages if not i % 10000: self.say(f"+SEND {i}") else: # with latency, print every 10 messages if not i % 10: self.say(f"+SEND {i}") if not prods: await asyncio.sleep(1.0) if max_latency: await asyncio.sleep(random.uniform(0, max_latency)) for producer in to_remove: prods.discard(producer) print("No more producers - exiting", file=sys.stderr)
# 2) wait for order to be sent to Kafka await self.order_sent_to_kafka.wait(timeout=30.0) # 3) wait for redis index to be updated. await self.order_cache_in_redis.wait(timeout=30.0) # make sure it's now actually in redis assert await livecheck.cache.client.sismember( f"order.{order.user_id}.orders", order.id) # 4) wait for execution agent to execute the order. await self.order_executed.wait(timeout=30.0) async def make_fake_request(self) -> None: await self.get_url("http://localhost:6066/order/init/sell/?fake=1") @app.command( cli.option("--side", default="sell", help="Order side: buy, sell"), cli.option("--base-url", default="http://localhost:6066"), ) async def post_order(self: cli.AppCommand, side: str, base_url: str) -> None: path = self.app.web.url_for("orders:init", side=side) url = "".join([base_url.rstrip("/"), path]) async with self.app.http_client.get(url) as response: response.raise_for_status() print(await response.read()) if __name__ == "__main__": app.main()
@app.agent(withdrawals_topic) async def track_user_withdrawal(withdrawals): async for withdrawal in withdrawals: user_to_total[withdrawal.user] += withdrawal.amount @app.agent() async def track_country_withdrawal(withdrawals): async for withdrawal in withdrawals.group_by(Withdrawal.country): country_to_total[withdrawal.country] += withdrawal.amount @app.command( option('--max-latency', type=float, default=0.5, envvar='PRODUCE_LATENCY', help='Add delay of (at most) n seconds between publishing.'), option('--max-messages', type=int, default=None, help='Send at most N messages or 0 for infinity.'), ) async def produce(self, max_latency: float, max_messages: int): """Produce example Withdrawal events.""" for i, withdrawal in enumerate(generate_withdrawals(max_messages)): await withdrawals_topic.send(key=withdrawal.user, value=withdrawal) if not i % 10000: self.say(f'+SEND {i}') if max_latency: await asyncio.sleep(random.uniform(0, max_latency))
class Benchmark: _agent = None _produce = None produce_options = [ option('--max-latency', type=float, default=0.0, envvar='PRODUCE_LATENCY', help='Add delay of (at most) n seconds between publishing.'), option('--max-messages', type=int, default=None, help='Send at most N messages or 0 for infinity.'), ] def __init__(self, app, topic, *, n=TIME_EVERY, consume_topic=None): self.app = app self.topic = topic if consume_topic is None: consume_topic = topic self.consume_topic = consume_topic self.n = n self.app.finalize() self.app.conf.producer_acks = ACKS self.app.conf.producer_max_batch_size = BSIZE self.app.conf.producer_linger = LINGER def install(self, main_name): self.create_benchmark_agent() self.create_produce_command() if main_name == '__main__': # we use this for cProfile bench_args = { 'worker': ['worker', '-l', 'info'], 'produce': ['produce'], } if len(sys.argv) < 2: sys.argv.extend(bench_args[BENCH_TYPE]) self.app.main() def create_benchmark_agent(self): self._agent = self.app.agent(self.consume_topic)(self.process) async def process(self, stream): time_last = None async for i, value in stream.enumerate(): if not i: time_last = monotonic() await self.process_value(value) if i and not i % self.n: # noqa: S001 now = monotonic() runtime, time_last = now - time_last, now print(f'RECV {i} in {runtime}s') async def process_value(self, value): # put worker benchmark-specific code in here ... def create_produce_command(self): self._produce = self.app.command(*self.produce_options)(self.produce) async def produce(self, max_latency: float, max_messages: int, **kwargs): i = 0 time_start = None app = self.app topic = self.topic for i, (key, value) in enumerate(self.generate_values(max_messages)): callback = None if not i: time_start = monotonic() time_1st = monotonic() def on_published(meta): print(f'1ST OK: {meta} AFTER {monotonic() - time_1st}s') callback = on_published await topic.send(key=key, value=value, callback=callback) if i and not i % self.n: # noqa: S001 print(f'+SEND {i} in {monotonic() - time_start}s') time_start = monotonic() if max_latency: await asyncio.sleep(random.uniform(0, max_latency)) await asyncio.sleep(10) await app.producer.stop() print(f'Time spent on {i} messages: {monotonic() - time_start}s') def generate_values(self, max_messages: int): # override this to generate other types of values return ( # convert int to bytes, cannot use bytes(int) as that gives # character code in Python3. (None, str(i).encode()) for i in (range(max_messages) if max_messages else count()))
import logging from faust.cli import option from app import app logger = logging.getLogger(__name__) @app.command(option("--answer", type=str, help="Send your question to the server", required=False)) async def answer(self, track: str): """ Do something here """ logger.info("Command invoked ")
import uuid from typing import Tuple from faust.cli import AppCommand, option from visit_aggregator.app import app from visit_aggregator.models import Visit logger: logging.Logger = logging.getLogger("visit_aggregator") @app.command( option( "--frequency", type=float, default=1, help="The frequency in which messages are produced per second.", show_default=True, ), option( "--max-messages", type=int, default=300, help="The maximum number of messages to produce.", show_default=True, ), option( "--account-id", type=int, default=(352, ), multiple=True,
from faust import cli from ..app import app @app.command( cli.option('--side', default='sell', help='Order side: buy, sell'), cli.option('--base-url', default='http://localhost:6066'), ) async def post_order(self: cli.AppCommand, side: str, base_url: str) -> None: path = self.app.web.url_for('orders:init', side=side) url = ''.join([base_url.rstrip('/'), path]) async with self.app.http_client.get(url) as response: response.raise_for_status() self.say(await response.read())
package_index.url, ) except Exception as exc: _LOGGER.exception( f"Failed to do release notification for {package_name} ({package_version} " f"from {package_index.url}), error is not fatal: {str(exc)}" ) return async_tasks, package_releases_messages_sent @app.command( cli.option( "-v", "--verbose", is_flag=True, envvar="THOTH_PACKAGE_RELEASES_DEBUG", help="Be verbose about what's going on.", ), cli.option( "--version", is_flag=True, is_eager=True, callback=_print_version, expose_value=False, help="Print version and exit.", ), cli.option( "--monitoring-config", "-m", type=str,
from app import app, events_topic from random import randint from faust.cli import option @app.command(option('--max-events', type=int, required=True)) async def send_events(self, max_events:int): print(f'Sending {max_events} events...') half_events = max_events // 2 for i in range(max_events): await events_topic.send(key=str(randint(1, 1500000)), value=randint(0, 1000)) if i == half_events: print(f'Half amount of events sent: {i}')
import logging from typing import List import peony from faust.cli import option from peony import PeonyClient from src.app import app, hashtags_topic from src.settings import ACCESS_TOKEN, ACCESS_TOKEN_SECRET, CONSUMER_KEY, CONSUMER_SECRET logger = logging.getLogger(__name__) @app.command(option("--track", type=str, help="List of words in CSV format to filter stream of tweets", required=True)) async def hashtags_events_generator(self, track: str): """ Generate Kafka events for each hashtag found in tweets matching with the track used to filter the stream. """ logger.info(f"Searching tweets with track: {track}") client = _get_twitter_client() req = client.stream.statuses.filter.post(track=track) async with req as stream: async for tweet in stream: tweet_hashtags = _get_hashtags_from_tweet(tweet=tweet) await _produce_events_for_hashtags(hashtags=tweet_hashtags) def _get_twitter_client() -> PeonyClient: creds = dict(
) from kafkaaggregator.generator import AgentGenerator logger = logging.getLogger("kafkaaggregator") def main() -> None: """Entrypoint for Faust CLI.""" app.main() @app.command( option( "--frequency", type=float, default=config.frequency, help="The frequency in Hz in wich messages are produced.", show_default=True, ), option( "--max-messages", type=int, default=config.max_messages, help="The maximum number of messages to produce.", show_default=True, ), ) async def produce(self: AppCommand, frequency: float, max_messages: int) -> None: """Produce messages for the aggregation example.""" example = AggregationExample()
app = MessageBase().app init_logging() _LOGGER = logging.getLogger("thoth.messaging") __service_version__ = f"{__version__}+common.{__common__version__}" _LOGGER.info("This is Thoth Messaging CLI v%s", __service_version__) ## create cli @app.command( cli.option( "--topic-name", "-n", envvar="THOTH_MESSAGING_TOPIC_NAME", type=str, help="Name of topic to send message to.", required=False, ), cli.option( "--create-if-not-exist", envvar="THOTH_MESSAGING_CREATE_IF_NOT_EXIST", default=False, help="If topic doesn't already exist on Kafka then create it.", flag_value=True, ), cli.option( "--message-contents", "-m", envvar="THOTH_MESSAGING_MESSAGE_CONTENTS", type=str,