def main(): reader, writer = make_pair(INBOUND_TOPIC, GROUP, BOOT) produce_queue = deque() loop.create_task(reader.run(partial(consumer, produce_queue=produce_queue))) c = make_producer(send_item, produce_queue) loop.create_task(writer.run(c)) metrics.start() loop.run_forever()
def main(): reader, writer = make_pair(QUEUE, GROUP, BOOT) produce_queue = deque() loop.create_task(reader.run(partial(consumer, produce_queue=produce_queue))) c = make_producer(handoff, produce_queue) loop.create_task(writer.run(c)) metrics.start() loop.run_forever()
def main(): reader, writer = make_pair(QUEUE, GROUP, BOOT) produce_queue = deque() consumer_task = loop.create_task( reader.run(partial(consumer, produce_queue=produce_queue))) consumer_task.add_done_callback(partial(crash, name="consumer")) c = make_producer(handoff, produce_queue) producer_task = loop.create_task(writer.run(c)) producer_task.add_done_callback(partial(crash, name="producer")) metrics.start() loop.run_forever()
def main(): signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) for s in signals: loop.add_signal_handler( s, lambda s=s: loop.create_task(shutdown(s, loop))) mnm.start_http_server(port=9126) loop.set_default_executor(thread_pool_executor) TASK_LOOPS["consumer"] = make_task( CONSUMER.get_callback(consume)(), "consumer", loop) TASK_LOOPS["producer"] = make_task( PRODUCER.get_callback(make_responder(produce_queue))(), "producer", loop) TASK_LOOPS["sysprofile_producer"] = make_task( SYSTEM_PROFILE_PRODUCER.get_callback( make_producer(send_system_profile, system_profile_queue))(), "sysprofile_producer", loop) logger.info("PUP Service Activated") loop.run_forever()
async def consumer(client): data = await client.getmany() for msgs in (msgs for tp, msgs in data.items()): for msg in msgs: doc = json.loads(msg.value) REQUEST_ID.set(doc["payload_id"]) url = doc["url"] payload = await fetch(url) bucket = BUCKET_MAP[doc["category"]] await store(payload, bucket) produce_queue.append({ "validation": "handoff", "payload_id": doc["payload_id"] }) await asyncio.sleep(0.5) async def handoff(client, item): await client.send_and_wait(RESPONSE_QUEUE, json.dumps(item).encode("utf-8")) if __name__ == "__main__": reader, writer = make_pair(QUEUE, GROUP, BOOT) loop.create_task(reader.run(consumer)) c = make_producer(handoff, produce_queue) loop.create_task(writer.run(c)) metrics.start() loop.run_forever()