def main():
reader, writer = make_pair(INBOUND_TOPIC, GROUP, BOOT)
produce_queue = deque()
loop.create_task(reader.run(partial(consumer, produce_queue=produce_queue)))
c = make_producer(send_item, produce_queue)
loop.create_task(writer.run(c))
metrics.start()
loop.run_forever()
def main():
reader, writer = make_pair(QUEUE, GROUP, BOOT)
produce_queue = deque()
loop.create_task(reader.run(partial(consumer, produce_queue=produce_queue)))
c = make_producer(handoff, produce_queue)
loop.create_task(writer.run(c))
metrics.start()
loop.run_forever()
def main():
reader, writer = make_pair(QUEUE, GROUP, BOOT)
produce_queue = deque()
consumer_task = loop.create_task(
reader.run(partial(consumer, produce_queue=produce_queue)))
consumer_task.add_done_callback(partial(crash, name="consumer"))
c = make_producer(handoff, produce_queue)
producer_task = loop.create_task(writer.run(c))
producer_task.add_done_callback(partial(crash, name="producer"))
metrics.start()
loop.run_forever()
def main():
signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT)
for s in signals:
loop.add_signal_handler(
s, lambda s=s: loop.create_task(shutdown(s, loop)))
mnm.start_http_server(port=9126)
loop.set_default_executor(thread_pool_executor)
TASK_LOOPS["consumer"] = make_task(
CONSUMER.get_callback(consume)(), "consumer", loop)
TASK_LOOPS["producer"] = make_task(
PRODUCER.get_callback(make_responder(produce_queue))(), "producer",
loop)
TASK_LOOPS["sysprofile_producer"] = make_task(
SYSTEM_PROFILE_PRODUCER.get_callback(
make_producer(send_system_profile, system_profile_queue))(),
"sysprofile_producer", loop)
logger.info("PUP Service Activated")
loop.run_forever()
async def consumer(client):
data = await client.getmany()
for msgs in (msgs for tp, msgs in data.items()):
for msg in msgs:
doc = json.loads(msg.value)
REQUEST_ID.set(doc["payload_id"])
url = doc["url"]
payload = await fetch(url)
bucket = BUCKET_MAP[doc["category"]]
await store(payload, bucket)
produce_queue.append({
"validation": "handoff",
"payload_id": doc["payload_id"]
})
await asyncio.sleep(0.5)
async def handoff(client, item):
await client.send_and_wait(RESPONSE_QUEUE,
json.dumps(item).encode("utf-8"))
if __name__ == "__main__":
reader, writer = make_pair(QUEUE, GROUP, BOOT)
loop.create_task(reader.run(consumer))
c = make_producer(handoff, produce_queue)
loop.create_task(writer.run(c))
metrics.start()
loop.run_forever()