def test_publish_and_streaming_pull(publisher: PubsubPublisher):
publisher.publish(project_id, topic1, header_1, gzip.compress(b'[]'))
publisher.publish(project_id, topic2, header_1, gzip.compress(b'[]'))
loop = asyncio.get_event_loop()
subscriber1 = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient())
subscriber2 = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient())
task1 = subscriber1.stream(project_id,
subscription1,
callback=callback,
timeout=2)
task2 = subscriber2.stream(project_id,
subscription2,
callback=callback,
timeout=2)
loop.run_until_complete(asyncio.wait([task1, task2]))
loop.close()
json_data = get_envelope(clean_header, [])
response = requests.post(cleaner_url, headers=headers, json=json_data)
return response
def cleaner_callback(s: PubsubSubscriber, message: dict, source,
subscription_id):
global project_id
global firestore_db
global gcs_storer
depositor = FirestoreDepositor(db=firestore_db)
archiver = GCSListArchiver(storer=gcs_storer)
cleaner = Cleaner(archiver=archiver, depositor=depositor)
header, data, id = s.unpack_message(message)
header = dict(header)
cleaner.clean_data(header['topic_id'], header['table_id'],
header['start_seq'])
s.ack(project_id, subscription_id, id)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
subscriber = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient())
task = subscriber.stream('x-i-a-test',
'insight-cleaner-debug',
callback=cleaner_callback)
loop.run_until_complete(asyncio.wait([task]))
loop.close()
def loader_callback(s: PubsubSubscriber, message: dict, source,
subscription_id):
global project_id
global firestore_db
global gcs_storer
publishers = {'pubsub': PubsubPublisher(pub_client=pub_client)}
depositor = FirestoreDepositor(db=firestore_db)
archiver = GCSListArchiver(storer=gcs_storer)
storers = [gcs_storer]
loader = Loader(publishers=publishers,
depositor=depositor,
archiver=archiver,
storers=storers)
header, data, id = s.unpack_message(message)
print(header)
#cleaner.clean_data(header['topic_id'], header['table_id'], header['start_seq'])
loader.load(load_config=json.loads(header['load_config']))
s.ack(project_id, subscription_id, id)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
subscriber = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient())
task = subscriber.stream('x-i-a-test',
'insight-loader-debug',
callback=loader_callback)
loop.run_until_complete(asyncio.wait([task]))
loop.close()