def main():
if request.method == 'GET':
return render_template("index.html"), 200
envelope = request.get_json()
if not envelope:
return "no Pub/Sub message received", 204
if not isinstance(envelope, dict) or 'message' not in envelope:
return "invalid Pub/Sub message format", 204
data_header = envelope['message']['attributes']
global firestore_db
global pub_client
global gcs_storer
publishers = {'pubsub': PubsubPublisher(pub_client=pub_client)}
depositor = FirestoreDepositor(db=firestore_db)
archiver = GCSListArchiver(storer=gcs_storer)
storers = [gcs_storer]
loader = Loader(publishers=publishers,
depositor=depositor,
archiver=archiver,
storers=storers)
if loader.load(load_config=json.loads(data_header['load_config'])):
return "load message received", 200
else: # pragma: no cover
return "load message to be resent", 400 # pragma: no cover
def packager_callback(s: PubsubSubscriber, message: dict, source,
subscription_id):
global firestore_db
global gcs_storer
depositor = FirestoreDepositor(db=firestore_db)
archiver = GCSListArchiver(storer=gcs_storer)
packager = Packager(archiver=archiver, depositor=depositor)
header, data, id = s.unpack_message(message)
header = dict(header)
packager.package_data(header['topic_id'], header['table_id'])
s.ack(project_id, subscription_id, id)
def receiver_callback(s: PubsubSubscriber, message: dict, source,
subscription_id):
global project_id
global firestore_db
global gcs_storer
publishers = {'pubsub': PubsubPublisher(pub_client=pub_client)}
depositor = FirestoreDepositor(db=firestore_db)
archiver = GCSListArchiver(storer=gcs_storer)
storers = [gcs_storer]
receiver = Dispatcher(publishers=publishers,
depositor=depositor,
storers=storers)
header, data, id = s.unpack_message(message)
receiver.receive_data(header, data)
s.ack(project_id, subscription_id, id)
def loader_callback(s: PubsubSubscriber, message: dict, source,
subscription_id):
global project_id
global firestore_db
global gcs_storer
publishers = {'pubsub': PubsubPublisher(pub_client=pub_client)}
depositor = FirestoreDepositor(db=firestore_db)
archiver = GCSListArchiver(storer=gcs_storer)
storers = [gcs_storer]
loader = Loader(publishers=publishers,
depositor=depositor,
archiver=archiver,
storers=storers)
header, data, id = s.unpack_message(message)
print(header)
#cleaner.clean_data(header['topic_id'], header['table_id'], header['start_seq'])
loader.load(load_config=json.loads(header['load_config']))
s.ack(project_id, subscription_id, id)
def insight_receiver():
if request.method == 'GET':
return render_template("index.html"), 200
envelope = request.get_json()
if not envelope:
return "no Pub/Sub message received", 204
if not isinstance(envelope, dict) or 'message' not in envelope:
return "invalid Pub/Sub message format", 204
data_header = envelope['message']['attributes']
global firestore_db
global gcs_storer
depositor = FirestoreDepositor(db=firestore_db)
archiver = GCSListArchiver(storer=gcs_storer)
packager = Packager(archiver=archiver, depositor=depositor)
if packager.package_data(data_header['topic_id'], data_header['table_id']):
return "package message received", 200
else: # pragma: no cover
return "package message to be resent", 400 # pragma: no cover
def test_exceptions(archiver):
with pytest.raises(FileNotFoundError):
a2 = archiver.set_current_topic_table('error', 'error')
with pytest.raises(TypeError):
a2 = GCSListArchiver(storer=object())
def archiver():
archiver = GCSListArchiver()
archiver.set_current_topic_table('test-001', 'person_complex')
yield archiver