def test_publish_and_pull(publisher: PubsubPublisher):
publisher.publish(project_id, topic2, header_1, gzip.compress(b'[]'))
subscriber = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient())
for message in subscriber.pull(project_id, subscription2):
header, data, id = subscriber.unpack_message(message)
assert int(header['age']) == 2
assert 'long_str' not in header
subscriber.ack(project_id, subscription2, id)
def callback(s: PubsubSubscriber, message: dict, source, subscription_id):
global bigquery_db
global gcs_storer
global project_id
storers = [gcs_storer]
adapter = BigQueryAdaptor(connection=bigquery_db, project_id=project_id)
adapter_dict = {'.': adapter, 'NPL.': adapter}
pusher = Pusher(storers=storers, adaptor_dict=adapter_dict)
header, data, id = s.unpack_message(message)
pusher.push_data(dict(header), data)
def packager_callback(s: PubsubSubscriber, message: dict, source,
subscription_id):
global firestore_db
global gcs_storer
depositor = FirestoreDepositor(db=firestore_db)
archiver = GCSListArchiver(storer=gcs_storer)
packager = Packager(archiver=archiver, depositor=depositor)
header, data, id = s.unpack_message(message)
header = dict(header)
packager.package_data(header['topic_id'], header['table_id'])
s.ack(project_id, subscription_id, id)
def receiver_callback(s: PubsubSubscriber, message: dict, source,
subscription_id):
global project_id
global firestore_db
global gcs_storer
publishers = {'pubsub': PubsubPublisher(pub_client=pub_client)}
depositor = FirestoreDepositor(db=firestore_db)
archiver = GCSListArchiver(storer=gcs_storer)
storers = [gcs_storer]
receiver = Dispatcher(publishers=publishers,
depositor=depositor,
storers=storers)
header, data, id = s.unpack_message(message)
receiver.receive_data(header, data)
s.ack(project_id, subscription_id, id)
def callback(s: PubsubSubscriber, message: dict, source, subscription_id):
global project_id
global storer
global adaptor_dict
global locks
pusher = Pusher(storers=[storer], adaptor_dict=adaptor_dict)
header, data, id = s.unpack_message(message)
s.ack(project_id, subscription_id, id)
if subscription_id == 'agent-001-debug':
if locks.get(header['table_id'], None) is None:
locks[None].acquire(True)
if locks.get(header['table_id'], None) is None:
locks[header['table_id']] = threading.Lock()
locks[None].release()
locks[header['table_id']].acquire(True)
print("{}: {}".format(subscription_id, header))
pusher.push_data(dict(header), data)
locks[header['table_id']].release()
def loader_callback(s: PubsubSubscriber, message: dict, source,
subscription_id):
global project_id
global firestore_db
global gcs_storer
publishers = {'pubsub': PubsubPublisher(pub_client=pub_client)}
depositor = FirestoreDepositor(db=firestore_db)
archiver = GCSListArchiver(storer=gcs_storer)
storers = [gcs_storer]
loader = Loader(publishers=publishers,
depositor=depositor,
archiver=archiver,
storers=storers)
header, data, id = s.unpack_message(message)
print(header)
#cleaner.clean_data(header['topic_id'], header['table_id'], header['start_seq'])
loader.load(load_config=json.loads(header['load_config']))
s.ack(project_id, subscription_id, id)
def callback(s: PubsubSubscriber, message: dict, source, subscription_id):
global bigquery_db
global gcs_storer
global project_id
storers = [gcs_storer]
adapter = BigQueryAdaptor(connection=bigquery_db, project_id=project_id)
adapter_dict = {'.': adapter, 'NPL.': adapter}
pusher = Pusher(storers=storers, adaptor_dict=adapter_dict)
header, data, id = s.unpack_message(message)
print("size:{}-{}".format(len(data), header))
s.ack(project_id, subscription_id, id)
if subscription_id == 'agent-001-debug':
if locks.get(header['table_id'], None) is None:
locks[None].acquire(True)
if locks.get(header['table_id'], None) is None:
locks[header['table_id']] = threading.Lock()
locks[None].release()
locks[header['table_id']].acquire(True)
print("{}: {}".format(subscription_id, header))
pusher.push_data(header, data)
locks[header['table_id']].release()
def callback(s: PubsubSubscriber, message: dict, source, subscription_id):
header, data, id = s.unpack_message(message)
assert int(header['age']) == 2
s.ack(source, subscription_id, id)
def callback(s: PubsubSubscriber, message: dict, source, subscription_id):
header, data, id = s.unpack_message(message)
print(header)
print(json.loads(gzip.decompress(data).decode()))
def test_check_messages():
sub = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient())
for message in sub.pull('x-i-a-test', 'xialib-sub-01'):
header, data, id = sub.unpack_message(message)
assert len(json.loads(gzip.decompress(data).decode())) == 2
sub.ack('x-i-a-test', 'xialib-sub-01', id)
def callback(s: PubsubSubscriber, message: dict, source, subscription_id):
header, data, id = s.unpack_message(message)
print("{}: {}".format(subscription_id, header))
s.ack(project_id, subscription_id, id)