def packager_callback(s: PubsubSubscriber, message: dict, source,
subscription_id):
global firestore_db
global gcs_storer
depositor = FirestoreDepositor(db=firestore_db)
archiver = GCSListArchiver(storer=gcs_storer)
packager = Packager(archiver=archiver, depositor=depositor)
header, data, id = s.unpack_message(message)
header = dict(header)
packager.package_data(header['topic_id'], header['table_id'])
s.ack(project_id, subscription_id, id)
def receiver_callback(s: PubsubSubscriber, message: dict, source,
subscription_id):
global project_id
global firestore_db
global gcs_storer
publishers = {'pubsub': PubsubPublisher(pub_client=pub_client)}
depositor = FirestoreDepositor(db=firestore_db)
archiver = GCSListArchiver(storer=gcs_storer)
storers = [gcs_storer]
receiver = Dispatcher(publishers=publishers,
depositor=depositor,
storers=storers)
header, data, id = s.unpack_message(message)
receiver.receive_data(header, data)
s.ack(project_id, subscription_id, id)
def loader_callback(s: PubsubSubscriber, message: dict, source,
subscription_id):
global project_id
global firestore_db
global gcs_storer
publishers = {'pubsub': PubsubPublisher(pub_client=pub_client)}
depositor = FirestoreDepositor(db=firestore_db)
archiver = GCSListArchiver(storer=gcs_storer)
storers = [gcs_storer]
loader = Loader(publishers=publishers,
depositor=depositor,
archiver=archiver,
storers=storers)
header, data, id = s.unpack_message(message)
print(header)
#cleaner.clean_data(header['topic_id'], header['table_id'], header['start_seq'])
loader.load(load_config=json.loads(header['load_config']))
s.ack(project_id, subscription_id, id)
def callback(s: PubsubSubscriber, message: dict, source, subscription_id):
global project_id
global storer
global adaptor_dict
global locks
pusher = Pusher(storers=[storer], adaptor_dict=adaptor_dict)
header, data, id = s.unpack_message(message)
s.ack(project_id, subscription_id, id)
if subscription_id == 'agent-001-debug':
if locks.get(header['table_id'], None) is None:
locks[None].acquire(True)
if locks.get(header['table_id'], None) is None:
locks[header['table_id']] = threading.Lock()
locks[None].release()
locks[header['table_id']].acquire(True)
print("{}: {}".format(subscription_id, header))
pusher.push_data(dict(header), data)
locks[header['table_id']].release()
def callback(s: PubsubSubscriber, message: dict, source, subscription_id):
global bigquery_db
global gcs_storer
global project_id
storers = [gcs_storer]
adapter = BigQueryAdaptor(connection=bigquery_db, project_id=project_id)
adapter_dict = {'.': adapter, 'NPL.': adapter}
pusher = Pusher(storers=storers, adaptor_dict=adapter_dict)
header, data, id = s.unpack_message(message)
pusher.push_data(dict(header), data)
def test_publish_and_pull(publisher: PubsubPublisher):
publisher.publish(project_id, topic2, header_1, gzip.compress(b'[]'))
subscriber = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient())
for message in subscriber.pull(project_id, subscription2):
header, data, id = subscriber.unpack_message(message)
assert int(header['age']) == 2
assert 'long_str' not in header
subscriber.ack(project_id, subscription2, id)
def callback(s: PubsubSubscriber, message: dict, source, subscription_id):
global bigquery_db
global gcs_storer
global project_id
storers = [gcs_storer]
adapter = BigQueryAdaptor(connection=bigquery_db, project_id=project_id)
adapter_dict = {'.': adapter, 'NPL.': adapter}
pusher = Pusher(storers=storers, adaptor_dict=adapter_dict)
header, data, id = s.unpack_message(message)
print("size:{}-{}".format(len(data), header))
s.ack(project_id, subscription_id, id)
if subscription_id == 'agent-001-debug':
if locks.get(header['table_id'], None) is None:
locks[None].acquire(True)
if locks.get(header['table_id'], None) is None:
locks[header['table_id']] = threading.Lock()
locks[None].release()
locks[header['table_id']].acquire(True)
print("{}: {}".format(subscription_id, header))
pusher.push_data(header, data)
locks[header['table_id']].release()
def test_publish_and_streaming_pull(publisher: PubsubPublisher):
publisher.publish(project_id, topic1, header_1, gzip.compress(b'[]'))
publisher.publish(project_id, topic2, header_1, gzip.compress(b'[]'))
loop = asyncio.get_event_loop()
subscriber1 = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient())
subscriber2 = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient())
task1 = subscriber1.stream(project_id,
subscription1,
callback=callback,
timeout=2)
task2 = subscriber2.stream(project_id,
subscription2,
callback=callback,
timeout=2)
loop.run_until_complete(asyncio.wait([task1, task2]))
loop.close()
def test_exceptions():
with pytest.raises(TypeError):
sub = PubsubSubscriber(sub_client=object())
with pytest.raises(TypeError):
pub = PubsubPublisher(pub_client=object())
def callback(s: PubsubSubscriber, message: dict, source, subscription_id):
header, data, id = s.unpack_message(message)
assert int(header['age']) == 2
s.ack(source, subscription_id, id)
def callback(s: PubsubSubscriber, message: dict, source, subscription_id):
header, data, id = s.unpack_message(message)
print(header)
print(json.loads(gzip.decompress(data).decode()))
json_data = get_envelope(clean_header, [])
response = requests.post(cleaner_url, headers=headers, json=json_data)
return response
def cleaner_callback(s: PubsubSubscriber, message: dict, source,
subscription_id):
global project_id
global firestore_db
global gcs_storer
depositor = FirestoreDepositor(db=firestore_db)
archiver = GCSListArchiver(storer=gcs_storer)
cleaner = Cleaner(archiver=archiver, depositor=depositor)
header, data, id = s.unpack_message(message)
header = dict(header)
cleaner.clean_data(header['topic_id'], header['table_id'],
header['start_seq'])
s.ack(project_id, subscription_id, id)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
subscriber = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient())
task = subscriber.stream('x-i-a-test',
'insight-cleaner-debug',
callback=cleaner_callback)
loop.run_until_complete(asyncio.wait([task]))
loop.close()
global gcs_storer
global project_id
storers = [gcs_storer]
adapter = BigQueryAdaptor(connection=bigquery_db, project_id=project_id)
adapter_dict = {'.': adapter, 'NPL.': adapter}
pusher = Pusher(storers=storers, adaptor_dict=adapter_dict)
header, data, id = s.unpack_message(message)
print("size:{}-{}".format(len(data), header))
s.ack(project_id, subscription_id, id)
if subscription_id == 'agent-001-debug':
if locks.get(header['table_id'], None) is None:
locks[None].acquire(True)
if locks.get(header['table_id'], None) is None:
locks[header['table_id']] = threading.Lock()
locks[None].release()
locks[header['table_id']].acquire(True)
print("{}: {}".format(subscription_id, header))
pusher.push_data(header, data)
locks[header['table_id']].release()
if __name__ == '__main__':
loop = asyncio.get_event_loop()
task_pusher = PubsubSubscriber(
sub_client=pubsub_v1.SubscriberClient()).stream('x-i-a-test',
'agent-001-debug',
callback=callback)
loop.run_until_complete(asyncio.wait([task_pusher]))
loop.close()
from xialib_gcp import PubsubSubscriber, BigQueryAdaptor, GCSStorer
import google.auth
from xialib import SQLiteAdaptor, BasicStorer
from pyagent import Pusher
project_id = google.auth.default()[1]
bigquery_db = bigquery.Client()
gcs_storer = GCSStorer()
def callback(s: PubsubSubscriber, message: dict, source, subscription_id):
global bigquery_db
global gcs_storer
global project_id
storers = [gcs_storer]
adapter = BigQueryAdaptor(connection=bigquery_db, project_id=project_id)
adapter_dict = {'.': adapter, 'NPL.': adapter}
pusher = Pusher(storers=storers, adaptor_dict=adapter_dict)
header, data, id = s.unpack_message(message)
pusher.push_data(dict(header), data)
# s.ack(project_id, subscription_id, id)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
task_pusher = PubsubSubscriber(
sub_client=pubsub_v1.SubscriberClient()).stream(
'x-i-a-test', 'insight-dead-letter-debug', callback=callback)
loop.run_until_complete(asyncio.wait([task_pusher]))
loop.close()
def test_check_messages():
sub = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient())
for message in sub.pull('x-i-a-test', 'xialib-sub-01'):
header, data, id = sub.unpack_message(message)
assert len(json.loads(gzip.decompress(data).decode())) == 2
sub.ack('x-i-a-test', 'xialib-sub-01', id)
def callback(s: PubsubSubscriber, message: dict, source, subscription_id):
header, data, id = s.unpack_message(message)
print("{}: {}".format(subscription_id, header))
s.ack(project_id, subscription_id, id)
import google.auth
from google.cloud import pubsub_v1
from xialib_gcp import PubsubSubscriber
project_id = google.auth.default()[1]
def callback(s: PubsubSubscriber, message: dict, source, subscription_id):
header, data, id = s.unpack_message(message)
print("{}: {}".format(subscription_id, header))
s.ack(project_id, subscription_id, id)
loop = asyncio.get_event_loop()
task_backlog = PubsubSubscriber(
sub_client=pubsub_v1.SubscriberClient()).stream('x-i-a-test',
'insight-backlog-debug',
callback=callback)
task_cockpit = PubsubSubscriber(
sub_client=pubsub_v1.SubscriberClient()).stream('x-i-a-test',
'insight-cockpit-debug',
callback=callback)
task_cleaner = PubsubSubscriber(
sub_client=pubsub_v1.SubscriberClient()).stream('x-i-a-test',
'insight-cleaner-debug',
callback=callback)
task_loader = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient()).stream(
'x-i-a-test', 'insight-loader-debug', callback=callback)
task_merger = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient()).stream(
'x-i-a-test', 'insight-merger-debug', callback=callback)
task_packager = PubsubSubscriber(
sub_client=pubsub_v1.SubscriberClient()).stream('x-i-a-test',
