def test_publish_and_pull(publisher: PubsubPublisher):
publisher.publish(project_id, topic2, header_1, gzip.compress(b'[]'))
subscriber = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient())
for message in subscriber.pull(project_id, subscription2):
header, data, id = subscriber.unpack_message(message)
assert int(header['age']) == 2
assert 'long_str' not in header
subscriber.ack(project_id, subscription2, id)
def test_publish_and_streaming_pull(publisher: PubsubPublisher):
publisher.publish(project_id, topic1, header_1, gzip.compress(b'[]'))
publisher.publish(project_id, topic2, header_1, gzip.compress(b'[]'))
loop = asyncio.get_event_loop()
subscriber1 = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient())
subscriber2 = PubsubSubscriber(sub_client=pubsub_v1.SubscriberClient())
task1 = subscriber1.stream(project_id,
subscription1,
callback=callback,
timeout=2)
task2 = subscriber2.stream(project_id,
subscription2,
callback=callback,
timeout=2)
loop.run_until_complete(asyncio.wait([task1, task2]))
loop.close()
def main():
if request.method == 'GET':
return render_template("index.html"), 200
envelope = request.get_json()
if not envelope:
return "no Pub/Sub message received", 204
if not isinstance(envelope, dict) or 'message' not in envelope:
return "invalid Pub/Sub message format", 204
data_header = envelope['message']['attributes']
global firestore_db
global pub_client
global gcs_storer
publishers = {'pubsub': PubsubPublisher(pub_client=pub_client)}
depositor = FirestoreDepositor(db=firestore_db)
archiver = GCSListArchiver(storer=gcs_storer)
storers = [gcs_storer]
loader = Loader(publishers=publishers,
depositor=depositor,
archiver=archiver,
storers=storers)
if loader.load(load_config=json.loads(data_header['load_config'])):
return "load message received", 200
else: # pragma: no cover
return "load message to be resent", 400 # pragma: no cover
def publish(publisher_id, destination, topic_id, table_id):
global pub_client
http_header_dict = dict(request.headers)
msg_headers = {
key.lower()[5:].replace('-', '_'): value
for (key, value) in http_header_dict.items()
if key.lower().startswith('xeed-')
}
topic_id = msg_headers.get('topic_id',
None) if topic_id is None else topic_id
table_id = msg_headers.get('table_id',
None) if table_id is None else table_id
publishers = {'pubsub': PubsubPublisher(pub_client=pub_client)}
active_publisher = publishers.get(publisher_id, None)
relayer = Relayer(publishers=publishers)
if active_publisher is None:
return 'Publisher not found', 400
if request.method == 'GET':
if active_publisher.check_destination(destination, topic_id):
return render_template("message.html",
project=destination,
topic=topic_id)
else:
return 'Destination/Topic not found', 400
if http_header_dict.get('Content-Encoding', 'gzip') != 'gzip':
return 'Content must be flat or gzip-encoded', 400
if topic_id is None or table_id is None:
return 'No topic_id or table_if found', 400
msg_headers.update({'topic_id': topic_id, 'table_id': table_id})
if any(key not in msg_headers for key in
['start_seq', 'data_encode', 'data_store', 'data_format']):
return 'Xeed Header check error, something goeswrong', 400
if msg_headers['data_store'] != 'body':
content = request.data
elif 'Content-Encoding' not in http_header_dict:
if msg_headers['data_encode'] == 'flat':
msg_headers['data_encode'] = 'gzip'
content = gzip.compress(request.data)
else:
content = request.data
else:
if msg_headers['data_encode'] == 'flat':
msg_headers['data_encode'] = 'gzip'
content = request.data
else:
content = gzip.decompress(request.data)
relayer.push_data(msg_headers, content, publisher_id, destination,
topic_id, table_id, 0)
return 'Data has been pushed to the destination', 200
def receiver_callback(s: PubsubSubscriber, message: dict, source,
subscription_id):
global project_id
global firestore_db
global gcs_storer
publishers = {'pubsub': PubsubPublisher(pub_client=pub_client)}
depositor = FirestoreDepositor(db=firestore_db)
archiver = GCSListArchiver(storer=gcs_storer)
storers = [gcs_storer]
receiver = Dispatcher(publishers=publishers,
depositor=depositor,
storers=storers)
header, data, id = s.unpack_message(message)
receiver.receive_data(header, data)
s.ack(project_id, subscription_id, id)
def loader_callback(s: PubsubSubscriber, message: dict, source,
subscription_id):
global project_id
global firestore_db
global gcs_storer
publishers = {'pubsub': PubsubPublisher(pub_client=pub_client)}
depositor = FirestoreDepositor(db=firestore_db)
archiver = GCSListArchiver(storer=gcs_storer)
storers = [gcs_storer]
loader = Loader(publishers=publishers,
depositor=depositor,
archiver=archiver,
storers=storers)
header, data, id = s.unpack_message(message)
print(header)
#cleaner.clean_data(header['topic_id'], header['table_id'], header['start_seq'])
loader.load(load_config=json.loads(header['load_config']))
s.ack(project_id, subscription_id, id)
def main():
if request.method == 'GET':
return render_template("index.html"), 200
envelope = request.get_json()
if not envelope:
return "no Pub/Sub message received", 204
if not isinstance(envelope, dict) or 'message' not in envelope:
return "invalid Pub/Sub message format", 204
data_header = envelope['message']['attributes']
data_body = json.loads(
gzip.decompress(base64.b64decode(
envelope['message']['data'])).decode())
global firestore_db
global pub_client
global gcs_storer
publishers = {'pubsub': PubsubPublisher(pub_client=pub_client)}
depositor = FirestoreDepositor(db=firestore_db)
storers = [gcs_storer]
if 'INSIGHT_SUB_LIST' in os.environ:
sub_list = json.loads(
gzip.decompress(
base64.b64decode(os.environ.get('INSIGHT_SUB_LIST'))).decode())
else:
sub_list = {}
dipatcher = Dispatcher(publishers=publishers,
depositor=depositor,
storers=storers,
subscription_list=sub_list)
if dipatcher.receive_data(data_header, data_body):
return "message received", 200
else: # pragma: no cover
return "message to be resent", 400 # pragma: no cover
def test_exceptions():
with pytest.raises(TypeError):
sub = PubsubSubscriber(sub_client=object())
with pytest.raises(TypeError):
pub = PubsubPublisher(pub_client=object())
def test_check_destination(publisher: PubsubPublisher):
assert publisher.check_destination(project_id, topic1)
assert not publisher.check_destination(project_id, 'error_topic')
def publisher():
pub_client = pubsub_v1.PublisherClient()
publisher = PubsubPublisher(pub_client=pub_client)
yield publisher
import gzip
import hashlib
from flask import Flask, request, Response, render_template
import google.auth
from google.cloud import pubsub_v1
from google.cloud import bigquery
from xialib_gcp import BigQueryAdaptor, GCSStorer, PubsubPublisher
from pyagent import Agent, Pusher
app = Flask(__name__)
project_id = google.auth.default()[1]
bigquery_db = bigquery.Client()
gcs_storer = GCSStorer()
Agent.set_internal_channel(messager=PubsubPublisher(pub_client=pubsub_v1.PublisherClient()),
channel=project_id,
topic_cockpit='insight-cockpit')
@app.route('/', methods=['GET', 'POST'])
def main():
if request.method == 'GET':
return render_template("index.html"), 200
envelope = request.get_json()
if not envelope:
return "no Pub/Sub message received", 204
if not isinstance(envelope, dict) or 'message' not in envelope:
return "invalid Pub/Sub message format", 204
data_header = envelope['message']['attributes']
data_body = json.loads(gzip.decompress(base64.b64decode(envelope['message']['data'])).decode())
import base64
import gzip
import hashlib
from flask import Flask, request, Response, render_template
import google.auth
from google.cloud import pubsub_v1
from google.cloud import firestore
import google.cloud.logging
from xialib_gcp import PubsubPublisher, FirestoreDepositor, GCSStorer, GCSListArchiver
from pyinsight import Insight, Cleaner
app = Flask(__name__)
project_id = google.auth.default()[1]
Insight.set_internal_channel(
messager=PubsubPublisher(pub_client=pubsub_v1.PublisherClient()),
channel=project_id,
topic_cockpit='insight-cockpit',
topic_cleaner='insight-cleaner',
topic_merger='insight-merger',
topic_packager='insight-packager',
topic_loader='insight-loader',
topic_backlog='insight-backlog')
firestore_db = firestore.Client()
gcs_storer = GCSStorer()
client = google.cloud.logging.Client()
client.get_default_handler()
client.setup_logging()
