def main():
if request.method == 'GET':
pusher = service_factory(object_config, global_connectors)
return render_template("index.html"), 200
pusher = service_factory(object_config, global_connectors)
envelope = request.get_json()
if not envelope:
return "no Pub/Sub message received", 204
if not isinstance(envelope, dict) or 'message' not in envelope:
logging.warning("invalid Pub/Sub message format")
return "invalid Pub/Sub message format", 204
data_header = envelope['message']['attributes']
if data_header["topic_id"] not in connected_topics:
logging.warning("topic {} is not connected to me".format(
data_header["topic_id"]))
return "topic {} is not connected to me".format(
data_header["topic_id"]), 204
data_body = json.loads(
gzip.decompress(base64.b64decode(
envelope['message']['data'])).decode())
if pusher.push_data(data_header, data_body):
return "message received", 200
else: # pragma: no cover
return "message to be resent", 400 # pragma: no cover
def push():
http_header_dict = dict(request.headers)
msg_headers = {key.lower()[5:].replace('-', '_'): value for (key, value) in http_header_dict.items() if
key.lower().startswith('xeed-')}
destination = current_app.config["DESTINATION"]
topic_id = current_app.config["TOPIC_ID"]
table_id = msg_headers.get('table_id', "")
seeder = service_factory(object_config, global_connectors)
if request.method == 'GET':
if seeder.check_destination(destination, topic_id):
return render_template("message.html", project=destination, topic=topic_id)
else:
return 'Destination/Topic not found', 400
if http_header_dict.get('Content-Encoding', 'gzip') != 'gzip':
return 'Content must be flat or gzip-encoded', 400
if topic_id is None or table_id is None:
return 'No topic_id or table_if found', 400
msg_headers.update({'topic_id': topic_id, 'table_id': table_id})
if any(key not in msg_headers for key in ['start_seq', 'data_encode', 'data_store', 'data_format']):
return 'Xeed Header check error, something goeswrong', 400
if msg_headers['data_store'] != 'body':
content = request.data
elif 'Content-Encoding' not in http_header_dict:
if msg_headers['data_encode'] == 'flat':
msg_headers['data_encode'] = 'gzip'
content = gzip.compress(request.data)
else:
content = request.data
else:
if msg_headers['data_encode'] == 'flat':
msg_headers['data_encode'] = 'gzip'
content = request.data
else:
content = gzip.decompress(request.data)
# Case 1: Send to Data Laker
if current_app.config.get("INSIGHT", ""):
default_seeder = Seeder(publisher=PubsubGcrPublisher())
default_seeder.push_data(msg_headers, content,
current_app.config["INSIGHT"], topic_id, table_id,
current_app.config["SIZE_LIMIT"])
logging.info("Data has been pushed to {}".format(current_app.config["INSIGHT"]))
# Case 2: Send to Destination: insight == destination means save to insight data-lake only
if current_app.config.get("INSIGHT", "") != current_app.config["DESTINATION"]:
seeder.push_data(msg_headers, content, destination, topic_id, table_id, current_app.config["SIZE_LIMIT"])
logging.info("Data has been pushed to {}".format(current_app.config["DESTINATION"]))
return 'Data has been pushed to the destination', 200
def main():
loader = service_factory(object_config, global_connectors)
if request.method == 'GET':
return render_template("index.html"), 200
envelope = request.get_json()
if not envelope:
return "no Pub/Sub message received", 204
if not isinstance(envelope, dict) or 'message' not in envelope:
return "invalid Pub/Sub message format", 204
data_header = envelope['message']['attributes']
if loader.load(**json.loads(data_header['load_config'])):
return "load message received", 200
else: # pragma: no cover
return "load message to be resent", 400 # pragma: no cover
def insight_merger():
merger = service_factory(object_config, global_connectors)
if request.method == 'GET':
return render_template("index.html"), 200
envelope = request.get_json()
if not envelope:
return "no Pub/Sub message received", 204
if not isinstance(envelope, dict) or 'message' not in envelope:
return "invalid Pub/Sub message format", 204
data_header = envelope['message']['attributes']
if merger.merge_all_data(data_header['topic_id'], data_header['table_id']):
return "merge message received", 200 # pragma: no cover
else: # pragma: no cover
return "merge message to be resent", 400 # pragma: no cover
def main():
receiver = service_factory(object_config, global_connectors)
if request.method == 'GET':
return render_template("index.html"), 200
envelope = request.get_json()
if not envelope:
return "no Pub/Sub message received", 204
if not isinstance(envelope, dict) or 'message' not in envelope:
return "invalid Pub/Sub message format", 204
data_header = envelope['message']['attributes']
data_body = json.loads(
gzip.decompress(base64.b64decode(
envelope['message']['data'])).decode())
if receiver.receive_data(data_header, data_body):
return "message received", 200
else: # pragma: no cover
return "message to be resent", 400 # pragma: no cover
import google.cloud.logging
import google.auth
from xialib.service import service_factory
# Global Setting
app = Flask(__name__)
project_id = google.auth.default()[1]
# Configuration Load
with open(os.path.join('.', 'config', 'global_conn_config.json')) as fp:
global_conn_config = json.load(fp)
with open(os.path.join('.', 'config', 'object_config.json')) as fp:
object_config = json.load(fp)
# Global Object Factory
global_connectors = service_factory(global_conn_config)
# Log configuration
client = google.cloud.logging.Client()
client.get_default_handler()
client.setup_logging()
@app.route('/', methods=['GET', 'POST'])
def main():
loader = service_factory(object_config, global_connectors)
if request.method == 'GET':
return render_template("index.html"), 200
envelope = request.get_json()
if not envelope:
return "no Pub/Sub message received", 204
