def main(): if request.method == 'GET': pusher = service_factory(object_config, global_connectors) return render_template("index.html"), 200 pusher = service_factory(object_config, global_connectors) envelope = request.get_json() if not envelope: return "no Pub/Sub message received", 204 if not isinstance(envelope, dict) or 'message' not in envelope: logging.warning("invalid Pub/Sub message format") return "invalid Pub/Sub message format", 204 data_header = envelope['message']['attributes'] if data_header["topic_id"] not in connected_topics: logging.warning("topic {} is not connected to me".format( data_header["topic_id"])) return "topic {} is not connected to me".format( data_header["topic_id"]), 204 data_body = json.loads( gzip.decompress(base64.b64decode( envelope['message']['data'])).decode()) if pusher.push_data(data_header, data_body): return "message received", 200 else: # pragma: no cover return "message to be resent", 400 # pragma: no cover
def push(): http_header_dict = dict(request.headers) msg_headers = {key.lower()[5:].replace('-', '_'): value for (key, value) in http_header_dict.items() if key.lower().startswith('xeed-')} destination = current_app.config["DESTINATION"] topic_id = current_app.config["TOPIC_ID"] table_id = msg_headers.get('table_id', "") seeder = service_factory(object_config, global_connectors) if request.method == 'GET': if seeder.check_destination(destination, topic_id): return render_template("message.html", project=destination, topic=topic_id) else: return 'Destination/Topic not found', 400 if http_header_dict.get('Content-Encoding', 'gzip') != 'gzip': return 'Content must be flat or gzip-encoded', 400 if topic_id is None or table_id is None: return 'No topic_id or table_if found', 400 msg_headers.update({'topic_id': topic_id, 'table_id': table_id}) if any(key not in msg_headers for key in ['start_seq', 'data_encode', 'data_store', 'data_format']): return 'Xeed Header check error, something goeswrong', 400 if msg_headers['data_store'] != 'body': content = request.data elif 'Content-Encoding' not in http_header_dict: if msg_headers['data_encode'] == 'flat': msg_headers['data_encode'] = 'gzip' content = gzip.compress(request.data) else: content = request.data else: if msg_headers['data_encode'] == 'flat': msg_headers['data_encode'] = 'gzip' content = request.data else: content = gzip.decompress(request.data) # Case 1: Send to Data Laker if current_app.config.get("INSIGHT", ""): default_seeder = Seeder(publisher=PubsubGcrPublisher()) default_seeder.push_data(msg_headers, content, current_app.config["INSIGHT"], topic_id, table_id, current_app.config["SIZE_LIMIT"]) logging.info("Data has been pushed to {}".format(current_app.config["INSIGHT"])) # Case 2: Send to Destination: insight == destination means save to insight data-lake only if current_app.config.get("INSIGHT", "") != current_app.config["DESTINATION"]: seeder.push_data(msg_headers, content, destination, topic_id, table_id, current_app.config["SIZE_LIMIT"]) logging.info("Data has been pushed to {}".format(current_app.config["DESTINATION"])) return 'Data has been pushed to the destination', 200
def main(): loader = service_factory(object_config, global_connectors) if request.method == 'GET': return render_template("index.html"), 200 envelope = request.get_json() if not envelope: return "no Pub/Sub message received", 204 if not isinstance(envelope, dict) or 'message' not in envelope: return "invalid Pub/Sub message format", 204 data_header = envelope['message']['attributes'] if loader.load(**json.loads(data_header['load_config'])): return "load message received", 200 else: # pragma: no cover return "load message to be resent", 400 # pragma: no cover
def insight_merger(): merger = service_factory(object_config, global_connectors) if request.method == 'GET': return render_template("index.html"), 200 envelope = request.get_json() if not envelope: return "no Pub/Sub message received", 204 if not isinstance(envelope, dict) or 'message' not in envelope: return "invalid Pub/Sub message format", 204 data_header = envelope['message']['attributes'] if merger.merge_all_data(data_header['topic_id'], data_header['table_id']): return "merge message received", 200 # pragma: no cover else: # pragma: no cover return "merge message to be resent", 400 # pragma: no cover
def main(): receiver = service_factory(object_config, global_connectors) if request.method == 'GET': return render_template("index.html"), 200 envelope = request.get_json() if not envelope: return "no Pub/Sub message received", 204 if not isinstance(envelope, dict) or 'message' not in envelope: return "invalid Pub/Sub message format", 204 data_header = envelope['message']['attributes'] data_body = json.loads( gzip.decompress(base64.b64decode( envelope['message']['data'])).decode()) if receiver.receive_data(data_header, data_body): return "message received", 200 else: # pragma: no cover return "message to be resent", 400 # pragma: no cover
import google.cloud.logging import google.auth from xialib.service import service_factory # Global Setting app = Flask(__name__) project_id = google.auth.default()[1] # Configuration Load with open(os.path.join('.', 'config', 'global_conn_config.json')) as fp: global_conn_config = json.load(fp) with open(os.path.join('.', 'config', 'object_config.json')) as fp: object_config = json.load(fp) # Global Object Factory global_connectors = service_factory(global_conn_config) # Log configuration client = google.cloud.logging.Client() client.get_default_handler() client.setup_logging() @app.route('/', methods=['GET', 'POST']) def main(): loader = service_factory(object_config, global_connectors) if request.method == 'GET': return render_template("index.html"), 200 envelope = request.get_json() if not envelope: return "no Pub/Sub message received", 204