def receive_messages(from_beginning=False): """ we subscribe to our tenant-topic to see all the sent messages. Our client-ID is tenant-bound so that we receive all msgs for that tenant, and so that kafka can maintain a global offset for this tenant our group-id is token bound so that it is unique across all consumers within this tenant; this will make kafka broadcast msgs to all consumers within this tenant --> every logged in session will see all new messages for the tenant and every message will be seen by at least one of the sessions :return: """ from pykafka.exceptions import SocketDisconnectedError try: msg_tenant = accountServer.get_and_assert_tenant_from_request(request) msg_client_id = request.cookies.get(accountServer.COOKIE_NAME_SESSION_ID) consumer_group = "mcmbb-{}-{}".format(msg_tenant, msg_client_id).encode("utf-8") consumer = __get_kafka_consumer(topic=msg_tenant, consumer_group=consumer_group) if from_beginning: partition_offset_pairs = [(p, p.earliest_available_offset()) for p in consumer.partitions.values()] consumer.reset_offsets(partition_offsets=partition_offset_pairs) vals = [__try_parse_msg_content(m) for m in consumer] if not from_beginning: consumer.commit_offsets() log.debug(vals) return Response(json.dumps(vals), mimetype="application/json") except HttpError as e: """ make sure we don't catch existing HTTP errors here and turn them into meaningless 500s """ raise e except SocketDisconnectedError as e: log.exception("Connection to Broker closed unexpectedly; returned empty response to client.") return Response(json.dumps({}), mimetype="application/json") except Exception: m = "Error retrieving messages" log.exception(m) raise HttpError(m, 500)
def getTableStructure(): # check if user is logged in t = accountServer.get_and_assert_tenant_from_request(request) d = __get_db_connection_for_tenant(t) # Establish connection to PostgreSQL database # conn = sqlite3.connect("/tmp/metadata.sqlite") #SQLITE with psycopg2.connect(**d) as conn: with conn.cursor() as cursor: # Retrieve all table names # cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name ASC") #SQLITE cursor.execute("SELECT table_name FROM information_schema.tables WHERE table_schema=%s;", ("public",)) tableNames = cursor.fetchall() tableData = {} # Retrieve the column names and first 5 rows for each table for table in tableNames: # cursor.execute("PRAGMA table_info(" + table[0] + ")") #SQLITE cursor.execute("SELECT column_name,data_type FROM information_schema.columns WHERE table_name = %s;", (table[0],)) columnNames = cursor.fetchall() # cursor.execute("SELECT * FROM " + table[0] + " LIMIT 5") #SQLITE cursor.execute("SELECT * FROM {} LIMIT 5".format(table[0])) rowEntries = cursor.fetchall() # Dictionary which combines column names and row entries columnStructure = {} # format the column names and types columnList = ["{} ({})".format(c[0], c[1]) for c in columnNames] # Populate final dictionary table Data columnStructure['columnNames'] = columnList columnStructure['rowEntries'] = rowEntries tableData[table[0]] = columnStructure return Response(json.dumps(tableData), mimetype="application/json")
def send_message(): log.debug("got message: {}".format(request.json)) worker_id = "MCMBluebox-{}-{}".format(socket.getfqdn(), os.getpid()) try: j = request.json msg_type = j["type"] j["correlation"] = str(uuid.uuid4()) j["worker"] = worker_id msg_tenant = accountServer.get_and_assert_tenant_from_request(request) if not msg_type or not msg_type in valid_task_types: raise HttpError("Request is invalid", 500) with __get_kafka_topic(msg_tenant).get_producer(linger_ms=100) as producer: producer.produce(value_serializer(request.json)) r = Response() return r except HttpError as e: raise (e) except Exception: m = "Error sending message" log.exception(m) raise HttpError(m, 500)