def receive_messages(from_beginning=False):
"""
we subscribe to our tenant-topic to see all the sent messages. Our client-ID is tenant-bound
so that we receive all msgs for that tenant, and so that kafka can maintain a global offset for this tenant
our group-id is token bound so that it is unique across all consumers within this tenant; this will
make kafka broadcast msgs to all consumers within this tenant
--> every logged in session will see all new messages for the tenant and every message
will be seen by at least one of the sessions
:return:
"""
from pykafka.exceptions import SocketDisconnectedError
try:
msg_tenant = accountServer.get_and_assert_tenant_from_request(request)
msg_client_id = request.cookies.get(accountServer.COOKIE_NAME_SESSION_ID)
consumer_group = "mcmbb-{}-{}".format(msg_tenant, msg_client_id).encode("utf-8")
consumer = __get_kafka_consumer(topic=msg_tenant, consumer_group=consumer_group)
if from_beginning:
partition_offset_pairs = [(p, p.earliest_available_offset()) for p in consumer.partitions.values()]
consumer.reset_offsets(partition_offsets=partition_offset_pairs)
vals = [__try_parse_msg_content(m) for m in consumer]
if not from_beginning:
consumer.commit_offsets()
log.debug(vals)
return Response(json.dumps(vals), mimetype="application/json")
except HttpError as e:
"""
make sure we don't catch existing HTTP errors here and turn them into
meaningless 500s
"""
raise e
except SocketDisconnectedError as e:
log.exception("Connection to Broker closed unexpectedly; returned empty response to client.")
return Response(json.dumps({}), mimetype="application/json")
except Exception:
m = "Error retrieving messages"
log.exception(m)
raise HttpError(m, 500)
def getTableStructure():
# check if user is logged in
t = accountServer.get_and_assert_tenant_from_request(request)
d = __get_db_connection_for_tenant(t)
# Establish connection to PostgreSQL database
# conn = sqlite3.connect("/tmp/metadata.sqlite") #SQLITE
with psycopg2.connect(**d) as conn:
with conn.cursor() as cursor:
# Retrieve all table names
# cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name ASC") #SQLITE
cursor.execute("SELECT table_name FROM information_schema.tables WHERE table_schema=%s;", ("public",))
tableNames = cursor.fetchall()
tableData = {}
# Retrieve the column names and first 5 rows for each table
for table in tableNames:
# cursor.execute("PRAGMA table_info(" + table[0] + ")") #SQLITE
cursor.execute("SELECT column_name,data_type FROM information_schema.columns WHERE table_name = %s;",
(table[0],))
columnNames = cursor.fetchall()
# cursor.execute("SELECT * FROM " + table[0] + " LIMIT 5") #SQLITE
cursor.execute("SELECT * FROM {} LIMIT 5".format(table[0]))
rowEntries = cursor.fetchall()
# Dictionary which combines column names and row entries
columnStructure = {}
# format the column names and types
columnList = ["{} ({})".format(c[0], c[1]) for c in columnNames]
# Populate final dictionary table Data
columnStructure['columnNames'] = columnList
columnStructure['rowEntries'] = rowEntries
tableData[table[0]] = columnStructure
return Response(json.dumps(tableData), mimetype="application/json")
def send_message():
log.debug("got message: {}".format(request.json))
worker_id = "MCMBluebox-{}-{}".format(socket.getfqdn(), os.getpid())
try:
j = request.json
msg_type = j["type"]
j["correlation"] = str(uuid.uuid4())
j["worker"] = worker_id
msg_tenant = accountServer.get_and_assert_tenant_from_request(request)
if not msg_type or not msg_type in valid_task_types:
raise HttpError("Request is invalid", 500)
with __get_kafka_topic(msg_tenant).get_producer(linger_ms=100) as producer:
producer.produce(value_serializer(request.json))
r = Response()
return r
except HttpError as e:
raise (e)
except Exception:
m = "Error sending message"
log.exception(m)
raise HttpError(m, 500)
