def doPlot():
accountServer.assert_token_tenant_validity(request)
nrDataSource = json.loads(urlParse.unquote(request.args.get("nrDataSource")))
plotType = request.args.get("plotType")
logging.info("producing plot: {} for: {}".format(plotType, nrDataSource))
df = getDataFromNodeRed(nrDataSource=nrDataSource)
try:
logging.info(df)
if ('2bar' == plotType):
c = doPlot2(data=df, nrDataSource=nrDataSource)
elif ('bar' == plotType):
c = doPlot1(data=df, nrDataSource=nrDataSource)
elif ('bar_log' == plotType):
c = doPlot1log(data=df, nrDataSource=nrDataSource)
elif ('line' == plotType):
c = doPlot11(data=df, nrDataSource=nrDataSource)
elif ('box' == plotType):
c = doPlot_Box(data=df, nrDataSource=nrDataSource)
elif ('stackedBar' == plotType):
c = doPlot_stackedBar(data=df, nrDataSource=nrDataSource)
elif ('area' == plotType):
c = doPlot_Area(data=df, nrDataSource=nrDataSource)
else:
return Response("Plot type unknown", status=500)
return Response(json.dumps(c), mimetype="application/json")
except Exception as e:
logging.exception("plotting error:")
raise HttpError(
"the Node-RED result could not be plotted. Maybe wrong data format for the plot type? Check result table: {}".format(
str(e)), 500)
def doPlot():
accountServer.assert_token_tenant_validity(request)
nrDataSource = json.loads(
urlParse.unquote(request.args.get("nrDataSource")))
plotType = request.args.get("plotType")
logging.info("producing plot: {} for: {}".format(plotType, nrDataSource))
df = getDataFromNodeRed(nrDataSource=nrDataSource)
try:
logging.info(df)
if ('2bar' == plotType):
c = doPlot2(data=df, nrDataSource=nrDataSource)
elif ('bar' == plotType):
c = doPlot1(data=df, nrDataSource=nrDataSource)
elif ('bar_log' == plotType):
c = doPlot1log(data=df, nrDataSource=nrDataSource)
elif ('line' == plotType):
c = doPlot11(data=df, nrDataSource=nrDataSource)
elif ('box' == plotType):
c = doPlot_Box(data=df, nrDataSource=nrDataSource)
elif ('stackedBar' == plotType):
c = doPlot_stackedBar(data=df, nrDataSource=nrDataSource)
elif ('area' == plotType):
c = doPlot_Area(data=df, nrDataSource=nrDataSource)
else:
return Response("Plot type unknown", status=500)
return Response(json.dumps(c), mimetype="application/json")
except Exception as e:
logging.exception("plotting error:")
raise HttpError(
"the Node-RED result could not be plotted. Maybe wrong data format for the plot type? Check result table: {}"
.format(str(e)), 500)
def send_message():
log.debug("got message: {}".format(request.json))
worker_id = "MCMBluebox-{}-{}".format(socket.getfqdn(), os.getpid())
try:
msg_type = request.json.get("type")
msg_tenant = request.cookies.get(accountServer.COOKIE_NAME_TENANT)
if (not msg_type or not msg_type in valid_task_types):
raise HttpError("Request is invalid", 500)
"""
we only assert that the tenant/token in the request is valid
the token in the message is not validated, this is up to the recipient.
some msgs may not even contain a token...
"""
accountServer.assert_token_tenant_validity(request)
j = request.json
j["correlation"] = str(uuid.uuid4())
j["worker"] = worker_id
with __get_kafka_topic(msg_tenant).get_producer(
linger_ms=100) as producer:
producer.produce(value_serializer(request.json))
r = Response()
return r
except HttpError as e:
raise (e)
except Exception:
m = "Error sending message"
log.exception(m)
raise HttpError(m, 500)
def getNodeRedEnpointList():
accountServer.assert_token_tenant_validity(request)
n = requests.get(appConfig.nodered_url + "/flows").json()
sources = []
for s in n:
# Node-RED has a strange API... we can't reconstruct node/flow relationships...
# if ('tab' == s['type'] and 'label' in s):
# thisFlowName = s['label'] + "->"
if ('http in' == s['type'] and 'url' in s):
sources.append({"url": s['url'], "name": s['name']})
return Response(json.dumps(sources), mimetype="application/json")
def getNodeRedEnpointList():
accountServer.assert_token_tenant_validity(request)
n = requests.get(appConfig.nodered_url + "/flows").json()
sources = []
for s in n:
# Node-RED has a strange API... we can't reconstruct node/flow relationships...
# if ('tab' == s['type'] and 'label' in s):
# thisFlowName = s['label'] + "->"
if ('http in' == s['type'] and 'url' in s):
sources.append({"url": s['url'], "name": s['name']})
return Response(json.dumps(sources), mimetype="application/json")
def receive_messages(from_beginning=False):
"""
we subscribe to our tenant-topic to see all the sent messages. Our client-ID is tenant-bound
so that we receive all msgs for that tenant, and so that kafka can maintain a global offset for this tenant
our group-id is token bound so that it is unique across all consumers within this tenant; this will
make kafka broadcast msgs to all consumers within this tenant
--> every logged in session will see all new messages for the tenant and every message
will be seen by at least one of the sessions
:return:
"""
from pykafka.exceptions import SocketDisconnectedError
try:
accountServer.assert_token_tenant_validity(request)
msg_tenant = request.cookies.get(accountServer.COOKIE_NAME_TENANT)
msg_client_id = request.cookies.get(
accountServer.COOKIE_NAME_SESSION_ID)
consumer_group = 'mcmbb-{}-{}'.format(msg_tenant,
msg_client_id).encode('utf-8')
consumer = __get_kafka_consumer(topic=msg_tenant,
consumer_group=consumer_group)
if from_beginning:
partition_offset_pairs = [(p, p.earliest_available_offset())
for p in consumer.partitions.values()]
consumer.reset_offsets(partition_offsets=partition_offset_pairs)
vals = [__try_parse_msg_content(m) for m in consumer]
if not from_beginning:
consumer.commit_offsets()
return Response(json.dumps(vals), mimetype="application/json")
except HttpError as e:
"""
make sure we don't catch existing HTTP errors here and turn them into
meaningless 500s
"""
raise e
except SocketDisconnectedError as e:
log.exception(
"Connection to Broker closed unexpectedly; returned empty response to client."
)
return Response(json.dumps({}), mimetype="application/json")
except Exception:
m = "Error retrieving messages"
log.exception(m)
raise HttpError(m, 500)
def doTable():
accountServer.assert_token_tenant_validity(request)
nrDataSource = json.loads(urlParse.unquote(request.args.get("nrDataSource")))
logging.info("producing table for: {}".format(nrDataSource))
data = getDataFromNodeRed(nrDataSource=nrDataSource)
# log.debug("our pandas data frame is: {}".format(data.to_json(orient="records")))
info = StringIO()
data.info(verbose=False, buf=info)
r = {
"table": data[:50].to_json(orient="records"),
"info": info.getvalue(),
"truncated": (len(data) > 50)
}
# print(r)
return Response(json.dumps(r), mimetype="application/json")
def doTable():
accountServer.assert_token_tenant_validity(request)
nrDataSource = json.loads(
urlParse.unquote(request.args.get("nrDataSource")))
logging.info("producing table for: {}".format(nrDataSource))
data = getDataFromNodeRed(nrDataSource=nrDataSource)
# log.debug("our pandas data frame is: {}".format(data.to_json(orient="records")))
info = StringIO()
data.info(verbose=False, buf=info)
r = {
"table": data[:50].to_json(orient="records"),
"info": info.getvalue(),
"truncated": (len(data) > 50)
}
# print(r)
return Response(json.dumps(r), mimetype="application/json")
def receive_messages(from_beginning=False):
"""
we subscribe to our tenant-topic to see all the sent messages. Our client-ID is tenant-bound
so that we receive all msgs for that tenant, and so that kafka can maintain a global offset for this tenant
our group-id is token bound so that it is unique across all consumers within this tenant; this will
make kafka broadcast msgs to all consumers within this tenant
--> every logged in session will see all new messages for the tenant and every message
will be seen by at least one of the sessions
:return:
"""
log.debug("receiving messages for: {}".format(request.json))
try:
msg_tenant = request.json.get("tenant")
msg_client_id = request.json.get("client_id")
accountServer.assert_no_xsrf(request)
accountServer.assert_token_tenant_validity(request)
accountServer.assert_correct_tenant(request, msg_tenant)
consumer_group = 'mcmbb-{}-{}'.format(msg_tenant,
msg_client_id).encode('utf-8')
topic = __get_kafka_topic(msg_tenant)
consumer = topic.get_simple_consumer(consumer_group=consumer_group,
consumer_id=consumer_group,
consumer_timeout_ms=100,
auto_commit_enable=False)
if from_beginning:
partition_offset_pairs = [(p, p.latest_available_offset())
for p in consumer.partitions.values()]
consumer.reset_offsets(partition_offsets=partition_offset_pairs)
vals = [__try_parse_msg_content(m) for m in consumer]
if not from_beginning:
consumer.commit_offsets()
consumer.stop()
return Response(json.dumps(vals), mimetype="application/json")
except HttpError as e:
raise e
except Exception:
m = "Error retrieving messages"
log.exception(m)
raise HttpError(m, 500)
def getTableStructure():
# check if user is logged in
accountServer.assert_token_tenant_validity(request)
t = accountServer.get_tenant_from_request(request)
d = __get_db_connection_for_tenant(t)
# Establish connection to PostgreSQL database
# conn = sqlite3.connect("/tmp/metadata.sqlite") #SQLITE
with psycopg2.connect(**d) as conn:
with conn.cursor() as cursor:
# Retrieve all table names
# cursor.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name ASC") #SQLITE
cursor.execute(
"SELECT table_name FROM information_schema.tables WHERE table_schema=%s;",
("public", ))
tableNames = cursor.fetchall()
tableData = {}
# Retrieve the column names and first 5 rows for each table
for table in tableNames:
# cursor.execute("PRAGMA table_info(" + table[0] + ")") #SQLITE
cursor.execute(
"SELECT column_name,data_type FROM information_schema.columns WHERE table_name = %s;",
(table[0], ))
columnNames = cursor.fetchall()
# cursor.execute("SELECT * FROM " + table[0] + " LIMIT 5") #SQLITE
cursor.execute("SELECT * FROM {} LIMIT 5".format(table[0]))
rowEntries = cursor.fetchall()
# Dictionary which combines column names and row entries
columnStructure = {}
# format the column names and types
columnList = [
"{} ({})".format(c[0], c[1]) for c in columnNames
]
# Populate final dictionary table Data
columnStructure['columnNames'] = columnList
columnStructure['rowEntries'] = rowEntries
tableData[table[0]] = columnStructure
return Response(json.dumps(tableData), mimetype="application/json")
def doPlot():
accountServer.assert_token_tenant_validity(request)
nrDataSource = json.loads(
urllib.parse.unquote(request.args.get("nrDataSource")))
plotType = request.args.get("plotType")
container_filter = request.args.get("container_filter", None)
logging.info("producing plot: {} for: {}".format(plotType, nrDataSource))
df = getDataFromNodeRed(nrDataSource=nrDataSource,
container_filter=container_filter)
try:
logging.info(df)
if ('bar' == plotType):
c = bokeh_plot_bar(data=df,
nrDataSource=nrDataSource,
container_filter=container_filter)
elif ('bar_log' == plotType):
c = bokeh_plot_bar(data=df,
nrDataSource=nrDataSource,
logScale="log",
container_filter=container_filter)
elif ('line' == plotType):
c = bokeh_plot_line(data=df,
nrDataSource=nrDataSource,
container_filter=container_filter)
elif ('line_log' == plotType):
c = bokeh_plot_line(data=df,
nrDataSource=nrDataSource,
logScale="log",
container_filter=container_filter)
elif ('pie' == plotType):
c = bokeh_plot_pie(data=df,
nrDataSource=nrDataSource,
container_filter=container_filter)
else:
return Response("Plot type unknown", status=500)
return Response(json.dumps(c), mimetype="application/json")
except Exception as e:
logging.exception("plotting error:")
raise HttpError(
"the Node-RED result could not be plotted. Maybe wrong data format for the plot type? Check result table: {}"
.format(e), 500)
