def send_message():
log.debug("got message: {}".format(request.json))
try:
msg_type = request.json.get("type")
msg_container = request.json.get("container")
msg_tenant = request.json.get("tenant")
msg_token = request.json.get("token")
if (not msg_type or not msg_type in valid_task_types):
raise HttpError("Task type is invalid", 500)
if not are_tenant_token_valid(tenant=msg_tenant, token=msg_token):
raise HttpError("Credentials are not valid", 500)
kafka_producer.send(msg_tenant,
request.json).get(timeout=kafka_timeout)
r = Response()
return r
except KafkaTimeoutError:
m = "Could not send msg to Kafka broker; timed out after {} sec.".format(
kafka_timeout)
log.exception(m)
raise HttpError(m, 500)
except KafkaError:
m = "Error handling message"
log.exception(m)
raise HttpError(m, 500)
def send_message():
log.debug("got message: {}".format(request.json))
worker_id = "MCMBluebox-{}-{}".format(socket.getfqdn(), os.getpid())
try:
msg_type = request.json.get("type")
msg_tenant = request.cookies.get(accountServer.COOKIE_NAME_TENANT)
if (not msg_type or not msg_type in valid_task_types):
raise HttpError("Request is invalid", 500)
"""
we only assert that the tenant/token in the request is valid
the token in the message is not validated, this is up to the recipient.
some msgs may not even contain a token...
"""
accountServer.assert_token_tenant_validity(request)
j = request.json
j["correlation"] = str(uuid.uuid4())
j["worker"] = worker_id
with __get_kafka_topic(msg_tenant).get_producer(
linger_ms=100) as producer:
producer.produce(value_serializer(request.json))
r = Response()
return r
except HttpError as e:
raise (e)
except Exception:
m = "Error sending message"
log.exception(m)
raise HttpError(m, 500)
def createConnection(req):
try:
t = req.cookies.get(COOKIE_NAME)
if not t: raise HttpError()
return SwiftConnect.SwiftConnect(t)
except:
raise HttpError("no token received from client", 401)
def create_objectclass():
swift = createConnection(request)
internal_data = InternalStorageManager(swift)
try:
class_definition = xform_header_names_on_classdef(request.json.get("objectClass"))
class_name = class_definition.get("name")
class_schema = class_definition.get("schema")
except AttributeError:
raise HttpError("malformed request", 400)
if not class_name or not class_schema:
raise HttpError("class name or class schema definition missing", 400)
class_names = internal_data.get_keys(INTERNALOCNAME)
if class_name in class_names:
raise HttpError("class already exists", 422)
try:
Draft4Validator(CLASS_SCHEMA, format_checker=FormatChecker()).validate(class_definition)
except ValidationError as e:
raise HttpError("invalid class definition: {}".format(e), 400)
internal_data.store_data(INTERNALOCNAME, class_name, json.dumps(class_definition))
return "", 201
def change_container(container_name):
swift = createConnection(request)
# TODO: check schema validity since somebody else could store a rouge class definition in the object store (via direct interfacing with the object store)
try:
container_definition = request.json.get("container")
container_name = container_definition.get("name")
except AttributeError:
raise HttpError("malformed request", 400)
if not container_name:
raise HttpError("container name is missing", 400)
containers = swift.get_container_list()[1]
if container_name not in [
container.get("name") for container in containers
]:
raise HttpError("container does not exist", 404)
container_metadata = {}
# object class
try:
class_name = container_definition.get("objectClass")
internal_data = InternalStorageManager(swift)
class_definition = internal_data.get_data(INTERNALOCNAME, class_name)
if class_name:
if class_definition is None:
raise HttpError("class does not exist", 404)
container_metadata[OBJECTCLASSFIELD] = class_name
except AttributeError:
pass # ignore empty or missing class definition
# selected fields
try:
internal_fields = container_definition.get("mdfi")
# print(internal_fields)
if (internal_fields != None):
container_metadata["x-container-meta-mdfi"] = json.dumps(
internal_fields)
except AttributeError:
pass # ignore empty or missing class definition
try:
fields = container_definition.get("mdf")
# print(fields)
if (fields != None):
container_metadata["x-container-meta-mdf"] = json.dumps(fields)
except AttributeError:
pass # ignore empty or missing class definition
swift.create_container(container_name, container_metadata)
return "", 201
def doLogin():
try:
user = request.json.get("username")
log.debug("authenticating user: {}".format(user))
password = request.json.get("password")
token = SwiftConnect.doAuthGetToken(user, password)
r = Response()
r.set_cookie(COOKIE_NAME, value=token)
return r
except ClientException as e:
log.exception("Login error")
raise HttpError(e.msg, 401)
except Exception:
log.exception("Login error")
raise HttpError("Internal Server Error", 500)
def update_object(container_name, object_name):
swift = createConnection(request)
try:
object_definition = request.json.get("metadata")
except AttributeError:
raise HttpError("malformed request", 400)
if not object_definition:
raise HttpError("object_definition is missing", 400)
# print(object_definition)
h = cleanHeaders(object_definition)
# print(h)
rsp = swift.update_object_metadata(object_name=object_name, container_name=container_name, metadata_dict=h)
return rsp["reason"], rsp["status"]
def doPlot():
accountServer.assert_token_tenant_validity(request)
nrDataSource = json.loads(
urlParse.unquote(request.args.get("nrDataSource")))
plotType = request.args.get("plotType")
logging.info("producing plot: {} for: {}".format(plotType, nrDataSource))
df = getDataFromNodeRed(nrDataSource=nrDataSource)
try:
logging.info(df)
if ('2bar' == plotType):
c = doPlot2(data=df, nrDataSource=nrDataSource)
elif ('bar' == plotType):
c = doPlot1(data=df, nrDataSource=nrDataSource)
elif ('bar_log' == plotType):
c = doPlot1log(data=df, nrDataSource=nrDataSource)
elif ('line' == plotType):
c = doPlot11(data=df, nrDataSource=nrDataSource)
elif ('box' == plotType):
c = doPlot_Box(data=df, nrDataSource=nrDataSource)
elif ('stackedBar' == plotType):
c = doPlot_stackedBar(data=df, nrDataSource=nrDataSource)
elif ('area' == plotType):
c = doPlot_Area(data=df, nrDataSource=nrDataSource)
else:
return Response("Plot type unknown", status=500)
return Response(json.dumps(c), mimetype="application/json")
except Exception as e:
logging.exception("plotting error:")
raise HttpError(
"the Node-RED result could not be plotted. Maybe wrong data format for the plot type? Check result table: {}"
.format(str(e)), 500)
def assert_no_xsrf(request):
"""
prevent cross-site-request-forgery (XSRF)
this is achieved by comparing a header field and a cookie.
see explanation here: https://en.wikipedia.org/wiki/Cross-site_request_forgery#Cookie-to-Header_Token
:param request:
:return:
"""
c = request.cookies.get(COOKIE_NAME_TOKEN)
h = request.headers.get(HEADER_NAME_TOKEN)
log.debug("CHECKING XSRF")
if (not c or not h):
raise (HttpError("no auth", 401))
log.debug("cookie: {} - header: {}".format(c, h))
if c != h:
raise (HttpError("XSRF?", 500))
def get_objects_in_container(container_name):
swift = createConnection(request)
optional_params = {}
limit = request.args.get("limit")
if limit is not None:
if limit.isdigit() and int(limit) > 0:
optional_params["limit"] = int(limit)
else:
log.debug("invalid query parameter limit: {}, for request: {}".format(limit, request.url))
raise HttpError("specified query parameter limit: {}, must be a positive integer".format(limit), 400)
marker = request.args.get("marker")
if marker is not None:
optional_params["marker"] = marker
prefix = request.args.get("prefix")
if prefix is not None:
optional_params["prefix"] = prefix
cts = swift.get_object_list(container_name, **optional_params)
resp = {}
resp["metadata"] = cts[0]
resp["metadata"]["objectClass"] = cts[0].get(OBJECTCLASSFIELD)
resp["metadata"]["objectCount"] = cts[0].get("x-container-object-count")
resp["objects"] = cts[1]
return Response(json.dumps(resp, sort_keys=True), mimetype="application/json")
def index(path=""):
if path[:5] != "swift":
# return render_template('index.html')
return send_file("angular/index.html")
else:
# this function is only called, when no other route matches
raise HttpError("the requested endpoint does not exist", 404)
def get_objectclass(class_name):
swift = createConnection(request)
internal_data = InternalStorageManager(swift)
class_def = internal_data.get_data(INTERNALOCNAME, class_name)
if not class_def:
raise HttpError("class does not exist", 404)
return Response(class_def, mimetype="application/json")
def getDataFromNodeRed(nrDataSource):
url = appConfig.nodered_url + nrDataSource['url']
logging.info("getting data from node red at: {}".format(url))
r = requests.get(url)
if r.status_code == 404:
raise HttpError(
"the Node-RED data source is not reachable: {}".format(url), 420)
try:
data = json.JSONDecoder(
object_pairs_hook=collections.OrderedDict).decode(
r.content.decode())
dataKeys = getListOfKeys(data[0])
df = pandas.DataFrame(data, columns=dataKeys)
df[dataKeys[0]] = df[dataKeys[0]].map(lambda x: str(x)[:20])
return df
except:
logging.exception("JSON parse error:")
raise HttpError("the Node-RED result is no valid JSON", 500)
def doLogin():
try:
user = request.json.get("user")
tenant = request.json.get("tenant")
password = request.json.get("password")
log.debug("authenticating tenant: {}, user: {}".format(tenant, user))
token = doAuthGetToken(tenant, user, password)
r = Response()
r.set_cookie(COOKIE_NAME_TOKEN, value=token)
r.set_cookie(COOKIE_NAME_TENANT, value=tenant)
r.set_cookie(COOKIE_NAME_USER, value=user)
return r
except ClientException as e:
log.exception("Login error")
raise HttpError(e.msg, 401)
except Exception:
log.exception("Login error")
raise HttpError("Internal Server Error", 500)
def delete_objectclass(class_name):
swift = createConnection(request)
internal_data = InternalStorageManager(swift)
class_def = internal_data.get_data(INTERNALOCNAME, class_name)
if not class_def:
raise HttpError("class does not exist", 404)
internal_data.remove_data(INTERNALOCNAME, class_name)
return "", 204
def assert_correct_tenant(request, tenant):
"""
this app needs to be configured for a single customer
until this is changed, we must only accept requests for this tenant
:param tenant:
:return:
"""
assert_token_tenant_validity(request)
t = get_tenant_from_request(request)
if (tenant != t):
raise HttpError("Tenant is invalid", 401)
def assert_token_tenant_validity(request):
"""
the credentials are checked against swift.
:param tenant:
:param token:
:return:
"""
assert_no_xsrf(request)
swift_store_url = get_swift_store_url_from_request(request)
try:
sw = client.Connection(
preauthtoken=request.cookies.get(COOKIE_NAME_TOKEN),
preauthurl=swift_store_url)
h = sw.head_account()
if not h:
raise HttpError("Token is not valid", 401)
except HttpError as e:
raise (e)
except Exception:
raise HttpError("Error checking token", 401)
def create_container():
swift = createConnection(request)
internal_data = InternalStorageManager(swift)
# TODO: check schema validity since somebody else could store a rouge class definition in the object store (via direct interfacing with the object store)
try:
container_definition = request.json.get("container")
container_name = container_definition.get("name")
container_sdos = container_definition.get("sdos", False)
except AttributeError:
raise HttpError("malformed request", 400)
if not container_name:
raise HttpError("container name is missing", 400)
if "/" in container_name:
raise HttpError(
"Container name contains '/'. This is only allowed in object names.",
400)
containers = swift.get_container_list()[1]
if container_name in [container.get("name") for container in containers]:
raise HttpError("container already exists", 422)
container_metadata = {}
if container_sdos:
container_metadata["x-container-meta-sdos"] = True
try:
class_name = container_definition.get("objectClass")
class_definition = internal_data.get_data(INTERNALOCNAME, class_name)
if class_name:
if class_definition is None:
raise HttpError("class does not exist", 404)
container_metadata = {OBJECTCLASSFIELD: class_name}
except AttributeError:
pass # ignore empty or missing class definition
swift.create_container(container_name, container_metadata)
return "", 201
def receive_messages(from_beginning=False):
"""
we subscribe to our tenant-topic to see all the sent messages. Our client-ID is tenant-bound
so that we receive all msgs for that tenant, and so that kafka can maintain a global offset for this tenant
our group-id is token bound so that it is unique across all consumers within this tenant; this will
make kafka broadcast msgs to all consumers within this tenant
--> every logged in session will see all new messages for the tenant and every message
will be seen by at least one of the sessions
:return:
"""
log.debug("receiving messages for: {}".format(request.json))
try:
msg_tenant = request.json.get("tenant")
msg_token = request.json.get("token")
if not are_tenant_token_valid(tenant=msg_tenant, token=msg_token):
raise HttpError("Credentials are not valid", 500)
c = KafkaConsumer(msg_tenant,
bootstrap_servers=appConfig.kafka_broker_endpoint,
client_id='mcmbb-{}'.format(msg_tenant),
group_id='mcmbb-{}-{}'.format(
msg_tenant, msg_token[25:]),
consumer_timeout_ms=5000,
enable_auto_commit=False)
if from_beginning:
c.poll()
c.seek_to_beginning()
# next line we actually get the msgs
msgs = list(c)
if not from_beginning:
c.commit()
c.close()
vals = [json.loads(m.value.decode("utf-8")) for m in msgs]
return Response(json.dumps(vals), mimetype="application/json")
except Exception:
m = "Error retrieving messages"
log.exception(m)
raise HttpError(m, 500)
def delete_object(container_name, object_name):
swift = createConnection(request)
metadata = swift.get_object_metadata(container_name, object_name)
retentimestamp = metadata.get("x-object-meta-retentiontime")
if retentimestamp and not isRetentionPeriodExpired(retentimestamp):
error_msg = "Deletion failed due to retention enforcement, file cannot be deleted till {}!".format(
time.strftime("%a, %d. %B %Y", time.localtime(int(retentimestamp))))
log.debug(error_msg)
raise HttpError(error_msg, 412)
swift.delete_object(container_name, object_name)
return "", 204
def assert_token_tenant_validity(request, check_xsrf=True):
"""
the credentials are checked against swift.
:param tenant:
:param token:
:return:
"""
if check_xsrf:
assert_no_xsrf(request)
url = get_swift_url_from_request(request)
token = get_token_from_request(request)
try:
sw = client.Connection(preauthtoken=token, preauthurl=url)
h = sw.head_account()
if not h:
raise HttpError("Token is not valid", 401)
except HttpError as e:
raise (e)
except Exception:
raise HttpError("Error checking token", 401)
def receive_messages(from_beginning=False):
"""
we subscribe to our tenant-topic to see all the sent messages. Our client-ID is tenant-bound
so that we receive all msgs for that tenant, and so that kafka can maintain a global offset for this tenant
our group-id is token bound so that it is unique across all consumers within this tenant; this will
make kafka broadcast msgs to all consumers within this tenant
--> every logged in session will see all new messages for the tenant and every message
will be seen by at least one of the sessions
:return:
"""
from pykafka.exceptions import SocketDisconnectedError
try:
accountServer.assert_token_tenant_validity(request)
msg_tenant = request.cookies.get(accountServer.COOKIE_NAME_TENANT)
msg_client_id = request.cookies.get(
accountServer.COOKIE_NAME_SESSION_ID)
consumer_group = 'mcmbb-{}-{}'.format(msg_tenant,
msg_client_id).encode('utf-8')
consumer = __get_kafka_consumer(topic=msg_tenant,
consumer_group=consumer_group)
if from_beginning:
partition_offset_pairs = [(p, p.earliest_available_offset())
for p in consumer.partitions.values()]
consumer.reset_offsets(partition_offsets=partition_offset_pairs)
vals = [__try_parse_msg_content(m) for m in consumer]
if not from_beginning:
consumer.commit_offsets()
return Response(json.dumps(vals), mimetype="application/json")
except HttpError as e:
"""
make sure we don't catch existing HTTP errors here and turn them into
meaningless 500s
"""
raise e
except SocketDisconnectedError as e:
log.exception(
"Connection to Broker closed unexpectedly; returned empty response to client."
)
return Response(json.dumps({}), mimetype="application/json")
except Exception:
m = "Error retrieving messages"
log.exception(m)
raise HttpError(m, 500)
def getDataFromNodeRed(nrDataSource, container_filter=None):
url = configuration.nodered_url + nrDataSource['url']
params = {"container_filter": container_filter}
logging.info("getting data from node red at: {} with params: {}".format(
url, params))
r = requests.get(url, params=params)
if r.status_code == 404:
raise HttpError(
"the Node-RED data source is not reachable: {}".format(url), 420)
try:
data = json.JSONDecoder(
object_pairs_hook=collections.OrderedDict).decode(
r.content.decode())
dataKeys = getListOfKeys(data[0])
df = pandas.DataFrame(data, columns=dataKeys)
df[dataKeys[0]] = df[dataKeys[0]].map(lambda x: str(x)[:20])
return df
except IndexError as e:
logging.warning("no data returned {}".format(e))
raise HttpError("query returned no data", 404)
except:
logging.exception("JSON parse error:")
raise HttpError("the Node-RED result is no valid JSON", 500)
def create_object(container_name):
swift = createConnection(request)
# returns werkzeug.datastructures.FileStorage i.e. file-like
# Underlying stream is either BytesIO for small files or _TemporaryFileWrapper for large files
file = request.files["objectName"]
object_name = file.filename
headers = {}
retentionDate = request.form["retentionDate"]
if retentionDate:
try:
dateutil.parser.parse(retentionDate)
headers[RETENTIONFIELD] = retentionDate
except Exception as e:
log.debug(
"invalid date format for form parameter retentionDate: {}".
format(retentionDate))
raise HttpError(
"invalid date format for form parameter retentionDate: {}".
format(retentionDate), 400)
class_metadata_json = request.form["metadata"]
if class_metadata_json:
class_metadata = json.loads(class_metadata_json)
class_name = swift.get_container_metadata(container_name).get(
OBJECTCLASSFIELD)
if class_name:
internal_data = InternalStorageManager(swift)
class_definition = json.loads(
internal_data.get_data(INTERNALOCNAME, class_name))
Draft4Validator(
class_definition,
format_checker=FormatChecker()).validate(class_metadata)
for field in class_metadata.keys():
val = class_metadata[field]
if val is not None:
field_header = xform_header_names(field)
xformed_class_name = xform_header_names(class_name)
headers["X-Object-Meta-Class-" + xformed_class_name + "-" +
field_header] = class_metadata[field]
swift.object_upload(object_name,
container_name,
file,
headers,
as_stream=False)
return "", 201
def receive_messages(from_beginning=False):
"""
we subscribe to our tenant-topic to see all the sent messages. Our client-ID is tenant-bound
so that we receive all msgs for that tenant, and so that kafka can maintain a global offset for this tenant
our group-id is token bound so that it is unique across all consumers within this tenant; this will
make kafka broadcast msgs to all consumers within this tenant
--> every logged in session will see all new messages for the tenant and every message
will be seen by at least one of the sessions
:return:
"""
log.debug("receiving messages for: {}".format(request.json))
try:
msg_tenant = request.json.get("tenant")
msg_client_id = request.json.get("client_id")
accountServer.assert_no_xsrf(request)
accountServer.assert_token_tenant_validity(request)
accountServer.assert_correct_tenant(request, msg_tenant)
consumer_group = 'mcmbb-{}-{}'.format(msg_tenant,
msg_client_id).encode('utf-8')
topic = __get_kafka_topic(msg_tenant)
consumer = topic.get_simple_consumer(consumer_group=consumer_group,
consumer_id=consumer_group,
consumer_timeout_ms=100,
auto_commit_enable=False)
if from_beginning:
partition_offset_pairs = [(p, p.latest_available_offset())
for p in consumer.partitions.values()]
consumer.reset_offsets(partition_offsets=partition_offset_pairs)
vals = [__try_parse_msg_content(m) for m in consumer]
if not from_beginning:
consumer.commit_offsets()
consumer.stop()
return Response(json.dumps(vals), mimetype="application/json")
except HttpError as e:
raise e
except Exception:
m = "Error retrieving messages"
log.exception(m)
raise HttpError(m, 500)
def doPlot():
accountServer.assert_token_tenant_validity(request)
nrDataSource = json.loads(
urllib.parse.unquote(request.args.get("nrDataSource")))
plotType = request.args.get("plotType")
container_filter = request.args.get("container_filter", None)
logging.info("producing plot: {} for: {}".format(plotType, nrDataSource))
df = getDataFromNodeRed(nrDataSource=nrDataSource,
container_filter=container_filter)
try:
logging.info(df)
if ('bar' == plotType):
c = bokeh_plot_bar(data=df,
nrDataSource=nrDataSource,
container_filter=container_filter)
elif ('bar_log' == plotType):
c = bokeh_plot_bar(data=df,
nrDataSource=nrDataSource,
logScale="log",
container_filter=container_filter)
elif ('line' == plotType):
c = bokeh_plot_line(data=df,
nrDataSource=nrDataSource,
container_filter=container_filter)
elif ('line_log' == plotType):
c = bokeh_plot_line(data=df,
nrDataSource=nrDataSource,
logScale="log",
container_filter=container_filter)
elif ('pie' == plotType):
c = bokeh_plot_pie(data=df,
nrDataSource=nrDataSource,
container_filter=container_filter)
else:
return Response("Plot type unknown", status=500)
return Response(json.dumps(c), mimetype="application/json")
except Exception as e:
logging.exception("plotting error:")
raise HttpError(
"the Node-RED result could not be plotted. Maybe wrong data format for the plot type? Check result table: {}"
.format(e), 500)
def strip_url_to_tenant_id(url):
if not url.startswith(configuration.swift_store_url_valid_prefix):
raise HttpError("swift returned wrong storage URL")
return url[len(configuration.swift_store_url_valid_prefix):]
def change_object_metadata(container_name, object_name):
raise HttpError("funcion not implemented yet")
def change_container_metadata(container_name, metadata):
# print(metadata)
raise HttpError("funcion not implemented yet")
def get_user_from_request(request):
t = request.cookies.get(COOKIE_NAME_USER)
if t:
return t
else:
raise HttpError("cookie missing (user)", 401)
