def send_message(): log.debug("got message: {}".format(request.json)) try: msg_type = request.json.get("type") msg_container = request.json.get("container") msg_tenant = request.json.get("tenant") msg_token = request.json.get("token") if (not msg_type or not msg_type in valid_task_types): raise HttpError("Task type is invalid", 500) if not are_tenant_token_valid(tenant=msg_tenant, token=msg_token): raise HttpError("Credentials are not valid", 500) kafka_producer.send(msg_tenant, request.json).get(timeout=kafka_timeout) r = Response() return r except KafkaTimeoutError: m = "Could not send msg to Kafka broker; timed out after {} sec.".format( kafka_timeout) log.exception(m) raise HttpError(m, 500) except KafkaError: m = "Error handling message" log.exception(m) raise HttpError(m, 500)
def send_message(): log.debug("got message: {}".format(request.json)) worker_id = "MCMBluebox-{}-{}".format(socket.getfqdn(), os.getpid()) try: msg_type = request.json.get("type") msg_tenant = request.cookies.get(accountServer.COOKIE_NAME_TENANT) if (not msg_type or not msg_type in valid_task_types): raise HttpError("Request is invalid", 500) """ we only assert that the tenant/token in the request is valid the token in the message is not validated, this is up to the recipient. some msgs may not even contain a token... """ accountServer.assert_token_tenant_validity(request) j = request.json j["correlation"] = str(uuid.uuid4()) j["worker"] = worker_id with __get_kafka_topic(msg_tenant).get_producer( linger_ms=100) as producer: producer.produce(value_serializer(request.json)) r = Response() return r except HttpError as e: raise (e) except Exception: m = "Error sending message" log.exception(m) raise HttpError(m, 500)
def createConnection(req): try: t = req.cookies.get(COOKIE_NAME) if not t: raise HttpError() return SwiftConnect.SwiftConnect(t) except: raise HttpError("no token received from client", 401)
def create_objectclass(): swift = createConnection(request) internal_data = InternalStorageManager(swift) try: class_definition = xform_header_names_on_classdef(request.json.get("objectClass")) class_name = class_definition.get("name") class_schema = class_definition.get("schema") except AttributeError: raise HttpError("malformed request", 400) if not class_name or not class_schema: raise HttpError("class name or class schema definition missing", 400) class_names = internal_data.get_keys(INTERNALOCNAME) if class_name in class_names: raise HttpError("class already exists", 422) try: Draft4Validator(CLASS_SCHEMA, format_checker=FormatChecker()).validate(class_definition) except ValidationError as e: raise HttpError("invalid class definition: {}".format(e), 400) internal_data.store_data(INTERNALOCNAME, class_name, json.dumps(class_definition)) return "", 201
def change_container(container_name): swift = createConnection(request) # TODO: check schema validity since somebody else could store a rouge class definition in the object store (via direct interfacing with the object store) try: container_definition = request.json.get("container") container_name = container_definition.get("name") except AttributeError: raise HttpError("malformed request", 400) if not container_name: raise HttpError("container name is missing", 400) containers = swift.get_container_list()[1] if container_name not in [ container.get("name") for container in containers ]: raise HttpError("container does not exist", 404) container_metadata = {} # object class try: class_name = container_definition.get("objectClass") internal_data = InternalStorageManager(swift) class_definition = internal_data.get_data(INTERNALOCNAME, class_name) if class_name: if class_definition is None: raise HttpError("class does not exist", 404) container_metadata[OBJECTCLASSFIELD] = class_name except AttributeError: pass # ignore empty or missing class definition # selected fields try: internal_fields = container_definition.get("mdfi") # print(internal_fields) if (internal_fields != None): container_metadata["x-container-meta-mdfi"] = json.dumps( internal_fields) except AttributeError: pass # ignore empty or missing class definition try: fields = container_definition.get("mdf") # print(fields) if (fields != None): container_metadata["x-container-meta-mdf"] = json.dumps(fields) except AttributeError: pass # ignore empty or missing class definition swift.create_container(container_name, container_metadata) return "", 201
def doLogin(): try: user = request.json.get("username") log.debug("authenticating user: {}".format(user)) password = request.json.get("password") token = SwiftConnect.doAuthGetToken(user, password) r = Response() r.set_cookie(COOKIE_NAME, value=token) return r except ClientException as e: log.exception("Login error") raise HttpError(e.msg, 401) except Exception: log.exception("Login error") raise HttpError("Internal Server Error", 500)
def update_object(container_name, object_name): swift = createConnection(request) try: object_definition = request.json.get("metadata") except AttributeError: raise HttpError("malformed request", 400) if not object_definition: raise HttpError("object_definition is missing", 400) # print(object_definition) h = cleanHeaders(object_definition) # print(h) rsp = swift.update_object_metadata(object_name=object_name, container_name=container_name, metadata_dict=h) return rsp["reason"], rsp["status"]
def doPlot(): accountServer.assert_token_tenant_validity(request) nrDataSource = json.loads( urlParse.unquote(request.args.get("nrDataSource"))) plotType = request.args.get("plotType") logging.info("producing plot: {} for: {}".format(plotType, nrDataSource)) df = getDataFromNodeRed(nrDataSource=nrDataSource) try: logging.info(df) if ('2bar' == plotType): c = doPlot2(data=df, nrDataSource=nrDataSource) elif ('bar' == plotType): c = doPlot1(data=df, nrDataSource=nrDataSource) elif ('bar_log' == plotType): c = doPlot1log(data=df, nrDataSource=nrDataSource) elif ('line' == plotType): c = doPlot11(data=df, nrDataSource=nrDataSource) elif ('box' == plotType): c = doPlot_Box(data=df, nrDataSource=nrDataSource) elif ('stackedBar' == plotType): c = doPlot_stackedBar(data=df, nrDataSource=nrDataSource) elif ('area' == plotType): c = doPlot_Area(data=df, nrDataSource=nrDataSource) else: return Response("Plot type unknown", status=500) return Response(json.dumps(c), mimetype="application/json") except Exception as e: logging.exception("plotting error:") raise HttpError( "the Node-RED result could not be plotted. Maybe wrong data format for the plot type? Check result table: {}" .format(str(e)), 500)
def assert_no_xsrf(request): """ prevent cross-site-request-forgery (XSRF) this is achieved by comparing a header field and a cookie. see explanation here: https://en.wikipedia.org/wiki/Cross-site_request_forgery#Cookie-to-Header_Token :param request: :return: """ c = request.cookies.get(COOKIE_NAME_TOKEN) h = request.headers.get(HEADER_NAME_TOKEN) log.debug("CHECKING XSRF") if (not c or not h): raise (HttpError("no auth", 401)) log.debug("cookie: {} - header: {}".format(c, h)) if c != h: raise (HttpError("XSRF?", 500))
def get_objects_in_container(container_name): swift = createConnection(request) optional_params = {} limit = request.args.get("limit") if limit is not None: if limit.isdigit() and int(limit) > 0: optional_params["limit"] = int(limit) else: log.debug("invalid query parameter limit: {}, for request: {}".format(limit, request.url)) raise HttpError("specified query parameter limit: {}, must be a positive integer".format(limit), 400) marker = request.args.get("marker") if marker is not None: optional_params["marker"] = marker prefix = request.args.get("prefix") if prefix is not None: optional_params["prefix"] = prefix cts = swift.get_object_list(container_name, **optional_params) resp = {} resp["metadata"] = cts[0] resp["metadata"]["objectClass"] = cts[0].get(OBJECTCLASSFIELD) resp["metadata"]["objectCount"] = cts[0].get("x-container-object-count") resp["objects"] = cts[1] return Response(json.dumps(resp, sort_keys=True), mimetype="application/json")
def index(path=""): if path[:5] != "swift": # return render_template('index.html') return send_file("angular/index.html") else: # this function is only called, when no other route matches raise HttpError("the requested endpoint does not exist", 404)
def get_objectclass(class_name): swift = createConnection(request) internal_data = InternalStorageManager(swift) class_def = internal_data.get_data(INTERNALOCNAME, class_name) if not class_def: raise HttpError("class does not exist", 404) return Response(class_def, mimetype="application/json")
def getDataFromNodeRed(nrDataSource): url = appConfig.nodered_url + nrDataSource['url'] logging.info("getting data from node red at: {}".format(url)) r = requests.get(url) if r.status_code == 404: raise HttpError( "the Node-RED data source is not reachable: {}".format(url), 420) try: data = json.JSONDecoder( object_pairs_hook=collections.OrderedDict).decode( r.content.decode()) dataKeys = getListOfKeys(data[0]) df = pandas.DataFrame(data, columns=dataKeys) df[dataKeys[0]] = df[dataKeys[0]].map(lambda x: str(x)[:20]) return df except: logging.exception("JSON parse error:") raise HttpError("the Node-RED result is no valid JSON", 500)
def doLogin(): try: user = request.json.get("user") tenant = request.json.get("tenant") password = request.json.get("password") log.debug("authenticating tenant: {}, user: {}".format(tenant, user)) token = doAuthGetToken(tenant, user, password) r = Response() r.set_cookie(COOKIE_NAME_TOKEN, value=token) r.set_cookie(COOKIE_NAME_TENANT, value=tenant) r.set_cookie(COOKIE_NAME_USER, value=user) return r except ClientException as e: log.exception("Login error") raise HttpError(e.msg, 401) except Exception: log.exception("Login error") raise HttpError("Internal Server Error", 500)
def delete_objectclass(class_name): swift = createConnection(request) internal_data = InternalStorageManager(swift) class_def = internal_data.get_data(INTERNALOCNAME, class_name) if not class_def: raise HttpError("class does not exist", 404) internal_data.remove_data(INTERNALOCNAME, class_name) return "", 204
def assert_correct_tenant(request, tenant): """ this app needs to be configured for a single customer until this is changed, we must only accept requests for this tenant :param tenant: :return: """ assert_token_tenant_validity(request) t = get_tenant_from_request(request) if (tenant != t): raise HttpError("Tenant is invalid", 401)
def assert_token_tenant_validity(request): """ the credentials are checked against swift. :param tenant: :param token: :return: """ assert_no_xsrf(request) swift_store_url = get_swift_store_url_from_request(request) try: sw = client.Connection( preauthtoken=request.cookies.get(COOKIE_NAME_TOKEN), preauthurl=swift_store_url) h = sw.head_account() if not h: raise HttpError("Token is not valid", 401) except HttpError as e: raise (e) except Exception: raise HttpError("Error checking token", 401)
def create_container(): swift = createConnection(request) internal_data = InternalStorageManager(swift) # TODO: check schema validity since somebody else could store a rouge class definition in the object store (via direct interfacing with the object store) try: container_definition = request.json.get("container") container_name = container_definition.get("name") container_sdos = container_definition.get("sdos", False) except AttributeError: raise HttpError("malformed request", 400) if not container_name: raise HttpError("container name is missing", 400) if "/" in container_name: raise HttpError( "Container name contains '/'. This is only allowed in object names.", 400) containers = swift.get_container_list()[1] if container_name in [container.get("name") for container in containers]: raise HttpError("container already exists", 422) container_metadata = {} if container_sdos: container_metadata["x-container-meta-sdos"] = True try: class_name = container_definition.get("objectClass") class_definition = internal_data.get_data(INTERNALOCNAME, class_name) if class_name: if class_definition is None: raise HttpError("class does not exist", 404) container_metadata = {OBJECTCLASSFIELD: class_name} except AttributeError: pass # ignore empty or missing class definition swift.create_container(container_name, container_metadata) return "", 201
def receive_messages(from_beginning=False): """ we subscribe to our tenant-topic to see all the sent messages. Our client-ID is tenant-bound so that we receive all msgs for that tenant, and so that kafka can maintain a global offset for this tenant our group-id is token bound so that it is unique across all consumers within this tenant; this will make kafka broadcast msgs to all consumers within this tenant --> every logged in session will see all new messages for the tenant and every message will be seen by at least one of the sessions :return: """ log.debug("receiving messages for: {}".format(request.json)) try: msg_tenant = request.json.get("tenant") msg_token = request.json.get("token") if not are_tenant_token_valid(tenant=msg_tenant, token=msg_token): raise HttpError("Credentials are not valid", 500) c = KafkaConsumer(msg_tenant, bootstrap_servers=appConfig.kafka_broker_endpoint, client_id='mcmbb-{}'.format(msg_tenant), group_id='mcmbb-{}-{}'.format( msg_tenant, msg_token[25:]), consumer_timeout_ms=5000, enable_auto_commit=False) if from_beginning: c.poll() c.seek_to_beginning() # next line we actually get the msgs msgs = list(c) if not from_beginning: c.commit() c.close() vals = [json.loads(m.value.decode("utf-8")) for m in msgs] return Response(json.dumps(vals), mimetype="application/json") except Exception: m = "Error retrieving messages" log.exception(m) raise HttpError(m, 500)
def delete_object(container_name, object_name): swift = createConnection(request) metadata = swift.get_object_metadata(container_name, object_name) retentimestamp = metadata.get("x-object-meta-retentiontime") if retentimestamp and not isRetentionPeriodExpired(retentimestamp): error_msg = "Deletion failed due to retention enforcement, file cannot be deleted till {}!".format( time.strftime("%a, %d. %B %Y", time.localtime(int(retentimestamp)))) log.debug(error_msg) raise HttpError(error_msg, 412) swift.delete_object(container_name, object_name) return "", 204
def assert_token_tenant_validity(request, check_xsrf=True): """ the credentials are checked against swift. :param tenant: :param token: :return: """ if check_xsrf: assert_no_xsrf(request) url = get_swift_url_from_request(request) token = get_token_from_request(request) try: sw = client.Connection(preauthtoken=token, preauthurl=url) h = sw.head_account() if not h: raise HttpError("Token is not valid", 401) except HttpError as e: raise (e) except Exception: raise HttpError("Error checking token", 401)
def receive_messages(from_beginning=False): """ we subscribe to our tenant-topic to see all the sent messages. Our client-ID is tenant-bound so that we receive all msgs for that tenant, and so that kafka can maintain a global offset for this tenant our group-id is token bound so that it is unique across all consumers within this tenant; this will make kafka broadcast msgs to all consumers within this tenant --> every logged in session will see all new messages for the tenant and every message will be seen by at least one of the sessions :return: """ from pykafka.exceptions import SocketDisconnectedError try: accountServer.assert_token_tenant_validity(request) msg_tenant = request.cookies.get(accountServer.COOKIE_NAME_TENANT) msg_client_id = request.cookies.get( accountServer.COOKIE_NAME_SESSION_ID) consumer_group = 'mcmbb-{}-{}'.format(msg_tenant, msg_client_id).encode('utf-8') consumer = __get_kafka_consumer(topic=msg_tenant, consumer_group=consumer_group) if from_beginning: partition_offset_pairs = [(p, p.earliest_available_offset()) for p in consumer.partitions.values()] consumer.reset_offsets(partition_offsets=partition_offset_pairs) vals = [__try_parse_msg_content(m) for m in consumer] if not from_beginning: consumer.commit_offsets() return Response(json.dumps(vals), mimetype="application/json") except HttpError as e: """ make sure we don't catch existing HTTP errors here and turn them into meaningless 500s """ raise e except SocketDisconnectedError as e: log.exception( "Connection to Broker closed unexpectedly; returned empty response to client." ) return Response(json.dumps({}), mimetype="application/json") except Exception: m = "Error retrieving messages" log.exception(m) raise HttpError(m, 500)
def getDataFromNodeRed(nrDataSource, container_filter=None): url = configuration.nodered_url + nrDataSource['url'] params = {"container_filter": container_filter} logging.info("getting data from node red at: {} with params: {}".format( url, params)) r = requests.get(url, params=params) if r.status_code == 404: raise HttpError( "the Node-RED data source is not reachable: {}".format(url), 420) try: data = json.JSONDecoder( object_pairs_hook=collections.OrderedDict).decode( r.content.decode()) dataKeys = getListOfKeys(data[0]) df = pandas.DataFrame(data, columns=dataKeys) df[dataKeys[0]] = df[dataKeys[0]].map(lambda x: str(x)[:20]) return df except IndexError as e: logging.warning("no data returned {}".format(e)) raise HttpError("query returned no data", 404) except: logging.exception("JSON parse error:") raise HttpError("the Node-RED result is no valid JSON", 500)
def create_object(container_name): swift = createConnection(request) # returns werkzeug.datastructures.FileStorage i.e. file-like # Underlying stream is either BytesIO for small files or _TemporaryFileWrapper for large files file = request.files["objectName"] object_name = file.filename headers = {} retentionDate = request.form["retentionDate"] if retentionDate: try: dateutil.parser.parse(retentionDate) headers[RETENTIONFIELD] = retentionDate except Exception as e: log.debug( "invalid date format for form parameter retentionDate: {}". format(retentionDate)) raise HttpError( "invalid date format for form parameter retentionDate: {}". format(retentionDate), 400) class_metadata_json = request.form["metadata"] if class_metadata_json: class_metadata = json.loads(class_metadata_json) class_name = swift.get_container_metadata(container_name).get( OBJECTCLASSFIELD) if class_name: internal_data = InternalStorageManager(swift) class_definition = json.loads( internal_data.get_data(INTERNALOCNAME, class_name)) Draft4Validator( class_definition, format_checker=FormatChecker()).validate(class_metadata) for field in class_metadata.keys(): val = class_metadata[field] if val is not None: field_header = xform_header_names(field) xformed_class_name = xform_header_names(class_name) headers["X-Object-Meta-Class-" + xformed_class_name + "-" + field_header] = class_metadata[field] swift.object_upload(object_name, container_name, file, headers, as_stream=False) return "", 201
def receive_messages(from_beginning=False): """ we subscribe to our tenant-topic to see all the sent messages. Our client-ID is tenant-bound so that we receive all msgs for that tenant, and so that kafka can maintain a global offset for this tenant our group-id is token bound so that it is unique across all consumers within this tenant; this will make kafka broadcast msgs to all consumers within this tenant --> every logged in session will see all new messages for the tenant and every message will be seen by at least one of the sessions :return: """ log.debug("receiving messages for: {}".format(request.json)) try: msg_tenant = request.json.get("tenant") msg_client_id = request.json.get("client_id") accountServer.assert_no_xsrf(request) accountServer.assert_token_tenant_validity(request) accountServer.assert_correct_tenant(request, msg_tenant) consumer_group = 'mcmbb-{}-{}'.format(msg_tenant, msg_client_id).encode('utf-8') topic = __get_kafka_topic(msg_tenant) consumer = topic.get_simple_consumer(consumer_group=consumer_group, consumer_id=consumer_group, consumer_timeout_ms=100, auto_commit_enable=False) if from_beginning: partition_offset_pairs = [(p, p.latest_available_offset()) for p in consumer.partitions.values()] consumer.reset_offsets(partition_offsets=partition_offset_pairs) vals = [__try_parse_msg_content(m) for m in consumer] if not from_beginning: consumer.commit_offsets() consumer.stop() return Response(json.dumps(vals), mimetype="application/json") except HttpError as e: raise e except Exception: m = "Error retrieving messages" log.exception(m) raise HttpError(m, 500)
def doPlot(): accountServer.assert_token_tenant_validity(request) nrDataSource = json.loads( urllib.parse.unquote(request.args.get("nrDataSource"))) plotType = request.args.get("plotType") container_filter = request.args.get("container_filter", None) logging.info("producing plot: {} for: {}".format(plotType, nrDataSource)) df = getDataFromNodeRed(nrDataSource=nrDataSource, container_filter=container_filter) try: logging.info(df) if ('bar' == plotType): c = bokeh_plot_bar(data=df, nrDataSource=nrDataSource, container_filter=container_filter) elif ('bar_log' == plotType): c = bokeh_plot_bar(data=df, nrDataSource=nrDataSource, logScale="log", container_filter=container_filter) elif ('line' == plotType): c = bokeh_plot_line(data=df, nrDataSource=nrDataSource, container_filter=container_filter) elif ('line_log' == plotType): c = bokeh_plot_line(data=df, nrDataSource=nrDataSource, logScale="log", container_filter=container_filter) elif ('pie' == plotType): c = bokeh_plot_pie(data=df, nrDataSource=nrDataSource, container_filter=container_filter) else: return Response("Plot type unknown", status=500) return Response(json.dumps(c), mimetype="application/json") except Exception as e: logging.exception("plotting error:") raise HttpError( "the Node-RED result could not be plotted. Maybe wrong data format for the plot type? Check result table: {}" .format(e), 500)
def strip_url_to_tenant_id(url): if not url.startswith(configuration.swift_store_url_valid_prefix): raise HttpError("swift returned wrong storage URL") return url[len(configuration.swift_store_url_valid_prefix):]
def change_object_metadata(container_name, object_name): raise HttpError("funcion not implemented yet")
def change_container_metadata(container_name, metadata): # print(metadata) raise HttpError("funcion not implemented yet")
def get_user_from_request(request): t = request.cookies.get(COOKIE_NAME_USER) if t: return t else: raise HttpError("cookie missing (user)", 401)