def _get_measures_by_name(resources, metric_wildcards, operations, start, stop, granularity, needed_overlap, fill, details): references = [] for r in resources: references.extend([ processor.MetricReference(m, agg, r, wildcard) for wildcard, agg in metric_wildcards for m in r.metrics if fnmatch.fnmatch(m.name, wildcard) ]) if not references: api.abort( 400, { "cause": "Metrics not found", "detail": set((m for (m, a) in metric_wildcards)) }) response = { "measures": get_measures_or_abort(references, operations, start, stop, granularity, needed_overlap, fill) } if details: response["references"] = set((r.resource for r in references)) return response
def _get_measures_by_name(self, resources, metric_names, operations, start, stop, granularity, needed_overlap, fill, details): references = [ processor.MetricReference(r.get_metric(metric_name), agg, r) for (metric_name, agg) in metric_names for r in resources if r.get_metric(metric_name) is not None ] if not references: api.abort( 400, { "cause": "Metrics not found", "detail": set((m for (m, a) in metric_names)) }) response = { "measures": get_measures_or_abort(references, operations, start, stop, granularity, needed_overlap, fill) } if details: response["references"] = references return response
def get_current_user(request): user = request.remote_user if user is None: api.abort(401) try: return user.decode('iso-8859-1') except AttributeError: return user
def get_current_user(request): hdr = request.headers.get("Authorization") auth_hdr = (hdr.decode('utf-8') if isinstance(hdr, bytes) else hdr) auth = werkzeug.http.parse_authorization_header(auth_hdr) if auth is None: api.abort(401) return auth.username
def _write_get_lines(): encoding = pecan.request.headers.get('Transfer-Encoding', "").lower() if encoding == "chunked": if uwsgi is None: api.abort( 501, { "cause": "Not implemented error", "reason": "This server is not running with uwsgi" }) return encoding, uwsgi.chunked_read() return None, pecan.request.body
def OperationsSchema(v): if isinstance(v, six.text_type): try: v = pyparsing.OneOrMore( pyparsing.nestedExpr()).parseString(v).asList()[0] except pyparsing.ParseException as e: api.abort(400, {"cause": "Invalid operations", "reason": "Fail to parse the operations string", "detail": six.text_type(e)}) return voluptuous.Schema(voluptuous.Any(*OperationsSchemaBase), required=True)(v)
def OperationsSchema(v): if isinstance(v, six.text_type): try: v = pyparsing.OneOrMore( pyparsing.nestedExpr()).parseString(v).asList()[0] except pyparsing.ParseException as e: api.abort(400, {"cause": "Invalid operations", "reason": "Fail to parse the operations string", "detail": six.text_type(e)}) return voluptuous.Schema(voluptuous.Any(*OperationsSchemaBase), required=True)(v)
def _write_get_lines(): encoding = pecan.request.headers.get('Transfer-Encoding', "").lower() if encoding == "chunked": # TODO(sileht): Support reading chunk without uwsgi when # pecan.request.environ['wsgi.input_terminated'] is set. # https://github.com/unbit/uwsgi/issues/1428 if uwsgi is None: api.abort( 501, {"cause": "Not implemented error", "reason": "This server is not running with uwsgi"}) return encoding, uwsgi.chunked_read() return None, pecan.request.body
def _write_get_lines(): encoding = pecan.request.headers.get('Transfer-Encoding', "").lower() if encoding == "chunked": # TODO(sileht): Support reading chunk without uwsgi when # pecan.request.environ['wsgi.input_terminated'] is set. # https://github.com/unbit/uwsgi/issues/1428 if uwsgi is None: api.abort( 501, { "cause": "Not implemented error", "reason": "This server is not running with uwsgi" }) return encoding, uwsgi.chunked_read() return None, pecan.request.body
def get_resource_policy_filter(request, rule, resource_type, prefix=None): try: # Check if the policy allows the user to list any resource api.enforce(rule, { "resource_type": resource_type, }) except webob.exc.HTTPForbidden: policy_filter = [] project_id = request.headers.get("X-Project-Id") target = {} if prefix: resource = target[prefix] = {} else: resource = target resource["resource_type"] = resource_type resource["project_id"] = project_id try: # Check if the policy allows the user to list resources linked # to their project api.enforce(rule, target) except webob.exc.HTTPForbidden: pass else: policy_filter.append({"=": {"project_id": project_id}}) del resource["project_id"] resource["created_by_project_id"] = project_id try: # Check if the policy allows the user to list resources linked # to their created_by_project api.enforce(rule, target) except webob.exc.HTTPForbidden: pass else: if project_id: policy_filter.append( {"like": { "creator": "%:" + project_id }}) else: policy_filter.append({"=": {"creator": None}}) if not policy_filter: # We need to have at least one policy filter in place api.abort(403, "Insufficient privileges") return {"or": policy_filter}
def post_query(self, q=None): if q is not None: try: query = query_parser.parseString(q) except pyparsing.ParseException: api.abort(501, {"cause": "Not implemented error", "detail": "q", "reason": "Query not implemented"}) resource_type = query[0] api.enforce("create resource type", {"name": resource_type}) schema = pecan.request.indexer.get_resource_type_schema() rt = schema.resource_type_from_dict(resource_type, {}, 'creating') try: pecan.request.indexer.create_resource_type(rt) except indexer.ResourceTypeAlreadyExists: pass pecan.response.status = 204
def get_resource_policy_filter(request, rule, resource_type, prefix=None): try: # Check if the policy allows the user to list any resource api.enforce(rule, { "resource_type": resource_type, }) except webob.exc.HTTPForbidden: policy_filter = [] project_id = request.headers.get("X-Project-Id") target = {} if prefix: resource = target[prefix] = {} else: resource = target resource["resource_type"] = resource_type resource["project_id"] = project_id try: # Check if the policy allows the user to list resources linked # to their project api.enforce(rule, target) except webob.exc.HTTPForbidden: pass else: policy_filter.append({"=": {"project_id": project_id}}) del resource["project_id"] resource["created_by_project_id"] = project_id try: # Check if the policy allows the user to list resources linked # to their created_by_project api.enforce(rule, target) except webob.exc.HTTPForbidden: pass else: if project_id: policy_filter.append( {"like": {"creator": "%:" + project_id}}) else: policy_filter.append({"=": {"creator": None}}) if not policy_filter: # We need to have at least one policy filter in place api.abort(403, "Insufficient privileges") return {"or": policy_filter}
def post_query(self, q=None): if q is not None: try: query = query_parser.parseString(q) except pyparsing.ParseException: api.abort( 501, { "cause": "Not implemented error", "detail": "q", "reason": "Query not implemented" }) resource_type = query[0] api.enforce("create resource type", {"name": resource_type}) schema = pecan.request.indexer.get_resource_type_schema() rt = schema.resource_type_from_dict(resource_type, {}, 'creating') try: pecan.request.indexer.create_resource_type(rt) except indexer.ResourceTypeAlreadyExists: pass pecan.response.status = 204
def get_measures_or_abort(metrics_and_aggregations, operations, start, stop, granularity, needed_overlap, fill, ref_identifier): try: return processor.get_measures(pecan.request.storage, metrics_and_aggregations, operations, start, stop, granularity, needed_overlap, fill, ref_identifier=ref_identifier) except exceptions.UnAggregableTimeseries as e: api.abort(400, e) # TODO(sileht): We currently got only one metric for these exceptions but # we can improve processor to returns all missing metrics at once, so we # returns a list for the future except storage.MetricDoesNotExist as e: api.abort(404, { "cause": "Unknown metrics", "detail": [str(e.metric.id)] }) except storage.AggregationDoesNotExist as e: api.abort( 404, { "cause": "Metrics with unknown aggregation", "detail": [(str(e.metric.id), e.method)] })
def _get_measures_by_name(self, resources, metric_names, operations, start, stop, granularity, needed_overlap, fill): metrics_and_aggregations = list( filter(lambda x: x[0] is not None, ([r.get_metric(metric_name), agg] for (metric_name, agg) in metric_names for r in resources))) if not metrics_and_aggregations: api.abort( 400, { "cause": "Metrics not found", "detail": set((m for (m, a) in metric_names)) }) return get_measures_or_abort(metrics_and_aggregations, operations, start, stop, granularity, needed_overlap, fill, ref_identifier="name")
def get_metric_policy_filter(request, rule): try: # Check if the policy allows the user to list any metric api.enforce(rule, {}) except webob.exc.HTTPForbidden: policy_filter = [] project_id = request.headers.get("X-Project-Id") try: # Check if the policy allows the user to list metrics linked # to their created_by_project api.enforce(rule, { "created_by_project_id": project_id, }) except webob.exc.HTTPForbidden: pass else: policy_filter.append({"like": {"creator": "%:" + project_id}}) if not policy_filter: # We need to have at least one policy filter in place api.abort(403, "Insufficient privileges") return {"or": policy_filter}
def get_metric_policy_filter(request, rule): try: # Check if the policy allows the user to list any metric api.enforce(rule, {}) except webob.exc.HTTPForbidden: policy_filter = [] project_id = request.headers.get("X-Project-Id") try: # Check if the policy allows the user to list metrics linked # to their created_by_project api.enforce(rule, { "created_by_project_id": project_id, }) except webob.exc.HTTPForbidden: pass else: policy_filter.append( {"like": {"creator": "%:" + project_id}}) if not policy_filter: # We need to have at least one policy filter in place api.abort(403, "Insufficient privileges") return {"or": policy_filter}
def _get_measures_by_name(resources, metric_wildcards, operations, start, stop, granularity, needed_overlap, fill, details): references = [] for r in resources: references.extend([ processor.MetricReference(m, agg, r, wildcard) for wildcard, agg in metric_wildcards for m in r.metrics if fnmatch.fnmatch(m.name, wildcard) ]) if not references: api.abort(400, {"cause": "Metrics not found", "detail": set((m for (m, a) in metric_wildcards))}) response = { "measures": get_measures_or_abort( references, operations, start, stop, granularity, needed_overlap, fill) } if details: response["references"] = set((r.resource for r in references)) return response
def get_measures_or_abort(references, operations, start, stop, granularity, needed_overlap, fill): try: return processor.get_measures( pecan.request.storage, references, operations, start, stop, granularity, needed_overlap, fill) except exceptions.UnAggregableTimeseries as e: api.abort(400, e) # TODO(sileht): We currently got only one metric for these exceptions but # we can improve processor to returns all missing metrics at once, so we # returns a list for the future except storage.MetricDoesNotExist as e: api.abort(404, {"cause": "Unknown metrics", "detail": [str(e.metric.id)]}) except storage.AggregationDoesNotExist as e: api.abort(404, {"cause": "Metrics with unknown aggregation", "detail": [(str(e.metric.id), e.method)]})
def ResourceTypeSchema(resource_type): try: pecan.request.indexer.get_resource_type(resource_type) except indexer.NoSuchResourceType as e: api.abort(400, e) return resource_type
def post(self, start=None, stop=None, granularity=None, needed_overlap=None, fill=None, groupby=None, **kwargs): details = api.get_bool_param('details', kwargs) if fill is None and needed_overlap is None: fill = "dropna" start, stop, granularity, needed_overlap, fill = api.validate_qs( start, stop, granularity, needed_overlap, fill) body = api.deserialize_and_validate(self.FetchSchema) references = extract_references(body["operations"]) if not references: api.abort( 400, { "cause": "Operations is invalid", "reason": "At least one 'metric' is required", "detail": body["operations"] }) if "resource_type" in body: attr_filter = body["search"] policy_filter = ( pecan.request.auth_helper.get_resource_policy_filter( pecan.request, "search resource", body["resource_type"])) if policy_filter: if attr_filter: attr_filter = {"and": [policy_filter, attr_filter]} else: attr_filter = policy_filter groupby = sorted(set(api.arg_to_list(groupby))) sorts = groupby if groupby else api.RESOURCE_DEFAULT_PAGINATION try: resources = pecan.request.indexer.list_resources( body["resource_type"], attribute_filter=attr_filter, sorts=sorts) except indexer.IndexerException as e: api.abort(400, six.text_type(e)) if not groupby: return self._get_measures_by_name(resources, references, body["operations"], start, stop, granularity, needed_overlap, fill, details=details) def groupper(r): return tuple((attr, r[attr]) for attr in groupby) results = [] for key, resources in itertools.groupby(resources, groupper): results.append({ "group": dict(key), "measures": self._get_measures_by_name(resources, references, body["operations"], start, stop, granularity, needed_overlap, fill, details=details) }) return results else: try: metric_ids = set( six.text_type(utils.UUID(m)) for (m, a) in references) except ValueError as e: api.abort( 400, { "cause": "Invalid metric references", "reason": six.text_type(e), "detail": references }) metrics = pecan.request.indexer.list_metrics( attribute_filter={"in": { "id": metric_ids }}) missing_metric_ids = (set(metric_ids) - set(six.text_type(m.id) for m in metrics)) if missing_metric_ids: api.abort( 404, { "cause": "Unknown metrics", "reason": "Provided metrics don't exists", "detail": missing_metric_ids }) number_of_metrics = len(metrics) if number_of_metrics == 0: return [] for metric in metrics: api.enforce("get metric", metric) metrics_by_ids = dict((six.text_type(m.id), m) for m in metrics) references = [ processor.MetricReference(metrics_by_ids[m], a) for (m, a) in references ] response = { "measures": get_measures_or_abort(references, body["operations"], start, stop, granularity, needed_overlap, fill) } if details: response["references"] = metrics return response
def ResourceTypeSchema(resource_type): try: pecan.request.indexer.get_resource_type(resource_type) except indexer.NoSuchResourceType as e: api.abort(400, e) return resource_type
def get_current_user(request): user = request.remote_user if user is None: api.abort(401) return user.decode('iso-8859-1')
def get_current_user(request): auth = werkzeug.http.parse_authorization_header( request.headers.get("Authorization")) if auth is None: api.abort(401) return auth.username
def post_write(self, db="influxdb"): creator = pecan.request.auth_helper.get_current_user(pecan.request) tag_to_rid = pecan.request.headers.get( "X-Gnocchi-InfluxDB-Tag-Resource-ID", self.DEFAULT_TAG_RESOURCE_ID) while True: encoding, chunk = self._write_get_lines() # If chunk is empty then this is over. if not chunk: break # Compute now on a per-chunk basis now = numpy.datetime64(int(time.time() * 10e8), 'ns') # resources = { resource_id: { # metric_name: [ incoming.Measure(t, v), …], … # }, … # } resources = collections.defaultdict( lambda: collections.defaultdict(list)) for line_number, line in enumerate(chunk.split(b"\n")): # Ignore empty lines if not line: continue try: measurement, tags, fields, timestamp = ( line_protocol.parseString(line.decode()) ) except (UnicodeDecodeError, SyntaxError, pyparsing.ParseException): api.abort(400, { "cause": "Value error", "detail": "line", "reason": "Unable to parse line %d" % ( line_number + 1), }) if timestamp is None: timestamp = now try: resource_id = tags.pop(tag_to_rid) except KeyError: api.abort(400, { "cause": "Value error", "detail": "key", "reason": "Unable to find key `%s' in tags" % ( tag_to_rid), }) tags_str = (("@" if tags else "") + ",".join(("%s=%s" % (k, tags[k])) for k in sorted(tags))) for field_name, field_value in six.iteritems(fields): if isinstance(field_value, str): # We do not support field value that are not numerical continue # Metric name is the: # <measurement>.<field_key>@<tag_key>=<tag_value>,… # with tag ordered # Replace "/" with "_" because Gnocchi does not support / # in metric names metric_name = ( measurement + "." + field_name + tags_str ).replace("/", "_") resources[resource_id][metric_name].append( incoming.Measure(timestamp, field_value)) measures_to_batch = {} for resource_name, metrics_and_measures in six.iteritems( resources): resource_name = resource_name resource_id = utils.ResourceUUID( resource_name, creator=creator) LOG.debug("Getting metrics from resource `%s'", resource_name) timeout = pecan.request.conf.api.operation_timeout metrics = ( api.get_or_create_resource_and_metrics.retry_with( stop=tenacity.stop_after_delay(timeout))( creator, resource_id, resource_name, metrics_and_measures.keys(), {}, db) ) for metric in metrics: api.enforce("post measures", metric) measures_to_batch.update( dict((metric.id, metrics_and_measures[metric.name]) for metric in metrics if metric.name in metrics_and_measures)) LOG.debug("Add measures batch for %d metrics", len(measures_to_batch)) pecan.request.incoming.add_measures_batch(measures_to_batch) pecan.response.status = 204 if encoding != "chunked": return
def get_current_user(request): auth = werkzeug.http.parse_authorization_header( request.headers.get("Authorization")) if auth is None: api.abort(401) return auth.username
def get_current_user(request): user = request.remote_user if user is None: api.abort(401) return user.decode('iso-8859-1')
def post_write(self, db="influxdb"): creator = pecan.request.auth_helper.get_current_user(pecan.request) tag_to_rid = pecan.request.headers.get( "X-Gnocchi-InfluxDB-Tag-Resource-ID", self.DEFAULT_TAG_RESOURCE_ID) while True: encoding, chunk = self._write_get_lines() # If chunk is empty then this is over. if not chunk: break # Compute now on a per-chunk basis now = numpy.datetime64(int(time.time() * 10e8), 'ns') # resources = { resource_id: { # metric_name: [ incoming.Measure(t, v), …], … # }, … # } resources = collections.defaultdict( lambda: collections.defaultdict(list)) for line_number, line in enumerate(chunk.split(b"\n")): # Ignore empty lines if not line: continue try: measurement, tags, fields, timestamp = ( line_protocol.parseString(line.decode())) except (UnicodeDecodeError, SyntaxError, pyparsing.ParseException): api.abort( 400, { "cause": "Value error", "detail": "line", "reason": "Unable to parse line %d" % (line_number + 1), }) if timestamp is None: timestamp = now try: resource_id = tags.pop(tag_to_rid) except KeyError: api.abort( 400, { "cause": "Value error", "detail": "key", "reason": "Unable to find key `%s' in tags" % (tag_to_rid), }) tags_str = (("@" if tags else "") + ",".join( ("%s=%s" % (k, tags[k])) for k in sorted(tags))) for field_name, field_value in six.iteritems(fields): if isinstance(field_value, str): # We do not support field value that are not numerical continue # Metric name is the: # <measurement>.<field_key>@<tag_key>=<tag_value>,… # with tag ordered # Replace "/" with "_" because Gnocchi does not support / # in metric names metric_name = (measurement + "." + field_name + tags_str).replace("/", "_") resources[resource_id][metric_name].append( incoming.Measure(timestamp, field_value)) measures_to_batch = {} for resource_name, metrics_and_measures in six.iteritems( resources): resource_name = resource_name resource_id = utils.ResourceUUID(resource_name, creator=creator) LOG.debug("Getting metrics from resource `%s'", resource_name) timeout = pecan.request.conf.api.operation_timeout metrics = (api.get_or_create_resource_and_metrics.retry_with( stop=tenacity.stop_after_delay(timeout))( creator, resource_id, resource_name, metrics_and_measures.keys(), {}, db)) for metric in metrics: api.enforce("post measures", metric) measures_to_batch.update( dict((metric.id, metrics_and_measures[metric.name]) for metric in metrics if metric.name in metrics_and_measures)) LOG.debug("Add measures batch for %d metrics", len(measures_to_batch)) pecan.request.incoming.add_measures_batch(measures_to_batch) pecan.response.status = 204 if encoding != "chunked": return
def post(self, start=None, stop=None, granularity=None, needed_overlap=None, fill=None, groupby=None, **kwargs): details = api.get_bool_param('details', kwargs) if fill is None and needed_overlap is None: fill = "dropna" start, stop, granularity, needed_overlap, fill = api.validate_qs( start, stop, granularity, needed_overlap, fill) body = api.deserialize_and_validate(self.FetchSchema) references = extract_references(body["operations"]) if not references: api.abort(400, {"cause": "Operations is invalid", "reason": "At least one 'metric' is required", "detail": body["operations"]}) if "resource_type" in body: attr_filter = body["search"] policy_filter = ( pecan.request.auth_helper.get_resource_policy_filter( pecan.request, "search resource", body["resource_type"])) if policy_filter: if attr_filter: attr_filter = {"and": [ policy_filter, attr_filter ]} else: attr_filter = policy_filter groupby = sorted(set(api.arg_to_list(groupby))) sorts = groupby if groupby else api.RESOURCE_DEFAULT_PAGINATION try: resources = pecan.request.indexer.list_resources( body["resource_type"], attribute_filter=attr_filter, sorts=sorts) except indexer.IndexerException as e: api.abort(400, six.text_type(e)) if not groupby: return self._get_measures_by_name( resources, references, body["operations"], start, stop, granularity, needed_overlap, fill, details=details) def groupper(r): return tuple((attr, r[attr]) for attr in groupby) results = [] for key, resources in itertools.groupby(resources, groupper): results.append({ "group": dict(key), "measures": self._get_measures_by_name( resources, references, body["operations"], start, stop, granularity, needed_overlap, fill, details=details) }) return results else: try: metric_ids = set(six.text_type(utils.UUID(m)) for (m, a) in references) except ValueError as e: api.abort(400, {"cause": "Invalid metric references", "reason": six.text_type(e), "detail": references}) metrics = pecan.request.indexer.list_metrics( attribute_filter={"in": {"id": metric_ids}}) missing_metric_ids = (set(metric_ids) - set(six.text_type(m.id) for m in metrics)) if missing_metric_ids: api.abort(404, {"cause": "Unknown metrics", "reason": "Provided metrics don't exists", "detail": missing_metric_ids}) number_of_metrics = len(metrics) if number_of_metrics == 0: return [] for metric in metrics: api.enforce("get metric", metric) metrics_by_ids = dict((six.text_type(m.id), m) for m in metrics) references = [processor.MetricReference(metrics_by_ids[m], a) for (m, a) in references] response = { "measures": get_measures_or_abort( references, body["operations"], start, stop, granularity, needed_overlap, fill) } if details: response["references"] = metrics return response