def testPrometheusIntegration(self): registry = prometheus_client.CollectorRegistry(auto_describe=True) def MakeCollector(metadatas): return prometheus_stats_collector.PrometheusStatsCollector( metadatas, registry) with self.SetUpStatsCollector(MakeCollector): counter = metrics.Counter("foobars") counter.Increment(42) port = portpicker.pick_unused_port() with mock.patch.object(stats_server.StatsServerHandler, "registry", registry): server = stats_server.StatsServer(port) server.Start() self.addCleanup(server.Stop) res = requests.get("http://localhost:{}/metrics".format(port)) text_fd = io.StringIO(res.text) families = prometheus_parser.text_fd_to_metric_families(text_fd) families = {family.name: family for family in families} self.assertIn("foobars", families) self.assertEqual(families["foobars"].samples[0].value, 42)
def collect(self): try: with open(self.path, "r") as f: for metric in text_fd_to_metric_families(f): # Ignore non-baseline metrics if not metric.name.endswith("_baseline"): continue yield metric except FileNotFoundError: return []
def get_services_ipvs(self): """Return the set of ip:port services known to IPVS.""" req = self.get_url(self.args.prometheus_url) services = set() for metric in text_fd_to_metric_families(req.iter_lines()): if metric.name == "node_ipvs_backend_weight": for sample in metric.samples: address = sample[1]['local_address'] port = sample[1]['local_port'] services.add('{}:{}'.format(address, port)) return services
def get_remote_hosts_ipvs(self): """Return the set of hostnames known to IPVS by querying prometheus-node-exporter. As prometheus exposes the IP addresses, use gethostbyaddr to get the hostnames.""" req = self.get_url(self.args.prometheus_url) hosts = set() for metric in text_fd_to_metric_families(req.iter_lines()): if metric.name == "node_ipvs_backend_weight": for sample in metric.samples: address = sample[1]['remote_address'] hostname = socket.gethostbyaddr(address)[0] hosts.add(hostname) return hosts
def parse_metric_family(self, response, scraper_config): """ Parse the MetricFamily from a valid requests.Response object to provide a MetricFamily object (see [0]) The text format uses iter_lines() generator. :param response: requests.Response :return: core.Metric """ input_gen = response.iter_lines(chunk_size=self.REQUESTS_CHUNK_SIZE, decode_unicode=True) if scraper_config['_text_filter_blacklist']: input_gen = self._text_filter_input(input_gen, scraper_config) for metric in text_fd_to_metric_families(input_gen): metric.type = scraper_config['type_overrides'].get(metric.name, metric.type) if metric.type not in self.METRIC_TYPES: continue metric.name = self._remove_metric_prefix(metric.name, scraper_config) yield metric
def testPrometheusIntegration(self): registry = prometheus_client.CollectorRegistry(auto_describe=True) metadatas = [stats_utils.CreateCounterMetadata("foobars")] collector = prometheus_stats_collector.PrometheusStatsCollector( metadatas, registry=registry) collector.IncrementCounter("foobars", 42) handler = stats_server.StatsServerHandler( mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) handler.registry = registry handler.path = "/metrics" handler.headers = {} handler.wfile = io.BytesIO() handler.do_GET() handler.wfile.seek(0) families = prometheus_parser.text_fd_to_metric_families(handler.wfile) families = {family.name: family for family in families} self.assertIn("foobars", families) self.assertEqual(families["foobars"].samples[0].value, 42)
def testPrometheusIntegration(self): registry = prometheus_client.CollectorRegistry(auto_describe=True) metadatas = [stats_utils.CreateCounterMetadata("foobars")] collector = prometheus_stats_collector.PrometheusStatsCollector( metadatas, registry=registry) collector.IncrementCounter("foobars", 42) handler = stats_server.StatsServerHandler(mock.MagicMock(), mock.MagicMock(), mock.MagicMock()) handler.registry = registry handler.path = "/metrics" handler.headers = {} handler.wfile = io.BytesIO() handler.do_GET() handler.wfile.seek(0) families = prometheus_parser.text_fd_to_metric_families(handler.wfile) families = {family.name: family for family in families} self.assertIn("foobars", families) self.assertEqual(families["foobars"].samples[0].value, 42)
'nimbus', 'lodestar', ] class Entry(TypedDict): doc: Dict[str, str] type: Dict[str, str] labels: Dict[str, Dict[str, List[str]]] entries: Dict[str, Entry] = {} for client in clients: with open(f'client_data/{client}.txt', 'rt') as f: for family in text_fd_to_metric_families(f): if family.name not in entries: entries[family.name] = Entry( doc={}, type={}, labels={}, ) entry = entries[family.name] entry['doc'][client] = family.documentation entry['type'][client] = family.type entry_labels = entry['labels'] if client not in entry_labels: entry_labels[client] = {} sample: Sample for sample in family.samples: for k, v in sample.labels.items():
def parse_metric_family(self, response): """ Parse the MetricFamily from a valid requests.Response object to provide a MetricFamily object (see [0]) The text format uses iter_lines() generator. The protobuf format directly parse the response.content property searching for Prometheus messages of type MetricFamily [0] delimited by a varint32 [1] when the content-type is a `application/vnd.google.protobuf`. [0] https://github.com/prometheus/client_model/blob/086fe7ca28bde6cec2acd5223423c1475a362858/metrics.proto#L76-%20%20L81 # noqa: E501 [1] https://developers.google.com/protocol-buffers/docs/reference/java/com/google/protobuf/AbstractMessageLite#writeDelimitedTo(java.io.OutputStream) # noqa: E501 :param response: requests.Response :return: metrics_pb2.MetricFamily() """ if 'application/vnd.google.protobuf' in response.headers[ 'Content-Type']: n = 0 buf = response.content while n < len(buf): msg_len, new_pos = _DecodeVarint32(buf, n) n = new_pos msg_buf = buf[n:n + msg_len] n += msg_len message = metrics_pb2.MetricFamily() message.ParseFromString(msg_buf) message.name = self.remove_metric_prefix(message.name) # Lookup type overrides: if self.type_overrides and message.name in self.type_overrides: new_type = self.type_overrides[message.name] if new_type in self.METRIC_TYPES: message.type = self.METRIC_TYPES.index(new_type) else: self.log.debug( "type override %s for %s is not a valid type name" % (new_type, message.name)) yield message elif 'text/plain' in response.headers['Content-Type']: input_gen = response.iter_lines( chunk_size=self.REQUESTS_CHUNK_SIZE, decode_unicode=True) if self._text_filter_blacklist: input_gen = self._text_filter_input(input_gen) messages = defaultdict( list) # map with the name of the element (before the labels) # and the list of occurrences with labels and values obj_map = {} # map of the types of each metrics obj_help = {} # help for the metrics for metric in text_fd_to_metric_families(input_gen): metric.name = self.remove_metric_prefix(metric.name) metric_name = "%s_bucket" % metric.name if metric.type == "histogram" else metric.name metric_type = self.type_overrides.get(metric_name, metric.type) if metric_type == "untyped" or metric_type not in self.METRIC_TYPES: continue for sample in metric.samples: if (sample[0].endswith("_sum") or sample[0].endswith("_count")) and metric_type in [ "histogram", "summary", ]: messages[sample[0]].append({ "labels": sample[1], 'value': sample[2] }) else: messages[metric_name].append({ "labels": sample[1], 'value': sample[2] }) obj_map[metric.name] = metric_type obj_help[metric.name] = metric.documentation for _m in obj_map: if _m in messages or (obj_map[_m] == 'histogram' and ('{}_bucket'.format(_m) in messages)): yield self._extract_metric_from_map( _m, messages, obj_map, obj_help) else: raise UnknownFormatError( 'Unsupported content-type provided: {}'.format( response.headers['Content-Type']))
def GET(self, date): validateAccessToken() web_logger.mes() file_path = "" # Allow origin web.header('Access-Control-Allow-Origin', '*') web.header('Access-Control-Allow-Credentials', 'true') # checks if any date has been specified, otherwise looks for the most recent statistics if (date != "last-month"): if self.__dates_regex.match(date): search = self.__dates_regex.search(date) month_from = search.group(2) year_from = search.group(1) month_to = search.group(4) year_to = search.group(3) if year_from > year_to or (year_from == year_to and month_from > month_to): raise web.HTTPError( "400", {"Content-Type": "text/plain"}, "Bad date provided, the ending date is lower than the beginning date." ) registry = CollectorRegistry() # Counter of accesses to different endpoints oc http_requests = Counter( 'opencitations_http_requests', 'Counter for HTTP requests to opencitations endpoints', ['endpoint'], registry=registry) # Aggregate counter of accesses to the different categories of endpoints oc agg_counter = Counter( 'opencitations_agg_counter', 'Aggregate HTTP requests counter to opencitations endpoints', ['category'], registry=registry) i = Info('opencitations_date', 'Date to which the statistics refers to', registry=registry) i.info({ 'month_from': str(month_from), 'year_from': str(year_from), "month_to": str(month_to), 'year_to': str(year_to) }) indexed_records = Gauge('opencitations_indexed_records', 'Indexed records', registry=registry) harvested_data_sources = Gauge( 'opencitations_harvested_data_sources', 'Harvested data sources', registry=registry) current_month = int(month_from) current_year = int(year_from) target_month = int(month_to) target_year = int(year_to) while (True): # For each month collects the statistics and adds # them to the ones to be returned. while (True): current_month_str = str(current_month) if len(current_month_str) == 1: current_month_str = '0' + current_month_str file_path = path.join( c["stats_dir"], "oc-" + str(current_year) + "-" + current_month_str + ".prom") if path.isfile(file_path): f = open(file_path, 'r') families = text_fd_to_metric_families(f) for family in families: for sample in family.samples: if sample[ 0] == "opencitations_agg_counter_total": agg_counter.labels(**sample[1]).inc( sample[2]) if sample[ 0] == "opencitations_http_requests_total": http_requests.labels(**sample[1]).inc( sample[2]) if sample[ 0] == "opencitations_indexed_records": indexed_records.set(sample[2]) if sample[ 0] == "opencitations_harvested_data_sources": harvested_data_sources.set(sample[2]) # If we reaches the target year and the month we are visiting is the last one # or if we visited the whole year i.e. the last month has just been visited # exit the months's loop if (current_year == target_year and current_month >= target_month) or current_month == 12: break current_month += 1 # If we visited all the years than we exit the years's loop if (current_year == target_year): break current_year += 1 current_month = 1 return generate_latest(registry) else: file_name = "oc-" + date + ".prom" if self.__file_regex.match(file_name): file_path = path.join(c["stats_dir"], file_name) if not path.isfile(file_path): file_path = '' else: raise web.HTTPError( "400", {"Content-Type": "text/plain"}, "Bad date format the required one is: year-month or year-month_year-month." ) else: max_year = 0 max_month = 0 for file in listdir(c["stats_dir"]): if self.__file_regex.match(file): groups = self.__file_regex.search(file).groups() # checks that the file respects the format in the name year = int(groups[0]) month = int(groups[1]) if year > max_year or (year == max_year and month > max_month): max_year = year max_month = month file_path = path.join(c["stats_dir"], file) # if the statistics file was found then it returns the content if file_path != "": web.header('Content-Type', "document") f = open(file_path, 'r') content = f.read() f.close() web.ctx.status = '200 OK' return content else: raise web.HTTPError("404", {"Content-Type": "text/plain"}, "No statistics found.")
def parse_metric_family(self, response): """ Parse the MetricFamily from a valid requests.Response object to provide a MetricFamily object (see [0]) The text format uses iter_lines() generator. The protobuf format directly parse the response.content property searching for Prometheus messages of type MetricFamily [0] delimited by a varint32 [1] when the content-type is a `application/vnd.google.protobuf`. [0] https://github.com/prometheus/client_model/blob/086fe7ca28bde6cec2acd5223423c1475a362858/metrics.proto#L76-%20%20L81 [1] https://developers.google.com/protocol-buffers/docs/reference/java/com/google/protobuf/AbstractMessageLite#writeDelimitedTo(java.io.OutputStream) :param response: requests.Response :return: metrics_pb2.MetricFamily() """ if 'application/vnd.google.protobuf' in response.headers['Content-Type']: n = 0 buf = response.content while n < len(buf): msg_len, new_pos = _DecodeVarint32(buf, n) n = new_pos msg_buf = buf[n:n+msg_len] n += msg_len message = metrics_pb2.MetricFamily() message.ParseFromString(msg_buf) # Lookup type overrides: if self.type_overrides and message.name in self.type_overrides: new_type = self.type_overrides[message.name] if new_type in self.METRIC_TYPES: message.type = self.METRIC_TYPES.index(new_type) else: self.log.debug("type override %s for %s is not a valid type name" % (new_type, message.name)) yield message elif 'text/plain' in response.headers['Content-Type']: messages = defaultdict(list) # map with the name of the element (before the labels) # and the list of occurrences with labels and values obj_map = {} # map of the types of each metrics obj_help = {} # help for the metrics for metric in text_fd_to_metric_families(response.iter_lines(chunk_size=self.REQUESTS_CHUNK_SIZE)): metric_name = "%s_bucket" % metric.name if metric.type == "histogram" else metric.name metric_type = self.type_overrides.get(metric_name, metric.type) if metric_type == "untyped" or metric_type not in self.METRIC_TYPES: continue for sample in metric.samples: if (sample[0].endswith("_sum") or sample[0].endswith("_count")) and \ metric_type in ["histogram", "summary"]: messages[sample[0]].append({"labels": sample[1], 'value': sample[2]}) else: messages[metric_name].append({"labels": sample[1], 'value': sample[2]}) obj_map[metric.name] = metric_type obj_help[metric.name] = metric.documentation for _m in obj_map: if _m in messages or (obj_map[_m] == 'histogram' and ('{}_bucket'.format(_m) in messages)): yield self._extract_metric_from_map(_m, messages, obj_map, obj_help) else: raise UnknownFormatError('Unsupported content-type provided: {}'.format( response.headers['Content-Type']))
def get_data(args): """Fetch data """ exit_code = None value = '' warning = '' critical = '' min_value = '' max_value = '' unit = '' found = False url = "http://%s:%s/metrics" % (args.hostname, args.port) if args.collector is not None: metric_name = '%s%s' % (args.metric, args.collector) else: metric_name = args.metric # metric.sample == [(metric_name, {u'host': u'example', u'unit': u'example_unit'}, value)] try: for metric in text_fd_to_metric_families(urllib.urlopen(url)): if metric.name == metric_name: value = float(metric.samples[0][2]) unit = metric.samples[0][1].get('unit', '') found = True elif metric.name == '%s_warning' % metric_name: warning = float(metric.samples[0][2]) elif metric.name == '%s_critical' % metric_name: critical = float(metric.samples[0][2]) elif metric.name == '%s_min' % metric_name: min_value = float(metric.samples[0][2]) elif metric.name == '%s_max' % metric_name: max_value = float(metric.samples[0][2]) elif args.collector is not None and metric.name == 'nagios_state%s_state' % args.collector: exit_code = int(metric.samples[0][2]) except IOError as err: print "UNKNOWN: Error while fetching metric %s: %s" % (metric_name, err) sys.exit(3) except ValueError as err: print "UNKNOWN: Error while fetching metric %s: %s" % (metric_name, err) sys.exit(3) if not found: print "UNKNOWN: Metric %s not found" % metric_name sys.exit(3) if args.warning is not None: warning = args.warning if args.critical is not None: critical = args.critical perf = '%s=%s%s;%s;%s;%s;%s' % (metric_name, value, unit, warning, critical, min_value, max_value) # Set output if exit_code is None: if critical != '' and value >= critical: exit_code = 2 elif warning != '' and value >= warning: exit_code = 1 else: exit_code = 0 # Assume OK as we managed to fetch metric state = STATES[exit_code] output = "Metric %s fetched successfully" % metric_name output = "%s: %s | %s" % (state, output, perf) print output sys.exit(exit_code)