def test_create_batched_time_series_with_many(self, monitor_resource_mock): client = mock.Mock() # First view with 3 view_name1 = "view-name1" view1 = view_module.View(view_name1, "test description", ['test'], VIDEO_SIZE_MEASURE, aggregation_module.LastValueAggregation()) v_data1 = view_data_module.ViewData(view=view1, start_time=TEST_TIME_STR, end_time=TEST_TIME_STR) v_data1.record(context=tag_map_module.TagMap({'test': '1'}), value=7, timestamp=None) v_data1.record(context=tag_map_module.TagMap({'test': '2'}), value=5, timestamp=None) v_data1.record(context=tag_map_module.TagMap({'test': '3'}), value=3, timestamp=None) # Second view with 2 view_name2 = "view-name2" view2 = view_module.View(view_name2, "test description", ['test'], VIDEO_SIZE_MEASURE, aggregation_module.LastValueAggregation()) v_data2 = view_data_module.ViewData(view=view2, start_time=TEST_TIME_STR, end_time=TEST_TIME_STR) v_data2.record(context=tag_map_module.TagMap({'test': '1'}), value=7, timestamp=None) v_data2.record(context=tag_map_module.TagMap({'test': '2'}), value=5, timestamp=None) view_data = [v_data1, v_data2] view_data = [ metric_utils.view_data_to_metric(vd, TEST_TIME) for vd in view_data ] option = stackdriver.Options(project_id="project-test") exporter = stackdriver.StackdriverStatsExporter(options=option, client=client) time_series_batches = exporter.create_batched_time_series(view_data, 2) self.assertEqual(len(time_series_batches), 3) [tsb1, tsb2, tsb3] = time_series_batches self.assertEqual(len(tsb1), 2) self.assertEqual(len(tsb2), 2) self.assertEqual(len(tsb3), 1)
def __init__(self, test_name, test_description=None): # oc will automatically search for the ENV VAR 'APPLICATIONINSIGHTS_CONNECTION_STRING' self.exporter = metrics_exporter.new_metrics_exporter() self.stats = stats_module.stats self.view_manager = self.stats.view_manager self.stats_recorder = self.stats.stats_recorder self.azure_logger = get_azure_logger(test_name) self.name = test_name self.desc = test_description events_measure_name = "The number of events handled by " + self.name events_measure_desc = "The number of events handled by " + self.desc if self.desc else None memory_measure_name = "memory usage percentage for " + self.name memory_measure_desc = "memory usage percentage for " + self.desc if self.desc else None cpu_measure_name = "cpu usage percentage for " + self.name cpu_measure_desc = "cpu usage percentage for " + self.desc if self.desc else None error_measure_name = "error count for " + self.name error_measure_desc = "The number of errors happened while running the test for " + self.desc if self.desc else None self.events_measure = measure_module.MeasureInt( events_measure_name, events_measure_desc, "events") self.memory_measure = measure_module.MeasureFloat( memory_measure_name, memory_measure_desc) self.cpu_measure = measure_module.MeasureFloat(cpu_measure_name, cpu_measure_desc) self.error_measure = measure_module.MeasureInt(error_measure_name, error_measure_desc) self.events_measure_view = view_module.View( events_measure_name, events_measure_desc, [], self.events_measure, aggregation_module.SumAggregation()) self.memory_measure_view = view_module.View( memory_measure_name, memory_measure_desc, [], self.memory_measure, aggregation_module.LastValueAggregation()) self.cpu_measure_view = view_module.View( cpu_measure_name, cpu_measure_desc, [], self.cpu_measure, aggregation_module.LastValueAggregation()) self.error_measure_view = view_module.View( error_measure_name, error_measure_desc, [], self.error_measure, aggregation_module.CountAggregation()) self.view_manager.register_view(self.events_measure_view) self.view_manager.register_view(self.memory_measure_view) self.view_manager.register_view(self.cpu_measure_view) self.view_manager.register_view(self.error_measure_view) self.mmap = self.stats_recorder.new_measurement_map()
def setup_open_census(): stats_stats = stats.Stats() app.m_response_ms = measure_module.MeasureFloat("flask_response_time", "The request duration", "ms") app.key_method = tag_key_module.TagKey("method") # Create the status key app.key_status = tag_key_module.TagKey("status") # Create the error key app.key_error = tag_key_module.TagKey("error") app.view_manager = stats_stats.view_manager app.stats_recorder = stats_stats.stats_recorder response_time_view = view.View( "response_time", "The time it took to respond", [app.key_method, app.key_status, app.key_error], app.m_response_ms, aggregation.LastValueAggregation()) app.exporter = stackdriver.new_stats_exporter(options=stackdriver.Options( project_id=os.getenv('PROJECT_ID'))) app.view_manager.register_exporter(app.exporter) app.view_manager.register_view(response_time_view)
def test_new_aggregation_data_explicit(self): measure = mock.Mock(spec=measure_module.MeasureInt) last_value_aggregation = aggregation_module.LastValueAggregation( value=6) agg_data = last_value_aggregation.new_aggregation_data(measure) self.assertEqual(6, agg_data.value) self.assertEqual(value.ValueLong, agg_data.value_type)
def track_metric(self, metric_name, metric_value): try: if (self.appinsights_key): if (not metric_name in self.metrics): metrics_measure = measure_module.MeasureInt( metric_name, metric_name, metric_name) metrics_view = view_module.View( metric_name, metric_name, [], metrics_measure, aggregation_module.LastValueAggregation( value=metric_value)) self.view_manager.register_view(metrics_view) mmap = self.stats_recorder.new_measurement_map() tmap = tag_map_module.TagMap() self.metrics[metric_name] = { 'measure': metrics_measure, 'measurement_map': mmap, 'tag_map': tmap } measure = self.metrics[metric_name]['measure'] mmap = self.metrics[metric_name]['measurement_map'] tmap = self.metrics[metric_name]['tag_map'] mmap.measure_int_put(measure, metric_value) mmap.record(tmap) except Exception as e: print('Exception when tracking a metric:') print(e)
def track_metric(self, metric_name, metric_value): if self.appinsights_key: print("Tracking metric:" + metric_name + ", Value: " + str(metric_value)) if not metric_name in self.metrics: metrics_measure = measure_module.MeasureInt( metric_name, metric_name, metric_name) metrics_view = view_module.View( metric_name, metric_name, [], metrics_measure, aggregation_module.LastValueAggregation( value=metric_value), ) view_manager.register_view(metrics_view) mmap = stats_recorder.new_measurement_map() tmap = tag_map_module.TagMap() self.metrics[metric_name] = { "measure": metrics_measure, "measurement_map": mmap, "tag_map": tmap, } measure = self.metrics[metric_name]["measure"] mmap = self.metrics[metric_name]["measurement_map"] tmap = self.metrics[metric_name]["tag_map"] print("Putting metric:" + metric_name + ", Value: " + str(metric_value)) mmap.measure_int_put(measure, metric_value) mmap.record(tmap)
def track_metric(self, metric_name, metric_value): try: if self.appinsights_key: if not metric_name in self.metrics: metrics_measure = measure_module.MeasureInt( metric_name, metric_name, metric_name) metrics_view = view_module.View( metric_name, metric_name, [], metrics_measure, aggregation_module.LastValueAggregation( value=metric_value), ) self.view_manager.register_view(metrics_view) mmap = self.stats_recorder.new_measurement_map() tmap = tag_map_module.TagMap() self.metrics[metric_name] = { "measure": metrics_measure, "measurement_map": mmap, "tag_map": tmap, } measure = self.metrics[metric_name]["measure"] mmap = self.metrics[metric_name]["measurement_map"] tmap = self.metrics[metric_name]["tag_map"] mmap.measure_int_put(measure, metric_value) mmap.record(tmap) except Exception as e: print("Exception when tracking a metric:") print(e)
def track_metric(self, metric_name, metric_value): if (self.appinsights_key): print("Tracking metric:" + metric_name + ", Value: " + str(metric_value)) if (not metric_name in self.metrics): metrics_measure = measure_module.MeasureInt( metric_name, metric_name, metric_name) metrics_view = view_module.View( metric_name, metric_name, [], metrics_measure, aggregation_module.LastValueAggregation( value=metric_value)) view_manager.register_view(metrics_view) mmap = stats_recorder.new_measurement_map() tmap = tag_map_module.TagMap() self.metrics[metric_name] = { 'measure': metrics_measure, 'measurement_map': mmap, 'tag_map': tmap } measure = self.metrics[metric_name]['measure'] mmap = self.metrics[metric_name]['measurement_map'] tmap = self.metrics[metric_name]['tag_map'] print("Putting metric:" + metric_name + ", Value: " + str(metric_value)) mmap.measure_int_put(measure, metric_value) mmap.record(tmap)
def init_app(self, app): self.app = app params = self.app.config.get('OPENCENSUS_TRACE_PARAMS', {}) self.blacklist_paths = params.get(BLACKLIST_PATHS, self.blacklist_paths) transport = params.get(TRANSPORT, sync.SyncTransport) # Initialize the exporter if not inspect.isclass(self.exporter): pass # handling of instantiated exporter elif self.exporter.__name__ == 'StackdriverExporter': _project_id = params.get(GCP_EXPORTER_PROJECT, None) self.exporter = self.exporter(project_id=_project_id, transport=transport) else: self.exporter = self.exporter(transport=transport) stats_stats = stats.Stats() self.app.stats_recorder = stats_stats.stats_recorder self.app.view_manager = stats_stats.view_manager response_time_view = view.View( "response_time", "The time it took to respond", [self.app.key_method, self.app.key_status, self.app.key_error], self.app.m_response_ms, aggregation.LastValueAggregation()) self.app.view_manager.register_exporter(self.exporter) self.app.view_manager.register_view(response_time_view) self.setup_metrics()
def test_collector_collect(self): agg = aggregation_module.LastValueAggregation(256) view = view_module.View("new_view", "processed video size over time", [FRONTEND_KEY], VIDEO_SIZE_MEASURE, agg) registry = mock.Mock() view_data = mock.Mock() options = prometheus.Options("test2", 8001, "localhost", registry) collector = prometheus.Collector(options=options, view_data=view_data) collector.register_view(view) desc = collector.registered_views[ 'test2_new_view-my.org/keys/frontend'] collector.to_metric(desc=desc, view=view) registry = mock.Mock() view_data = mock.Mock() options = prometheus.Options("test1", 8001, "localhost", registry) collector = prometheus.Collector(options=options, view_data=view_data) collector.register_view(VIDEO_SIZE_VIEW) desc = collector.registered_views[list(REGISTERED_VIEW)[0]] metric = collector.to_metric(desc=desc, view=VIDEO_SIZE_VIEW) self.assertEqual(desc['name'], metric.name) self.assertEqual(desc['documentation'], metric.documentation) self.assertEqual('histogram', metric.type) self.assertEqual(5, len(metric.samples))
def test_constructor_explicit(self): val = 16 last_value_aggregation = aggregation_module.LastValueAggregation( value=val) self.assertEqual(16, last_value_aggregation.value) self.assertEqual(aggregation_module.Type.LASTVALUE, last_value_aggregation.aggregation_type)
def set_view(metric, description, measure): """ Sets the view for the custom metric """ prompt_view = view_module.View( metric, description, [], measure, aggregation_module.LastValueAggregation()) stats_module.stats.view_manager.register_view(prompt_view)
def create_metric_view(view_manager, name, description, measure): # view must be registered prior to record ping_view = view_module.View( name=name, description=description, columns=[tag_key_isp, tag_key_server_host], measure=measure, aggregation=aggregation_module.LastValueAggregation()) view_manager.register_view(ping_view)
def configure(): """Globally enables metrics collection.""" global _METRICS_ENABLED if _METRICS_ENABLED: return _METRICS_ENABLED = True STATS.view_manager.register_exporter(_new_exporter()) latency_view = view.View( "fireci/latency", "Latency of fireci execution stages", _TAGS, _m_latency, aggregation.LastValueAggregation()) success_view = view.View( "fireci/success", "Success indication of fireci execution stages", _TAGS, _m_success, aggregation.LastValueAggregation()) STATS.view_manager.register_view(latency_view) STATS.view_manager.register_view(success_view) context = tags.TagMap() for tag in _TAGS: if tag.upper() in os.environ: context.insert(tag, tags.TagValue(os.environ[tag.upper()])) execution_context.set_current_tag_map(context)
def test_create_metric_descriptor_last_value_float(self): client = mock.Mock() option = stackdriver.Options( project_id="project-test", metric_prefix="teste") view_name_base = "view-base" agg_base = aggregation_module.LastValueAggregation() view_base = view_module.View( view_name_base, "processed video size over time", [FRONTEND_KEY], VIDEO_SIZE_MEASURE_FLOAT, agg_base) exporter = stackdriver.StackdriverStatsExporter( options=option, client=client) desc = exporter.create_metric_descriptor(view_base) self.assertIsNotNone(desc)
def create_measurement_view(self, measurement_name): "creates a measurement and a view" tg_key = tag_key_module.TagKey("TEST_ID") measurement = measure_module.MeasureInt( f"gw_m_{measurement_name}_response", "response time of the home page", "s") view_name = f"views_{measurement_name}_response" aggregation = aggregation_module.LastValueAggregation() view = view_module.View(view_name, f"glasswall {measurement_name} response time", [tg_key], measurement, aggregation) # Register view. self.view_manager.register_view(view) return measurement
def setup_open_census(): stats_stats = stats.Stats() app.view_manager = stats_stats.view_manager app.stats_recorder = stats_stats.stats_recorder response_time_view = view.View("response_time", "The time it took to respond", [key_method, key_status, key_error], m_response_ms, aggregation.LastValueAggregation()) app.exporter = prometheus.new_stats_exporter( prometheus.Options(namespace="flask_app", port=8000)) app.view_manager.register_exporter(app.exporter) app.view_manager.register_view(response_time_view)
def test_create_timeseries_last_value_float_tagvalue( self, monitor_resource_mock): client = mock.Mock() option = stackdriver.Options(project_id="project-test", resource="global") exporter = stackdriver.StackdriverStatsExporter(options=option, client=client) stats = stats_module.Stats() view_manager = stats.view_manager stats_recorder = stats.stats_recorder if len(view_manager.measure_to_view_map.exporters) > 0: view_manager.unregister_exporter( view_manager.measure_to_view_map.exporters[0]) view_manager.register_exporter(exporter) agg_1 = aggregation_module.LastValueAggregation(value=2) view_name1 = "view-name1" new_view1 = view_module.View(view_name1, "processed video size over time", [FRONTEND_KEY_FLOAT], VIDEO_SIZE_MEASURE_FLOAT, agg_1) view_manager.register_view(new_view1) tag_value_int = tag_value_module.TagValue("Abc") tag_map = tag_map_module.TagMap() tag_map.insert(FRONTEND_KEY_INT, tag_value_int) measure_map = stats_recorder.new_measurement_map() measure_map.measure_int_put(VIDEO_SIZE_MEASURE, 25 * MiB) measure_map.record(tag_map) v_data = measure_map.measure_to_view_map.get_view(view_name1, None) time_series = exporter.create_time_series_list(v_data, "global", "kubernetes.io/myorg") self.assertEquals(time_series.metric.type, "kubernetes.io/myorg/view-name1") self.assertIsNotNone(time_series)
def test_collector_to_metric_last_value(self): agg = aggregation_module.LastValueAggregation(256) view = view_module.View(VIDEO_SIZE_VIEW_NAME, "processed video size over time", [FRONTEND_KEY], VIDEO_SIZE_MEASURE, agg) registry = mock.Mock() view_data = mock.Mock() options = prometheus.Options("test1", 8001, "localhost", registry) collector = prometheus.Collector(options=options, view_data=view_data) collector.register_view(view) desc = collector.registered_views[list(REGISTERED_VIEW)[0]] metric = collector.to_metric(desc=desc, view=view) self.assertEqual(desc['name'], metric.name) self.assertEqual(desc['documentation'], metric.documentation) self.assertEqual('gauge', metric.type) self.assertEqual(1, len(metric.samples))
def test_collector_collect_with_none_label_value(self): agg = aggregation_module.LastValueAggregation(256) view = view_module.View("new_view", "processed video size over time", [FRONTEND_KEY], VIDEO_SIZE_MEASURE, agg) registry = mock.Mock() options = prometheus.Options("test3", 8001, "localhost", registry) collector = prometheus.Collector(options=options) collector.register_view(view) desc = collector.registered_views['test3_new_view'] metric = collector.to_metric( desc=desc, tag_values=[None], agg_data=agg.aggregation_data) self.assertEqual(1, len(metric.samples)) sample = metric.samples[0] # Sample is a namedtuple # ('Sample', ['name', 'labels', 'value', 'timestamp', 'exemplar']) label_map = sample[1] self.assertEqual({"myorg_keys_frontend": ""}, label_map)
def test_collector_collect(self): agg = aggregation_module.LastValueAggregation(256) view = view_module.View("new_view", "processed video size over time", [FRONTEND_KEY], VIDEO_SIZE_MEASURE, agg) registry = mock.Mock() options = prometheus.Options("test2", 8001, "localhost", registry) collector = prometheus.Collector(options=options) collector.register_view(view) desc = collector.registered_views['test2_new_view'] metric = collector.to_metric( desc=desc, tag_values=[tag_value_module.TagValue("value")], agg_data=agg.aggregation_data) self.assertEqual(desc['name'], metric.name) self.assertEqual(desc['documentation'], metric.documentation) self.assertEqual('gauge', metric.type) expected_samples = [ Sample(metric.name, {"myorg_keys_frontend": "value"}, 256)] self.assertEqual(expected_samples, metric.samples)
def test_create_timeseries_str_tagvalue(self, monitor_resource_mock): view_manager, stats_recorder, exporter = \ self.setup_create_timeseries_test() agg_1 = aggregation_module.LastValueAggregation(value=2) view_name1 = "view-name1" new_view1 = view_module.View(view_name1, "processed video size over time", [FRONTEND_KEY_INT], VIDEO_SIZE_MEASURE_2, agg_1) view_manager.register_view(new_view1) tag_value_int = tag_value_module.TagValue("Abc") tag_map = tag_map_module.TagMap() tag_map.insert(FRONTEND_KEY_INT, tag_value_int) measure_map = stats_recorder.new_measurement_map() measure_map.measure_int_put(VIDEO_SIZE_MEASURE_2, 25 * MiB) measure_map.record(tag_map) v_data = measure_map.measure_to_view_map.get_view(view_name1, None) v_data = metric_utils.view_data_to_metric(v_data, TEST_TIME) time_series_list = exporter.create_time_series_list(v_data) self.assertEqual(len(time_series_list), 1) time_series = time_series_list[0] self.check_labels(time_series.metric.labels, {FRONTEND_KEY_INT_CLEAN: "Abc"}, include_opencensus=True) self.assertIsNotNone(time_series.resource) self.assertEqual(len(time_series.points), 1) expected_value = monitoring_v3.types.TypedValue() # TODO: #565 expected_value.double_value = 25.0 * MiB self.assertEqual(time_series.points[0].value, expected_value)
def test_create_timeseries_last_value_float_tagvalue( self, monitor_resource_mock): view_manager, stats_recorder, exporter = \ self.setup_create_timeseries_test() agg_2 = aggregation_module.LastValueAggregation(value=2.2 * MiB) view_name2 = "view-name2" new_view2 = view_module.View(view_name2, "processed video size over time", [FRONTEND_KEY_FLOAT], VIDEO_SIZE_MEASURE_FLOAT, agg_2) view_manager.register_view(new_view2) tag_value_float = tag_value_module.TagValue("Abc") tag_map = tag_map_module.TagMap() tag_map.insert(FRONTEND_KEY_FLOAT, tag_value_float) measure_map = stats_recorder.new_measurement_map() measure_map.measure_float_put(VIDEO_SIZE_MEASURE_FLOAT, 25.7 * MiB) measure_map.record(tag_map) v_data = measure_map.measure_to_view_map.get_view(view_name2, None) time_series_list = exporter.create_time_series_list( v_data, "global", "kubernetes.io/myorg") self.assertEqual(len(time_series_list), 1) time_series = time_series_list[0] self.assertEqual(time_series.metric.type, "kubernetes.io/myorg/view-name2") self.assertCorrectLabels(time_series.metric.labels, {FRONTEND_KEY_FLOAT_CLEAN: "Abc"}, include_opencensus=True) self.assertIsNotNone(time_series.resource) self.assertEqual(len(time_series.points), 1) expected_value = monitoring_v3.types.TypedValue() expected_value.double_value = 25.7 * MiB self.assertEqual(time_series.points[0].value, expected_value)
APP_INSIGHT_ON) stats = stats_module.stats view_manager = stats.view_manager stats_recorder = stats.stats_recorder exporter = metrics_exporter.new_metrics_exporter( connection_string=APP_INSIGHT_CONN_STR) view_manager.register_exporter(exporter) # # Part # PARTS_MEASURE = measure_module.MeasureInt("part", "number of parts", "parts") PARTS_VIEW = view_module.View("part_view", "number of parts", [], PARTS_MEASURE, aggregation_module.LastValueAggregation()) view_manager.register_view(PARTS_VIEW) # # Image # IMAGES_MEASURE = measure_module.MeasureInt("images", "number of images", "images") IMAGES_VIEW = view_module.View("image_view", "number of images", [], IMAGES_MEASURE, aggregation_module.LastValueAggregation()) view_manager.register_view(IMAGES_VIEW) # # TRAINING_JOB #
def __init__(self, name, description, unit, tags: List[str]): self._measure = measure_module.MeasureInt(name, description, unit) tags = [tag_key_module.TagKey(tag) for tag in tags] self._view = View(name, description, tags, self.measure, aggregation.LastValueAggregation())
def __init__(self, name, description, unit, tags: List[tag_key_module.TagKey]): self._measure = measure_module.MeasureInt(name, description, unit) self._view = view.View(name, description, tags, self.measure, aggregation.LastValueAggregation())
from opencensus.stats import view_data as view_data_module from opencensus.stats import metric_utils from opencensus_ext_newrelic import NewRelicStatsExporter from newrelic_telemetry_sdk import MetricClient # The latency in milliseconds MEASURE = measure_module.MeasureFloat("number", "A number!", "things") GAUGE_VIEWS = { "last": view_module.View( "last", "A last value", ("tag",), MEASURE, aggregation_module.LastValueAggregation(), ) } COUNT_VIEWS = { "count": view_module.View( "count", "A count", ("tag",), MEASURE, aggregation_module.CountAggregation() ), "sum": view_module.View( "sum", "A sum", ("tag",), MEASURE, aggregation_module.SumAggregation() ), } DISTRIBUTION_VIEWS = { "distribution": view_module.View( "distribution", "A distribution", ("tag",),
def test_constructor_defaults(self): last_value_aggregation = aggregation_module.LastValueAggregation() self.assertEqual(0, last_value_aggregation.value) self.assertEqual(aggregation_module.Type.LASTVALUE, last_value_aggregation.aggregation_type)
def test_new_aggregation_data_float(self): measure = mock.Mock(spec=measure_module.MeasureFloat) last_value_aggregation = aggregation_module.LastValueAggregation() agg_data = last_value_aggregation.new_aggregation_data(measure) self.assertEqual(0, agg_data.value) self.assertEqual(value.ValueDouble, agg_data.value_type)
def test_new_aggregation_data_bad(self): measure = mock.Mock(spec=measure_module.BaseMeasure) last_value_aggregation = aggregation_module.LastValueAggregation() with self.assertRaises(ValueError): last_value_aggregation.new_aggregation_data(measure)