class TestOpenTSDBReporter(TimedTestCase): def setUp(self): super(TestOpenTSDBReporter, self).setUp() self.registry = MetricsRegistry(clock=self.clock) self.maxDiff = None def tearDown(self): super(TestOpenTSDBReporter, self).tearDown() def test_report_now(self): r = OpenTSDBReporter(application_name="app", write_key="key", registry=self.registry, reporting_interval=1, clock=self.clock, prefix="prefix.", url="http://opentsdb.com/api/put") h1 = self.registry.histogram("hist") for i in range(10): h1.add(2**i) t1 = self.registry.timer("t1") m1 = self.registry.meter("m1") m1.mark() with t1.time(): c1 = self.registry.counter("c1") c2 = self.registry.counter("counter-2") c1.inc() c2.dec() c2.dec() self.clock.add(1) output = r._collect_metrics(registry=self.registry) self.assertEqual(len(output), 31) for data in output: assert data['metric'].startswith("prefix.") def test_send_request(self): r = OpenTSDBReporter(application_name="app", write_key="key", registry=self.registry, reporting_interval=1, clock=self.clock, prefix="prefix.", url="http://opentsdb.com/api/put") h1 = self.registry.histogram("hist") for i in range(10): h1.add(2**i) t1 = self.registry.timer("t1") m1 = self.registry.meter("m1") m1.mark() with t1.time(): c1 = self.registry.counter("c1") c2 = self.registry.counter("counter-2") c1.inc() c2.dec() c2.dec() self.clock.add(1) with mock.patch("pyformance.reporters.opentsdb_reporter.urllib.urlopen" ) as patch: r.report_now() patch.assert_called()
class TestCloudWatchReporter(TimedTestCase): def setUp(self): super(TestCloudWatchReporter, self).setUp() self.registry = MetricsRegistry(clock=self.clock) self.maxDiff = None def tearDown(self): super(TestCloudWatchReporter, self).tearDown() def test_report_now(self): r = CloudWatchReporter(application_name="app", tags={"host":"localhost"}, registry=self.registry, reporting_interval=1, clock=self.clock, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, region=REGION) h1 = self.registry.histogram("hist") for i in range(10): h1.add(2 ** i) t1 = self.registry.timer("t1") m1 = self.registry.meter("m1") m1.mark() with t1.time(): c1 = self.registry.counter("c1") c2 = self.registry.counter("counter-2") c1.inc() c2.dec() c2.dec() self.clock.add(1) output = r._collect_metrics(registry=self.registry) self.assertEqual(len(output), 31) for data in output: assert data['metric'].startswith("prefix.") def test_send_request(self): r = CloudWatchReporter(application_name="app", tags={"host":"localhost"}, registry=self.registry, reporting_interval=1, clock=self.clock, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, region="ap-south-1") h1 = self.registry.histogram("hist") for i in range(10): h1.add(2 ** i) t1 = self.registry.timer("t1") m1 = self.registry.meter("m1") m1.mark() with t1.time(): c1 = self.registry.counter("c1") c2 = self.registry.counter("counter-2") c1.inc() c2.dec() c2.dec() self.clock.add(1) r.report_now() # with mock.patch("pyformance.reporters.cloudwatch_reporter.botocore.client.CloudWatch.put_metric_data") as patch: with mock.patch("pyformance.reporters.cloudwatch_report.CloudWatchReporter._collect_metrics") as patch: r.report_now() patch.assert_called()
class TestOpenTSDBReporter(TimedTestCase): def setUp(self): super(TestOpenTSDBReporter, self).setUp() self.registry = MetricsRegistry(clock=self.clock) self.maxDiff = None def tearDown(self): super(TestOpenTSDBReporter, self).tearDown() def test_report_now(self): r = OpenTSDBReporter(application_name="app", write_key="key", registry=self.registry, reporting_interval=1, clock=self.clock, prefix="prefix.", url="http://opentsdb.com/api/put") h1 = self.registry.histogram("hist") for i in range(10): h1.add(2 ** i) t1 = self.registry.timer("t1") m1 = self.registry.meter("m1") m1.mark() with t1.time(): c1 = self.registry.counter("c1") c2 = self.registry.counter("counter-2") c1.inc() c2.dec() c2.dec() self.clock.add(1) output = r._collect_metrics(registry=self.registry) self.assertEqual(len(output), 31) for data in output: assert data['metric'].startswith("prefix.") def test_send_request(self): r = OpenTSDBReporter(application_name="app", write_key="key", registry=self.registry, reporting_interval=1, clock=self.clock, prefix="prefix.", url="http://opentsdb.com/api/put") h1 = self.registry.histogram("hist") for i in range(10): h1.add(2 ** i) t1 = self.registry.timer("t1") m1 = self.registry.meter("m1") m1.mark() with t1.time(): c1 = self.registry.counter("c1") c2 = self.registry.counter("counter-2") c1.inc() c2.dec() c2.dec() self.clock.add(1) with mock.patch("pyformance.reporters.opentsdb_reporter.urllib.urlopen") as patch: r.report_now() patch.assert_called()
class TestSysLogReporter(TimedTestCase): def setUp(self): super(TestSysLogReporter, self).setUp() self.registry = MetricsRegistry(clock=self.clock) self.maxDiff = None self.clock.now = 0 def tearDown(self): super(TestSysLogReporter, self).tearDown() self.clock.now = 0 def test_report_now(self): #connect to a logal rsyslog server r = SysLogReporter(registry=self.registry, reporting_interval=1, clock=self.clock) h1 = self.registry.histogram("hist") for i in range(10): h1.add(2 ** i) gcb = self.registry.gauge("gcb", lambda: 123) gsimple = self.registry.gauge("gsimple").set_value(42) t1 = self.registry.timer("t1") m1 = self.registry.meter("m1") m1.mark() with t1.time(): c1 = self.registry.counter("c1") c2 = self.registry.counter("counter-2") c1.inc() c2.dec() c2.dec() self.clock.add(1) with mock.patch("pyformance.reporters.syslog_reporter.logging.Logger.info") as patch: r.report_now() expected = '{"c1.count": 1, "counter-2.count": -2, "gcb.value": 123, "gsimple.value": 42, "hist.75_percentile": 160.0, "hist.95_percentile": 512, "hist.999_percentile": 512, "hist.99_percentile": 512, "hist.avg": 102.3, "hist.count": 10.0, "hist.max": 512, "hist.min": 1, "hist.std_dev": 164.94851048466947, "m1.15m_rate": 0, "m1.1m_rate": 0, "m1.5m_rate": 0, "m1.count": 1.0, "m1.mean_rate": 1.0, "t1.15m_rate": 0, "t1.1m_rate": 0, "t1.50_percentile": 1, "t1.5m_rate": 0, "t1.75_percentile": 1, "t1.95_percentile": 1, "t1.999_percentile": 1, "t1.99_percentile": 1, "t1.avg": 1.0, "t1.count": 1.0, "t1.max": 1, "t1.mean_rate": 1.0, "t1.min": 1, "t1.std_dev": 0.0, "t1.sum": 1.0, "timestamp": 1}' patch.assert_called_with(expected)
class TestSysLogReporter(TimedTestCase): def setUp(self): super(TestSysLogReporter, self).setUp() self.registry = MetricsRegistry(clock=self.clock) self.maxDiff = None self.clock.now = 0 def tearDown(self): super(TestSysLogReporter, self).tearDown() self.clock.now = 0 def test_report_now(self): # connect to a local rsyslog server r = SysLogReporter(registry=self.registry, reporting_interval=1, clock=self.clock) h1 = self.registry.histogram("hist") for i in range(10): h1.add(2 ** i) gcb = self.registry.gauge("gcb", lambda: 123) gsimple = self.registry.gauge("gsimple").set_value(42) t1 = self.registry.timer("t1") m1 = self.registry.meter("m1") m1.mark() with t1.time(): c1 = self.registry.counter("c1") c2 = self.registry.counter("counter-2") c1.inc() c2.dec() c2.dec() self.clock.add(1) with mock.patch("pyformance.reporters.syslog_reporter.logging.Logger.info") as patch: r.report_now() expected = '{"c1.count": 1, "counter-2.count": -2, "gcb.value": 123, "gsimple.value": 42, "hist.75_percentile": 160.0, "hist.95_percentile": 512, "hist.999_percentile": 512, "hist.99_percentile": 512, "hist.avg": 102.3, "hist.count": 10.0, "hist.max": 512, "hist.min": 1, "hist.std_dev": 164.94851048466944, "m1.15m_rate": 0, "m1.1m_rate": 0, "m1.5m_rate": 0, "m1.count": 1.0, "m1.mean_rate": 1.0, "t1.15m_rate": 0, "t1.1m_rate": 0, "t1.50_percentile": 1, "t1.5m_rate": 0, "t1.75_percentile": 1, "t1.95_percentile": 1, "t1.999_percentile": 1, "t1.99_percentile": 1, "t1.avg": 1.0, "t1.count": 1.0, "t1.max": 1, "t1.mean_rate": 1.0, "t1.min": 1, "t1.std_dev": 0.0, "t1.sum": 1.0, "timestamp": 1}' patch.assert_called_with(expected)
class TestCarbonReporter(TimedTestCase): def setUp(self): super(TestCarbonReporter, self).setUp() self.output = StringIO() self.registry = MetricsRegistry(clock=self.clock) self.maxDiff = None def connect(self, *args): # part of fake socket interface pass def sendall(self, data): # part of fake socket interface self.output.write(data) def close(self): # part of fake socket interface pass def tearDown(self): super(TestCarbonReporter, self).tearDown() def test_report_now(self): r = CarbonReporter(registry=self.registry, reporting_interval=1, clock=self.clock, socket_factory=lambda: self) h1 = self.registry.histogram("hist") for i in range(10): h1.add(2**i) gcb = self.registry.gauge("gcb", lambda: 123) gsimple = self.registry.gauge("gsimple").set_value(42) t1 = self.registry.timer("t1") m1 = self.registry.meter("m1") m1.mark() with t1.time(): c1 = self.registry.counter("c1") c2 = self.registry.counter("counter-2") c1.inc() c2.dec() c2.dec() self.clock.add(1) r.report_now() self.assertEqual(self.output.getvalue().splitlines(), [ 'counter-2.count -2 1', 'gsimple.value 42 1', 'gcb.value 123 1', 't1.1m_rate 0 1', 't1.999_percentile 1 1', 't1.15m_rate 0 1', 't1.99_percentile 1 1', 't1.mean_rate 1.0 1', 't1.95_percentile 1 1', 't1.min 1 1', 't1.5m_rate 0 1', 't1.count 1.0 1', 't1.75_percentile 1 1', 't1.std_dev 0.0 1', 't1.max 1 1', 't1.avg 1.0 1', 'hist.count 10.0 1', 'hist.999_percentile 512 1', 'hist.99_percentile 512 1', 'hist.min 1 1', 'hist.95_percentile 512 1', 'hist.75_percentile 160.0 1', 'hist.std_dev 164.948510485 1', 'hist.max 512 1', 'hist.avg 102.3 1', 'm1.1m_rate 0 1', 'm1.15m_rate 0 1', 'm1.5m_rate 0 1', 'm1.mean_rate 1.0 1', 'c1.count 1 1' ])
def report_metrics(host, server, token): reg = MetricsRegistry() wf_proxy_reporter = WavefrontProxyReporter( host=host, port=2878, registry=reg, source="wavefront-pyformance-example", tags={ "key1": "val1", "key2": "val2" }, prefix="python.proxy.") wf_direct_reporter = WavefrontDirectReporter( server=server, token=token, registry=reg, source="wavefront-pyformance-exmaple", tags={ "key1": "val1", "key2": "val2" }, prefix="python.direct.") # counter c1 = reg.counter("foo_count") c1.inc() # delta counter d1 = delta.delta_counter(reg, "foo_delta_count") d1.inc() d1.inc() # gauge g1 = reg.gauge("foo_gauge") g1.set_value(2) # meter m1 = reg.meter("foo_meter") m1.mark() # timer t1 = reg.timer("foo_timer") timer_ctx = t1.time() time.sleep(3) timer_ctx.stop() # histogram h1 = reg.histogram("foo_histogram") h1.add(1.0) h1.add(1.5) wf_proxy_reporter.report_now() wf_proxy_reporter.stop() wf_direct_reporter.report_now()
class TestConsoleReporter(TimedTestCase): def setUp(self): super(TestConsoleReporter, self).setUp() self.output = StringIO() self.registry = MetricsRegistry(clock=self.clock) self.maxDiff = None self.clock.now = 0 def tearDown(self): super(TestConsoleReporter, self).tearDown() self.clock.now = 0 def test_report_now(self): r = ConsoleReporter(registry=self.registry, reporting_interval=1, stream=self.output, clock=self.clock) h1 = self.registry.histogram("hist") for i in range(10): h1.add(2**i) gcb = self.registry.gauge("gcb", lambda: 123) gsimple = self.registry.gauge("gsimple").set_value(42) t1 = self.registry.timer("t1") m1 = self.registry.meter("m1") m1.mark() with t1.time(): c1 = self.registry.counter("c1") c2 = self.registry.counter("counter-2") c1.inc() c2.dec() c2.dec() self.clock.add(1) r.report_now() self.assertEqual(self.output.getvalue().splitlines(), [ '== 1970-01-01 00:00:01 ===================================', 'counter-2:', ' count = -2', 'gsimple:', ' value = 42', 'gcb:', ' value = 123', 't1:', ' 1m_rate = 0', ' 999_percentile = 1', ' 15m_rate = 0', ' 99_percentile = 1', ' mean_rate = 1.0', ' 95_percentile = 1', ' min = 1', ' 5m_rate = 0', ' count = 1.0', ' 75_percentile = 1', ' std_dev = 0.0', ' max = 1', ' avg = 1.0', 'hist:', ' count = 10.0', ' 999_percentile = 512', ' 99_percentile = 512', ' min = 1', ' 95_percentile = 512', ' 75_percentile = 160.0', ' std_dev = 164.948510485', ' max = 512', ' avg = 102.3', 'm1:', ' 1m_rate = 0', ' 15m_rate = 0', ' 5m_rate = 0', ' mean_rate = 1.0', 'c1:', ' count = 1', '' ])
class TestCarbonReporter(TimedTestCase): def setUp(self): super(TestCarbonReporter, self).setUp() self.output = BytesIO() self.registry = MetricsRegistry(clock=self.clock) self.maxDiff = None def connect(self, *args): # part of fake socket interface pass def sendall(self, data): # part of fake socket interface self.output.write(data) def close(self): # part of fake socket interface pass def tearDown(self): super(TestCarbonReporter, self).tearDown() def capture_test_metrics(self): self.clock.now = 1 h1 = self.registry.histogram("hist") for i in range(10): h1.add(2**i) gcb = self.registry.gauge("gcb", lambda: 123) gsimple = self.registry.gauge("gsimple").set_value(42) t1 = self.registry.timer("t1") m1 = self.registry.meter("m1") m1.mark() with t1.time(): c1 = self.registry.counter("c1") c2 = self.registry.counter("counter-2") c1.inc() c2.dec() c2.dec() self.clock.add(1) def test_report_now_plain(self): r = CarbonReporter(registry=self.registry, reporting_interval=1, clock=self.clock, socket_factory=lambda: self) self.capture_test_metrics() r.report_now() test_data = sorted(self.output.getvalue().decode().splitlines()) expected_data = sorted([ 'counter-2.count -2 2', 'c1.count 1 2', 'gsimple.value 42 2', 'gcb.value 123 2', 't1.1m_rate 0 2', 't1.999_percentile 1 2', 't1.15m_rate 0 2', 't1.99_percentile 1 2', 't1.mean_rate 1.0 2', 't1.95_percentile 1 2', 't1.min 1 2', 't1.50_percentile 1 2', 't1.5m_rate 0 2', 't1.count 1.0 2', 't1.75_percentile 1 2', 't1.std_dev 0.0 2', 't1.max 1 2', 't1.sum 1.0 2', 't1.avg 1.0 2', 'hist.count 10.0 2', 'hist.999_percentile 512 2', 'hist.99_percentile 512 2', 'hist.min 1 2', 'hist.95_percentile 512 2', 'hist.75_percentile 160.0 2', 'hist.50_percentile 24.0 2', 'hist.std_dev 164.94851048466947 2' \ if PY3 else 'hist.std_dev 164.948510485 2', 'hist.max 512 2', 'hist.avg 102.3 2', 'm1.count 1.0 2', 'm1.1m_rate 0 2', 'm1.15m_rate 0 2', 'm1.5m_rate 0 2', 'm1.mean_rate 1.0 2', ]) self.assertEqual(test_data, expected_data) def test_report_now_pickle(self): r = CarbonReporter(registry=self.registry, reporting_interval=1, clock=self.clock, socket_factory=lambda: self, pickle_protocol=True) self.capture_test_metrics() r.report_now() test_data = sorted(pickle.loads(self.output.getvalue()[4:])) expected_data = sorted([('counter-2.count', (2, -2.0)), ('c1.count', (2, 1)), ('gsimple.value', (2, 42.0)), ('gcb.value', (2, 123.0)), ('t1.1m_rate', (2, 0.0)), ('t1.999_percentile', (2, 1)), ('t1.15m_rate', (2, 0.0)), ('t1.99_percentile', (2, 1)), ('t1.mean_rate', (2, 1)), ('t1.95_percentile', (2, 1)), ('t1.min', (2, 1)), ('t1.50_percentile', (2, 1)), ('t1.5m_rate', (2, 0.0)), ('t1.count', (2, 1)), ('t1.75_percentile', (2, 1)), ('t1.std_dev', (2, 0.0)), ('t1.max', (2, 1)), ('t1.sum', (2, 1)), ('t1.avg', (2, 1)), ('hist.count', (2, 10.0)), ('hist.999_percentile', (2, 512.0)), ('hist.99_percentile', (2, 512.0)), ('hist.min', (2, 1)), ('hist.95_percentile', (2, 512.0)), ('hist.75_percentile', (2, 160.0)), ('hist.50_percentile', (2, 24.0)), ('hist.std_dev', (2, 164.94851048466947)), ('hist.max', (2, 512.0)), ('hist.avg', (2, 102.3)), ('m1.count', (2, 1)), ('m1.1m_rate', (2, 0.0)), ('m1.15m_rate', (2, 0.0)), ('m1.5m_rate', (2, 0.0)), ('m1.mean_rate', (2, 1))]) self.assertEqual(test_data, expected_data)
class TestCarbonReporter(TimedTestCase): def setUp(self): super(TestCarbonReporter, self).setUp() self.output = BytesIO() self.registry = MetricsRegistry(clock=self.clock) self.maxDiff = None def connect(self, *args): # part of fake socket interface pass def sendall(self, data): # part of fake socket interface self.output.write(data) def close(self): # part of fake socket interface pass def tearDown(self): super(TestCarbonReporter, self).tearDown() def capture_test_metrics(self): self.clock.now = 1 h1 = self.registry.histogram("hist") for i in range(10): h1.add(2 ** i) gcb = self.registry.gauge("gcb", lambda: 123) gsimple = self.registry.gauge("gsimple").set_value(42) t1 = self.registry.timer("t1") m1 = self.registry.meter("m1") m1.mark() with t1.time(): c1 = self.registry.counter("c1") c2 = self.registry.counter("counter-2") c1.inc() c2.dec() c2.dec() self.clock.add(1) def test_report_now_plain(self): r = CarbonReporter( registry=self.registry, reporting_interval=1, clock=self.clock, socket_factory=lambda: self, ) self.capture_test_metrics() r.report_now() test_data = sorted(self.output.getvalue().decode().splitlines()) expected_data = sorted( [ "counter-2.count -2 2", "c1.count 1 2", "gsimple.value 42 2", "gcb.value 123 2", "t1.1m_rate 0 2", "t1.999_percentile 1 2", "t1.15m_rate 0 2", "t1.99_percentile 1 2", "t1.mean_rate 1.0 2", "t1.95_percentile 1 2", "t1.min 1 2", "t1.50_percentile 1 2", "t1.5m_rate 0 2", "t1.count 1.0 2", "t1.75_percentile 1 2", "t1.std_dev 0.0 2", "t1.max 1 2", "t1.sum 1.0 2", "t1.avg 1.0 2", "hist.count 10.0 2", "hist.999_percentile 512 2", "hist.99_percentile 512 2", "hist.min 1 2", "hist.95_percentile 512 2", "hist.75_percentile 160.0 2", "hist.std_dev 164.94851048466944 2" if PY3 else "hist.std_dev 164.948510485 2", "hist.max 512 2", "hist.avg 102.3 2", "m1.count 1.0 2", "m1.1m_rate 0 2", "m1.15m_rate 0 2", "m1.5m_rate 0 2", "m1.mean_rate 1.0 2", ] ) self.assertEqual(test_data, expected_data) def test_report_now_pickle(self): r = CarbonReporter( registry=self.registry, reporting_interval=1, clock=self.clock, socket_factory=lambda: self, pickle_protocol=True, ) self.capture_test_metrics() r.report_now() test_data = sorted(pickle.loads(self.output.getvalue()[4:])) expected_data = sorted( [ ("counter-2.count", (2, -2.0)), ("c1.count", (2, 1)), ("gsimple.value", (2, 42.0)), ("gcb.value", (2, 123.0)), ("t1.1m_rate", (2, 0.0)), ("t1.999_percentile", (2, 1)), ("t1.15m_rate", (2, 0.0)), ("t1.99_percentile", (2, 1)), ("t1.mean_rate", (2, 1)), ("t1.95_percentile", (2, 1)), ("t1.min", (2, 1)), ("t1.50_percentile", (2, 1)), ("t1.5m_rate", (2, 0.0)), ("t1.count", (2, 1)), ("t1.75_percentile", (2, 1)), ("t1.std_dev", (2, 0.0)), ("t1.max", (2, 1)), ("t1.sum", (2, 1)), ("t1.avg", (2, 1)), ("hist.count", (2, 10.0)), ("hist.999_percentile", (2, 512.0)), ("hist.99_percentile", (2, 512.0)), ("hist.min", (2, 1)), ("hist.95_percentile", (2, 512.0)), ("hist.75_percentile", (2, 160.0)), ("hist.std_dev", (2, 164.94851048466944)), ("hist.max", (2, 512.0)), ("hist.avg", (2, 102.3)), ("m1.count", (2, 1)), ("m1.1m_rate", (2, 0.0)), ("m1.15m_rate", (2, 0.0)), ("m1.5m_rate", (2, 0.0)), ("m1.mean_rate", (2, 1)), ] ) self.assertEqual(test_data, expected_data)
class ApptuitReporter(Reporter): """ Pyformance based reporter for Apptuit. It provides high level primitives, such as meter, counter, gauge, etc., for collecting data and reports them asynchronously to Apptuit. """ def __init__(self, registry=None, reporting_interval=10, token=None, api_endpoint="https://api.apptuit.ai", prefix="", tags=None, error_handler=default_error_handler, disable_host_tag=None, collect_process_metrics=False, sanitize_mode="prometheus"): """ Parameters ---------- registry: An instance of MetricsRegistry from pyformance. It is used as a container for all the metrics. If None, a new instance will be created internally reporting_interval: Reporting interval in seconds token: Apptuit API token prefix: Optional prefix for metric names, this will be prepended to all the metric names tags: A dictionary of tag keys and values which will be included with all the metrics reported by this reporter error_handler: A function to be executed in case of errors when reporting the data. If not specified, the default error handler will be used which by default writes the errors to stderr. The expected signature of an error handler is: error_handler(status_code, successful_points, failed_points, errors). Here status_code is the HTTP status code of the failed API call, successful_points is number of points processed successfully, failed_points is number of failed points and errors is a list of error messages describing reason of each failure. disable_host_tag: By default a host tag will be added to all the metrics reported by the reporter. Set disable_host_tag to False if you wish to disable it collect_process_metrics: A boolean variable specifying if process metrics should be collected or not, if set to True then process metrics will be collected. By default, this will collect resource, thread, and gc metrics. sanitize_mode: Is a string value which will enable sanitizer, sanitizer will automatically change your metric names to be compatible with apptuit or prometheus. Set it to None if not needed. """ super(ApptuitReporter, self).__init__(registry=registry, reporting_interval=reporting_interval) self.endpoint = api_endpoint self.token = token self.tags = tags environ_tags = _get_tags_from_environment() if environ_tags: if self.tags is not None: environ_tags.update(self.tags) self.tags = environ_tags if disable_host_tag is None: disable_host_tag = os.environ.get(DISABLE_HOST_TAG, False) if disable_host_tag: disable_host_tag = strtobool(disable_host_tag) if not disable_host_tag: if self.tags: if self.tags.get("host") is None: self.tags["host"] = socket.gethostname() else: self.tags = {"host": socket.gethostname()} self.prefix = prefix if prefix is not None else "" self.client = Apptuit(token=token, api_endpoint=api_endpoint, ignore_environ_tags=True, sanitize_mode=sanitize_mode) self._meta_metrics_registry = MetricsRegistry() self.error_handler = error_handler self.process_metrics = None if collect_process_metrics: self.process_metrics = ProcessMetrics(self.registry) def _update_counter(self, key, value): """ To increment the counter with `key` by `value`. :param key: Name of counter. :param value: value to increment. """ self._meta_metrics_registry.counter(key).inc(value) def report_now(self, registry=None, timestamp=None): """ Report the data Params: registry: pyformance Registry containing all metrics timestamp: timestamp of the data point """ if self.process_metrics: self.process_metrics.collect_process_metrics() dps = self._collect_data_points(registry or self.registry, timestamp) meta_dps = self._collect_data_points(self._meta_metrics_registry) if not dps: return dps_len = len(dps) success_count = 0 failed_count = 0 errors = [] for i in range(0, dps_len, BATCH_SIZE): try: with self._meta_metrics_registry.timer(API_CALL_TIMER).time(): end_index = min(dps_len, i + BATCH_SIZE) self.client.send(dps[i:end_index]) points_sent_count = end_index - i self._update_counter(NUMBER_OF_TOTAL_POINTS, points_sent_count) self._update_counter(NUMBER_OF_SUCCESSFUL_POINTS, points_sent_count) self._update_counter(NUMBER_OF_FAILED_POINTS, 0) success_count += points_sent_count except ApptuitSendException as exception: self._update_counter(NUMBER_OF_SUCCESSFUL_POINTS, exception.success) self._update_counter(NUMBER_OF_FAILED_POINTS, exception.failed) success_count += exception.success failed_count += exception.failed errors += exception.errors if self.error_handler: self.error_handler(exception.status_code, exception.success, exception.failed, exception.errors) self.client.send(meta_dps) if failed_count != 0: raise ApptuitSendException("Failed to send %d out of %d points" % (failed_count, dps_len), success=success_count, failed=failed_count, errors=errors) @staticmethod def _get_tags(key): """ Get tags of a metric Params: metric key Returns: metric name, dictionary of tags """ metric_name, metric_tags = TimeSeriesName.decode_metric(key) return metric_name, metric_tags def _collect_data_points(self, registry, timestamp=None): """ will collect all metrics from registry and convert them to DataPoints Params: registry: pyformance registry object timestamp: timestamp of the data point Returns: list of DataPoints """ timestamp = timestamp or int(round(self.clock.time())) metrics = registry.dump_metrics() dps = [] global_tags = self.tags if self.tags else {} for key in metrics.keys(): metric_name, metric_tags = self._get_tags(key) if metric_tags and global_tags: tags = global_tags.copy() tags.update(metric_tags) elif metric_tags: tags = metric_tags else: tags = global_tags for value_key in metrics[key].keys(): data_point = DataPoint(metric=self.prefix + metric_name + '.' + value_key, tags=tags, timestamp=timestamp, value=metrics[key][value_key]) dps.append(data_point) return dps def _loop(self): while not self._stopped.is_set(): start = time.time() try: self.report_now(self.registry) except Exception: pass end = time.time() wait_time = max(0, self.reporting_interval - (end - start)) time.sleep(wait_time)
class TestCarbonReporter(TimedTestCase): def setUp(self): super(TestCarbonReporter, self).setUp() self.output = StringIO() self.registry = MetricsRegistry(clock=self.clock) self.maxDiff = None def connect(self, *args): # part of fake socket interface pass def sendall(self, data): # part of fake socket interface self.output.write(data) def close(self): # part of fake socket interface pass def tearDown(self): super(TestCarbonReporter, self).tearDown() def test_report_now(self): r = CarbonReporter( registry=self.registry, reporting_interval=1, clock=self.clock, socket_factory=lambda: self) h1 = self.registry.histogram("hist") for i in range(10): h1.add(2 ** i) gcb = self.registry.gauge("gcb", lambda: 123) gsimple = self.registry.gauge("gsimple").set_value(42) t1 = self.registry.timer("t1") m1 = self.registry.meter("m1") m1.mark() with t1.time(): c1 = self.registry.counter("c1") c2 = self.registry.counter("counter-2") c1.inc() c2.dec() c2.dec() self.clock.add(1) r.report_now() self.assertEqual(self.output.getvalue().splitlines(), [ 'counter-2.count -2 1', 'gsimple.value 42 1', 'gcb.value 123 1', 't1.1m_rate 0 1', 't1.999_percentile 1 1', 't1.15m_rate 0 1', 't1.99_percentile 1 1', 't1.mean_rate 1.0 1', 't1.95_percentile 1 1', 't1.min 1 1', 't1.5m_rate 0 1', 't1.count 1.0 1', 't1.75_percentile 1 1', 't1.std_dev 0.0 1', 't1.max 1 1', 't1.avg 1.0 1', 'hist.count 10.0 1', 'hist.999_percentile 512 1', 'hist.99_percentile 512 1', 'hist.min 1 1', 'hist.95_percentile 512 1', 'hist.75_percentile 160.0 1', 'hist.std_dev 164.948510485 1', 'hist.max 512 1', 'hist.avg 102.3 1', 'm1.1m_rate 0 1', 'm1.15m_rate 0 1', 'm1.5m_rate 0 1', 'm1.mean_rate 1.0 1', 'c1.count 1 1' ])
class TestConsoleReporter(TimedTestCase): def setUp(self): super(TestConsoleReporter, self).setUp() self.output = StringIO() self.registry = MetricsRegistry(clock=self.clock) self.maxDiff = None self.clock.now = 0 def tearDown(self): super(TestConsoleReporter, self).tearDown() self.clock.now = 0 def test_report_now(self): r = ConsoleReporter( registry=self.registry, reporting_interval=1, stream=self.output, clock=self.clock) h1 = self.registry.histogram("hist") for i in range(10): h1.add(2 ** i) gcb = self.registry.gauge("gcb", lambda: 123) gsimple = self.registry.gauge("gsimple").set_value(42) t1 = self.registry.timer("t1") m1 = self.registry.meter("m1") m1.mark() with t1.time(): c1 = self.registry.counter("c1") c2 = self.registry.counter("counter-2") c1.inc() c2.dec() c2.dec() self.clock.add(1) r.report_now() self.assertEqual(self.output.getvalue().splitlines(), [ '== 1970-01-01 00:00:01 ===================================', 'counter-2:', ' count = -2', 'gsimple:', ' value = 42', 'gcb:', ' value = 123', 't1:', ' 1m_rate = 0', ' 999_percentile = 1', ' 15m_rate = 0', ' 99_percentile = 1', ' mean_rate = 1.0', ' 95_percentile = 1', ' min = 1', ' 5m_rate = 0', ' count = 1.0', ' 75_percentile = 1', ' std_dev = 0.0', ' max = 1', ' avg = 1.0', 'hist:', ' count = 10.0', ' 999_percentile = 512', ' 99_percentile = 512', ' min = 1', ' 95_percentile = 512', ' 75_percentile = 160.0', ' std_dev = 164.948510485', ' max = 512', ' avg = 102.3', 'm1:', ' 1m_rate = 0', ' 15m_rate = 0', ' 5m_rate = 0', ' mean_rate = 1.0', 'c1:', ' count = 1', ''])
class TestSysLogReporter(TimedTestCase): def setUp(self): super(TestSysLogReporter, self).setUp() self.registry = MetricsRegistry(clock=self.clock) self.maxDiff = None self.clock.now = 0 def tearDown(self): super(TestSysLogReporter, self).tearDown() self.clock.now = 0 def test_report_now(self): # connect to a local rsyslog server r = SysLogReporter(registry=self.registry, reporting_interval=1, clock=self.clock) h1 = self.registry.histogram("hist") for i in range(10): h1.add(2**i) gcb = self.registry.gauge("gcb", lambda: 123) gsimple = self.registry.gauge("gsimple").set_value(42) t1 = self.registry.timer("t1") m1 = self.registry.meter("m1") m1.mark() e1 = self.registry.event("e1") e1.add({"field": 1}) with t1.time(): c1 = self.registry.counter("c1") c2 = self.registry.counter("counter-2") c1.inc() c2.dec() c2.dec() self.clock.add(1) with mock.patch( "pyformance.reporters.syslog_reporter.logging.Logger.info" ) as patch: r.report_now() self.assertEqual(patch.call_count, 2) calls = patch.call_args_list expected = ( '{"c1.count": 1, "counter-2.count": -2, ' '"gcb.value": 123, "gsimple.value": 42, ' '"hist.75_percentile": 160.0, "hist.95_percentile": 512, ' '"hist.999_percentile": 512, "hist.99_percentile": 512, ' '"hist.avg": 102.3, "hist.count": 10.0, "hist.max": 512, ' '"hist.min": 1, "hist.std_dev": 164.94851048466944, ' '"m1.15m_rate": 0, "m1.1m_rate": 0, "m1.5m_rate": 0, "m1.count": 1.0, ' '"m1.mean_rate": 1.0, "t1.15m_rate": 0, "t1.1m_rate": 0, ' '"t1.50_percentile": 1, "t1.5m_rate": 0, "t1.75_percentile": 1, ' '"t1.95_percentile": 1, "t1.999_percentile": 1, "t1.99_percentile": 1, ' '"t1.avg": 1.0, "t1.count": 1.0, "t1.max": 1, "t1.mean_rate": 1.0, ' '"t1.min": 1, "t1.std_dev": 0.0, "t1.sum": 1.0, "timestamp": 1}' ) # First call should be event free because events are reported based on submission # rather than report timestamp. And a single line contains a single timestamp. self.assertEqual(calls[0][0][0], expected) # Second call should only be events self.assertEqual(calls[1][0][0], '{"e1.field": 1, "timestamp": 0}')
class TestConsoleReporter(TimedTestCase): def setUp(self): super(TestConsoleReporter, self).setUp() self.output = StringIO() self.registry = MetricsRegistry(clock=self.clock) self.maxDiff = None self.clock.now = 0 def tearDown(self): super(TestConsoleReporter, self).tearDown() self.clock.now = 0 def test_report_now(self): r = ConsoleReporter( registry=self.registry, reporting_interval=1, stream=self.output, clock=self.clock, ) h1 = self.registry.histogram("hist") for i in range(10): h1.add(2**i) gcb = self.registry.gauge("gcb", lambda: 123) gsimple = self.registry.gauge("gsimple").set_value(42) t1 = self.registry.timer("t1") m1 = self.registry.meter("m1") m1.mark() with t1.time(): c1 = self.registry.counter("c1") c2 = self.registry.counter("counter-2") c1.inc() c2.dec() c2.dec() self.clock.add(1) r.report_now() self.assertEqual( self.output.getvalue().splitlines().sort(), [ "== 1970-01-01 00:00:01 ===================================", "counter-2:", " count = -2", "gsimple:", " value = 42", "gcb:", " value = 123", "t1:", " 1m_rate = 0", " 999_percentile = 1", " 15m_rate = 0", " 99_percentile = 1", " mean_rate = 1.0", " 95_percentile = 1", " min = 1", " 5m_rate = 0", " count = 1.0", " 75_percentile = 1", " std_dev = 0.0", " max = 1", " avg = 1.0", "hist:", " count = 10.0", " 999_percentile = 512", " 99_percentile = 512", " min = 1", " 95_percentile = 512", " 75_percentile = 160.0", " std_dev = 164.94851048466944", " max = 512", " avg = 102.3", "m1:", " count = 1.0", " 1m_rate = 0", " 15m_rate = 0", " 5m_rate = 0", " mean_rate = 1.0", "c1:", " count = 1", "", ].sort(), )
class TestNewRelicReporter(TimedTestCase): def setUp(self): super(TestNewRelicReporter, self).setUp() self.registry = MetricsRegistry(clock=self.clock, sink=NewRelicSink) self.maxDiff = None def tearDown(self): super(TestNewRelicReporter, self).tearDown() def test_report_now(self): r = NewRelicReporter('license_key', registry=self.registry, reporting_interval=1, clock=self.clock, name='foo') h1 = self.registry.histogram("hist", 'a/b') for i in range(10): h1.add(2**i) t1 = self.registry.timer("t1") gauge = self.registry.gauge('g1', SimpleGauge(unit='g')) gauge_value = 10 gauge.set_value(gauge_value) m = self.registry.meter('m1', 'u1') m.mark() with t1.time(): m.mark() c1 = self.registry.counter("counter-1", 'c') c2 = self.registry.counter("counter-2", 'c') c1.inc() c2.dec() c2.dec() self.clock.add(1) m.mark() output = json.loads(r.collect_metrics(self.registry)) expected = { "agent": { "host": socket.gethostname(), "pid": os.getpid(), "version": "0.3.3" }, "components": [{ "duration": 1, "guid": "com.github.pyformance", "metrics": { "Component/hist/raw": { "max": 512, "total": 1023, "min": 1, "count": 10, "sum_of_squares": 349525 }, "Component/t1/95_percentile": 1., "Component/hist/999_percentile": 512, "Component/counter-2/raw": { "total": -3, "max": -1, "count": 2, "sum_of_squares": 5, "min": -2 }, "Component/t1/mean_rate": { "max": 1, "count": 1, "total": 1., "min": 1, "sum_of_squares": 1. }, "Component/t1/999_percentile": 1., "Component/m1/1m_rate": { "min": 0, "total": 0, "max": 0, "sum_of_squares": 0, "count": 0 }, "Component/t1/15m_rate": { "total": 0, "min": 0, "max": 0, "sum_of_squares": 0, "count": 0 }, "Component/hist/99_percentile": 512, "Component/t1/raw": { "min": 1., "count": 1, "sum_of_squares": 1., "max": 1., "total": 1. }, "Component/m1/mean_rate": { "sum_of_squares": 3., "count": 3, "total": 3., "min": 1, "max": 1 }, "Component/hist/std_dev": 164.94851048466947, "Component/counter-1/raw": { "count": 1, "sum_of_squares": 1, "min": 1, "max": 1, "total": 1 }, "Component/t1/50_percentile": 1., "Component/t1/99_percentile": 1., "Component/hist/95_percentile": 512, "Component/m1/15m_rate": { "count": 0, "sum_of_squares": 0, "max": 0, "min": 0, "total": 0 }, "Component/hist/75_percentile": 160., "Component/t1/5m_rate": { "count": 0, "min": 0, "total": 0, "sum_of_squares": 0, "max": 0 }, "Component/hist/mean_rate": { "count": 103, "max": 1, "sum_of_squares": 102.09, "min": 0.29999999999999716, "total": 102.3 }, "Component/t1/count": 1., "Component/g1/gauge": 10, "Component/t1/1m_rate": { "count": 0, "total": 0, "min": 0, "max": 0, "sum_of_squares": 0 }, "Component/t1/75_percentile": 1., "Component/t1/std_dev": 0., "Component/m1/raw": { "min": 1, "sum_of_squares": 3, "count": 3, "total": 3, "max": 1 }, "Component/m1/5m_rate": { "count": 0, "min": 0, "total": 0, "sum_of_squares": 0, "max": 0 } }, "name": "foo" }] } self.assertEqual(json.loads(json.dumps(expected)), json.loads(json.dumps(output)))