def test_publish_metrics(): publisher1 = mock.MagicMock() publisher2 = mock.MagicMock() counter1 = Counter('') counter2 = Counter('') timer1 = Timer('') timer2 = Timer('') config = Configuration(2, [ cast(MetricsPublisher, publisher1), cast(MetricsPublisher, publisher2) ]) publish_metrics([counter1, timer1, counter2], config) publisher1.publish.assert_called_once_with([counter1, timer1, counter2], None, False) publisher2.publish.assert_called_once_with([counter1, timer1, counter2], None, False) publisher1.reset_mock() publisher2.reset_mock() config = Configuration(2, [cast(MetricsPublisher, publisher2)], 't_py_log', True) publish_metrics([timer2, timer1, counter1], config) assert publisher1.publish.call_count == 0 publisher2.publish.assert_called_once_with([timer2, timer1, counter1], 't_py_log', True)
def test_recognized_exception_no_logger(self): metrics = [ Counter('foo.count', initial_value=5), Gauge('bar.gauge', initial_value=3), Histogram('baz.hist', initial_value=6), Timer('qux.time1', initial_value=15), Timer('qux.time2', initial_value=723, resolution=TimerResolution.MICROSECONDS), ] publisher = MockPublisher(E2, E2) publisher.publish(metrics) publisher.mock_execute.assert_has_calls( [ mock.call( 'INSERT INTO pymetrics_counters (metric_name, metric_value) VALUES (?, ?);', {('foo.count', 5)}, ), mock.call( 'REPLACE INTO pymetrics_gauges (metric_name, metric_value) VALUES (?, ?);', {('bar.gauge', 3)}, ), mock.call( 'INSERT INTO pymetrics_histograms (metric_name, metric_value) VALUES (?, ?);', {('baz.hist', 6)}, ), mock.call( 'INSERT INTO pymetrics_timers (metric_name, metric_value) VALUES (?, ?);', {('qux.time1', 0.015), ('qux.time2', 0.000723)}, ), ], any_order=True, )
def test_invalid_name(): with pytest.raises(TypeError) as error_context: Timer(None) # type: ignore assert error_context.value.args[0] == 'Metric names must be non-null strings' with pytest.raises(TypeError) as error_context: Timer(1) # type: ignore assert error_context.value.args[0] == 'Metric names must be non-null strings'
def test_invalid_initial_value(): with pytest.raises(TypeError) as error_context: Timer('Hello', None) # type: ignore assert error_context.value.args[0] == 'Metric values must be integers or floats' with pytest.raises(TypeError) as error_context: Timer('Hello', 'Not an int') # type: ignore assert error_context.value.args[0] == 'Metric values must be integers or floats'
def test_with_only_instrument_tags(self): counter = Counter('test.foo.timer.1', hello='world') counter.increment() gauge = Gauge('test.bar.gauge.1', extra='data', nothing=None, mail='snail', guitar='electric') gauge.set(5) timer = Timer('test.baz.timer.1', initial_value=2, number=5791) publisher = DogStatsdPublisher('localhost', 1234) metrics = publisher.get_formatted_metrics([counter, gauge, timer]) assert metrics[0] == b'test.foo.timer.1:1|c|#hello:world' assert metrics[2] == b'test.baz.timer.1:2|ms|#number:5791' assert metrics[1].startswith(b'test.bar.gauge.1:5|g|#') assert b'extra:data' in metrics[1] assert b'nothing' in metrics[1] assert b'nothing:' not in metrics[1] assert b'mail:snail' in metrics[1] assert b'guitar:electric' in metrics[1]
def test_no_metric_values_does_nothing(self): publisher = SqlitePublisher() publisher.publish([Timer(u'hello')]) connection = SqlitePublisher.get_connection() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute("SELECT * FROM pymetrics_counters;") assert len(list(cursor.fetchall())) == 0 finally: cursor.close() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute("SELECT * FROM pymetrics_gauges;") assert len(list(cursor.fetchall())) == 0 finally: cursor.close() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute("SELECT * FROM pymetrics_histograms;") assert len(list(cursor.fetchall())) == 0 finally: cursor.close() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute("SELECT * FROM pymetrics_timers;") assert len(list(cursor.fetchall())) == 0 finally: cursor.close()
def test_no_metric_values_does_nothing(self, mock_get_logger): publisher = LogPublisher(u'custom_metrics') mock_get_logger.assert_called_once_with(u'custom_metrics') assert publisher.log_level == logging.INFO publisher.publish([Timer(u'hello')]) assert mock_get_logger.return_value.log.call_count == 0
def get_formatted_metrics(self, metrics, enable_meta_metrics=False): # type: (Iterable[Metric], bool) -> List[six.binary_type] meta_timer = None if enable_meta_metrics: meta_timer = Timer('', resolution=TimerResolution.MICROSECONDS) formatted_metrics = [] for metric in metrics: if metric.value is None: continue existing_tags_string = self._global_tags_string if isinstance(metric, Counter): type_label = self.METRIC_TYPE_COUNTER elif isinstance(metric, Gauge): type_label = self.METRIC_TYPE_GAUGE existing_tags_string = self._global_gauge_tags_string elif isinstance(metric, Timer): type_label = self._metric_type_timer elif isinstance(metric, Histogram): type_label = self._metric_type_histogram else: continue metric_tags_string = self._generate_tag_string( metric.tags, existing_tags_string) formatted_metrics.append(b'%s:%d|%s%s' % (self._get_binary_value( metric.name), metric.value, type_label, metric_tags_string)) if not formatted_metrics: return [] if meta_timer: meta_timer.stop() formatted_metrics.insert( 0, b'pymetrics.meta.publish.statsd.format_metrics:%d|%s%s' % ( cast(int, meta_timer.value), self._metric_type_timer, self._global_tags_string, ), ) return formatted_metrics
def timer(self, name, force_new=False, resolution=TimerResolution.MILLISECONDS, initial_value=0, **tags): # type: (six.text_type, bool, TimerResolution, int, **Tag) -> Timer return Timer(name, initial_value, resolution, **tags)
def test_unrecognized_exception(self): metrics = [ Counter('foo.count', initial_value=5), Gauge('bar.gauge', initial_value=3), Histogram('baz.hist', initial_value=6), Timer('qux.time', initial_value=1), ] publisher = MockPublisher(E2, E1) with pytest.raises(E1): publisher.publish(metrics)
def get_formatted_metrics(self, metrics, enable_meta_metrics=False): # type: (Iterable[Metric], bool) -> List[six.binary_type] meta_timer = None if enable_meta_metrics: meta_timer = Timer('', resolution=TimerResolution.MICROSECONDS) formatted_metrics = [] for metric in metrics: if metric.value is None: continue if isinstance(metric, Counter): type_label = self.METRIC_TYPE_COUNTER elif isinstance(metric, Gauge): type_label = self.METRIC_TYPE_GAUGE elif isinstance(metric, Timer): type_label = self._metric_type_timer elif isinstance(metric, Histogram): type_label = self._metric_type_histogram else: continue # not possible unless a new metric type is added formatted_metrics.append( b'%s:%d|%s' % (self._get_binary_value(metric.name), metric.value, type_label) ) if not formatted_metrics: return [] if meta_timer: meta_timer.stop() formatted_metrics.insert( 0, b'pymetrics.meta.publish.statsd.format_metrics:%d|%s' % ( cast(int, meta_timer.value), self._metric_type_timer, ) ) return formatted_metrics
def get_all_metrics(self): # type: () -> List[Metric] meta_timer = None if self._configuration and self._configuration.enable_meta_metrics is True: meta_timer = Timer('pymetrics.meta.recorder.get_all_metrics', resolution=TimerResolution.MICROSECONDS) metrics = [] # type: List[Metric] metrics.extend(six.itervalues(self.counters)) metrics.extend(gauge for gauges in six.itervalues(self.gauges) for gauge in gauges if gauge.value is not None) metrics.extend(histogram for histograms in six.itervalues(self.histograms) for histogram in histograms if histogram.value is not None) metrics.extend(timer for timers in six.itervalues(self.timers) for timer in timers if timer.value is not None) if meta_timer: meta_timer.stop() metrics.insert(0, meta_timer) return metrics
def test_with_global_and_instrument_tags(self): counter = Counter('test.foo.timer.1', hello='world') counter.increment() gauge = Gauge('test.bar.gauge.1', extra='data', nothing=None, mail='snail', guitar='electric') gauge.set(5) timer = Timer('test.baz.timer.1', initial_value=2, number=5791.15, other_number=0) publisher = DogStatsdPublisher( 'localhost', 1234, global_tags=OrderedDict([('environment', 'qa'), ('acceptance', None), ('jenkins-build', 8293847)]), extra_gauge_tags={'worker': '52'}, ) metrics = publisher.get_formatted_metrics([counter, gauge, timer], enable_meta_metrics=True) assert metrics[0].startswith( b'pymetrics.meta.publish.statsd.format_metrics:') assert metrics[0].endswith( b'|ms|#environment:qa,acceptance,jenkins-build:8293847') assert metrics[ 1] == b'test.foo.timer.1:1|c|#environment:qa,acceptance,jenkins-build:8293847,hello:world' assert metrics[2].startswith( b'test.bar.gauge.1:5|g|#environment:qa,acceptance,jenkins-build:8293847,worker:52,' ) assert b',extra:data' in metrics[2] assert b',nothing' in metrics[2] assert b',nothing:' not in metrics[2] assert b',mail:snail' in metrics[2] assert b',guitar:electric' in metrics[2] assert metrics[3].startswith( b'test.baz.timer.1:2|ms|#environment:qa,acceptance,jenkins-build:8293847' ) assert b',number:5791.15' in metrics[3] assert b',other_number:0' in metrics[3]
def test_recognized_exception_with_logger(self): metrics = [ Counter('foo.count1', initial_value=1), Counter('foo.count2', initial_value=17), Gauge('bar.gauge1', initial_value=2), Gauge('bar.gauge2', initial_value=1), Histogram('baz.hist1', initial_value=33), Histogram('baz.hist2', initial_value=39), Timer('qux.time1', initial_value=21), Timer('qux.time2', initial_value=1837, resolution=TimerResolution.MICROSECONDS), ] publisher = MockPublisher(E1, E1) publisher.publish(metrics, error_logger='pymetrics') publisher.mock_execute.assert_has_calls( [ mock.call( 'INSERT INTO pymetrics_counters (metric_name, metric_value) VALUES (?, ?);', {('foo.count1', 1), ('foo.count2', 17)}, ), mock.call( 'REPLACE INTO pymetrics_gauges (metric_name, metric_value) VALUES (?, ?);', {('bar.gauge1', 2), ('bar.gauge2', 1)}, ), mock.call( 'INSERT INTO pymetrics_histograms (metric_name, metric_value) VALUES (?, ?);', {('baz.hist1', 33), ('baz.hist2', 39)}, ), mock.call( 'INSERT INTO pymetrics_timers (metric_name, metric_value) VALUES (?, ?);', {('qux.time1', 0.021), ('qux.time2', 0.001837)}, ), ], any_order=True, )
def test_with_one_global_tag_with_value(self): counter = Counter('test.foo.timer.1') counter.increment() gauge = Gauge('test.bar.gauge.1') gauge.set(5) timer = Timer('test.baz.timer.1', initial_value=2) publisher = DogStatsdPublisher('localhost', 1234, global_tags={'integration': 'abc123'}) assert (publisher.get_formatted_metrics([counter, gauge, timer]) == [ b'test.foo.timer.1:1|c|#integration:abc123', b'test.bar.gauge.1:5|g|#integration:abc123', b'test.baz.timer.1:2|ms|#integration:abc123', ])
def test_with_only_extra_gauge_tags(self): counter = Counter('test.foo.timer.1') counter.increment() gauge = Gauge('test.bar.gauge.1') gauge.set(5) timer = Timer('test.baz.timer.1', initial_value=2) publisher = DogStatsdPublisher('localhost', 1234, extra_gauge_tags={'worker': '456def'}) assert (publisher.get_formatted_metrics([counter, gauge, timer]) == [ b'test.foo.timer.1:1|c', b'test.bar.gauge.1:5|g|#worker:456def', b'test.baz.timer.1:2|ms', ])
def test_metrics(self, mock_get_logger): publisher = LogPublisher(u'py_metrics', logging.DEBUG) assert publisher.log_level == logging.DEBUG mock_get_logger.assert_called_once_with(u'py_metrics') publisher.publish([ Timer(u'hello.foo', initial_value=1), Counter(u'hello.bar', initial_value=2), Histogram(u'goodbye.baz', initial_value=3, neat_tag=u'production', other_tag=b'binary'), Gauge(u'goodbye.qux', initial_value=4), ]) mock_get_logger.return_value.log.assert_called_once_with( logging.DEBUG, u'counters.hello.bar 2; ' u'gauges.goodbye.qux 4; ' u'histograms.goodbye.baz{neat_tag:production,other_tag:binary} 3; ' u'timers.hello.foo 1', )
def test_no_metric_values_does_nothing(self, _mock_logging): port = self._start_udp_socket() publisher = StatsdPublisher('127.0.0.1', port) publisher.publish([Timer(u'hello')]) self.all_received = True assert self.sock is not None # noinspection PyBroadException try: bad = self.sock.recv(4096) raise AssertionError( 'Did not expect to receive any data, but received: {}'.format( bad)) except AssertionError: raise except Exception: pass # this is a good thing
def test_with_no_tags(self): counter = Counter('test.foo.timer.1') counter.increment() gauge = Gauge('test.bar.gauge.1') gauge.set(5) timer = Timer('test.baz.timer.1', initial_value=2) histogram = Histogram('test.qux.histogram.1') histogram.set(13) publisher = DogStatsdPublisher('localhost', 1234) assert (publisher.get_formatted_metrics( [counter, gauge, timer, histogram]) == [ b'test.foo.timer.1:1|c', b'test.bar.gauge.1:5|g', b'test.baz.timer.1:2|ms', b'test.qux.histogram.1:13|h' ])
def test_bytes(self, mock_logging): """ Test that byte-string metric names work properly This function intentionally uses byte-strings for every metric it records, to ensure they get recorded properly. On Python 2, this test is actually different from `test_unicode`. On Python 3, they are identical (except the port number and gauge value). """ port = self._start_udp_socket() metrics = [ Counter('test_bytes.counter', initial_value=1), Gauge('test_bytes.gauge', initial_value=17), Histogram('test_bytes.histogram', initial_value=3), Timer('test_bytes.timer', initial_value=1), ] # type: List[Metric] self.all_received = False publisher = StatsdPublisher('localhost', port) publisher.publish(metrics) # We want to make sure that no logging was called at all # We test it this way so that any unexpected calls are printed to the output if mock_logging.getLogger.return_value.error.called: raise AssertionError( 'No errors should have been logged. Instead got: {}'.format( mock_logging.getLogger.return_value.error.call_args_list)) self.all_received = True assert self.sock is not None received = self.sock.recv(2048) assert received is not None assert received == ( b'test_bytes.counter:1|c\ntest_bytes.gauge:17|g\n' b'test_bytes.histogram:3|ms\ntest_bytes.timer:1|ms')
def test_with_multiple_global_tags(self): counter = Counter('test.foo.timer.1') counter.increment() gauge = Gauge('test.bar.gauge.1') gauge.set(5) timer = Timer('test.baz.timer.1', initial_value=2) publisher = DogStatsdPublisher( 'localhost', 1234, global_tags=OrderedDict([('environment', 'qa'), ('acceptance', None), ('jenkins-build', 8293847)]), ) assert (publisher.get_formatted_metrics([counter, gauge, timer]) == [ b'test.foo.timer.1:1|c|#environment:qa,acceptance,jenkins-build:8293847', b'test.bar.gauge.1:5|g|#environment:qa,acceptance,jenkins-build:8293847', b'test.baz.timer.1:2|ms|#environment:qa,acceptance,jenkins-build:8293847', ])
def test_unicode(self, mock_logging): """ Test that unicode metric names work properly This function intentionally uses unicode strings for every metric it records, to ensure that they get recorded properly. It also uses unicode strings for the config, to make sure it works properly. _Everything_ in this method must use unicode literals. """ port = self._start_udp_socket() metrics = [ Counter(u'test_unicode.counter', initial_value=1), Gauge(u'test_unicode.gauge', initial_value=42), Histogram(u'test_unicode.histogram', initial_value=6), Timer(u'test_unicode.timer', initial_value=1), ] # type: List[Metric] self.all_received = False publisher = StatsdPublisher('localhost', port) publisher.publish(metrics) # We want to make sure that no logging was called at all # We test it this way so that any unexpected calls are printed to the output if mock_logging.getLogger.return_value.error.called: raise AssertionError( 'No errors should have been logged. Instead got: {}'.format( mock_logging.getLogger.return_value.error.call_args_list)) self.all_received = True assert self.sock is not None received = self.sock.recv(2048) assert received is not None assert received == ( b'test_unicode.counter:1|c\ntest_unicode.gauge:42|g\ntest_unicode.histogram:6|ms\ntest_unicode.timer:1|ms' )
def test_metrics(self): p_metrics = [ self._counter('foo.bar'), self._counter('baz.qux'), Gauge('a.b', initial_value=4), Gauge('a.b', initial_value=7), Gauge('a.b', initial_value=5), Gauge('c.d', initial_value=3), ] # type: List[Metric] timer = Timer('one.two') timer.stop() p_metrics.append(timer) with freezegun.freeze_time() as frozen_time: timer = Timer('one.two') frozen_time.tick(self._milliseconds(50)) timer.stop() p_metrics.append(timer) timer = Timer('one.two') frozen_time.tick(self._milliseconds(30)) timer.stop() p_metrics.append(timer) timer = Timer('three.four') frozen_time.tick(self._milliseconds(10)) timer.stop() p_metrics.append(timer) p_metrics.extend([ Histogram('h.foo', initial_value=1), Histogram('h.foo', initial_value=3), Histogram('h.foo', initial_value=2), Histogram('h.bar', initial_value=77), ]) publisher = SqlitePublisher() publisher.publish(p_metrics) connection = SqlitePublisher.get_connection() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute( "SELECT * FROM pymetrics_counters WHERE metric_name = 'foo.bar';" ) metrics = list(cursor.fetchall()) assert len(metrics) == 1 assert metrics[0][str('metric_name')] == 'foo.bar' assert metrics[0][str('metric_value')] == 1 finally: cursor.close() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute( "SELECT * FROM pymetrics_counters WHERE metric_name = 'baz.qux';" ) metrics = list(cursor.fetchall()) assert len(metrics) == 1 assert metrics[0][str('metric_name')] == 'baz.qux' assert metrics[0][str('metric_value')] == 1 finally: cursor.close() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute( "SELECT * FROM pymetrics_gauges WHERE metric_name = 'a.b';") metrics = list(cursor.fetchall()) assert len(metrics) == 1 assert metrics[0][str('metric_name')] == 'a.b' assert metrics[0][str('metric_value')] == 5 finally: cursor.close() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute( "SELECT * FROM pymetrics_gauges WHERE metric_name = 'c.d';") metrics = list(cursor.fetchall()) assert len(metrics) == 1 assert metrics[0][str('metric_name')] == 'c.d' assert metrics[0][str('metric_value')] == 3 finally: cursor.close() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute( "SELECT * FROM pymetrics_timers WHERE metric_name = 'one.two';" ) metrics = list(cursor.fetchall()) assert len(metrics) == 3 assert metrics[0][str('metric_name')] == 'one.two' assert metrics[0][str('metric_value')] < 0.001 assert metrics[1][str('metric_name')] == 'one.two' assert 0.05 <= metrics[1][str('metric_value')] < 0.06 assert metrics[2][str('metric_name')] == 'one.two' assert 0.03 <= metrics[2][str('metric_value')] < 0.04 finally: cursor.close() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute( "SELECT * FROM pymetrics_timers WHERE metric_name = 'three.four';" ) metrics = list(cursor.fetchall()) assert len(metrics) == 1 assert metrics[0][str('metric_name')] == 'three.four' assert 0.01 <= metrics[0][str('metric_value')] < 0.02 finally: cursor.close() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute( "SELECT * FROM pymetrics_histograms WHERE metric_name = 'h.foo';" ) metrics = list(cursor.fetchall()) assert len(metrics) == 3 assert metrics[0][str('metric_name')] == 'h.foo' assert metrics[0][str('metric_value')] == 1 assert metrics[1][str('metric_name')] == 'h.foo' assert metrics[1][str('metric_value')] == 3 assert metrics[2][str('metric_name')] == 'h.foo' assert metrics[2][str('metric_value')] == 2 finally: cursor.close() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute( "SELECT * FROM pymetrics_histograms WHERE metric_name = 'h.bar';" ) metrics = list(cursor.fetchall()) assert len(metrics) == 1 assert metrics[0][str('metric_name')] == 'h.bar' assert metrics[0][str('metric_value')] == 77 finally: cursor.close() p_metrics = [ self._counter('foo.bar'), Gauge('a.b', initial_value=22), Gauge('c.d', initial_value=11), ] with freezegun.freeze_time() as frozen_time: timer = Timer('three.four', resolution=TimerResolution.MICROSECONDS) frozen_time.tick(self._milliseconds(20)) timer.stop() p_metrics.append(timer) p_metrics.append(Histogram('h.foo', initial_value=8)) publisher = SqlitePublisher() publisher.initialize_if_necessary() publisher.publish(p_metrics) cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute( "SELECT * FROM pymetrics_counters WHERE metric_name = 'foo.bar';" ) metrics = list(cursor.fetchall()) assert len(metrics) == 2 assert metrics[0][str('metric_name')] == 'foo.bar' assert metrics[0][str('metric_value')] == 1 assert metrics[1][str('metric_name')] == 'foo.bar' assert metrics[1][str('metric_value')] == 1 finally: cursor.close() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute( "SELECT * FROM pymetrics_counters WHERE metric_name = 'baz.qux';" ) metrics = list(cursor.fetchall()) assert len(metrics) == 1 assert metrics[0][str('metric_name')] == 'baz.qux' assert metrics[0][str('metric_value')] == 1 finally: cursor.close() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute( "SELECT * FROM pymetrics_gauges WHERE metric_name = 'a.b';") metrics = list(cursor.fetchall()) assert len(metrics) == 1 assert metrics[0][str('metric_name')] == 'a.b' assert metrics[0][str('metric_value')] == 22 finally: cursor.close() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute( "SELECT * FROM pymetrics_gauges WHERE metric_name = 'c.d';") metrics = list(cursor.fetchall()) assert len(metrics) == 1 assert metrics[0][str('metric_name')] == 'c.d' assert metrics[0][str('metric_value')] == 11 finally: cursor.close() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute( "SELECT * FROM pymetrics_timers WHERE metric_name = 'one.two';" ) metrics = list(cursor.fetchall()) assert len(metrics) == 3 assert metrics[0][str('metric_name')] == 'one.two' assert metrics[0][str('metric_value')] < 0.001 assert metrics[1][str('metric_name')] == 'one.two' assert 0.05 <= metrics[1][str('metric_value')] < 0.06 assert metrics[2][str('metric_name')] == 'one.two' assert 0.03 <= metrics[2][str('metric_value')] < 0.04 finally: cursor.close() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute( "SELECT * FROM pymetrics_timers WHERE metric_name = 'three.four';" ) metrics = list(cursor.fetchall()) assert len(metrics) == 2 assert metrics[0][str('metric_name')] == 'three.four' assert 0.01 <= metrics[0][str('metric_value')] < 0.02 assert metrics[1][str('metric_name')] == 'three.four' assert 0.02 <= metrics[1][str('metric_value')] < 0.03 finally: cursor.close() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute( "SELECT * FROM pymetrics_histograms WHERE metric_name = 'h.foo';" ) metrics = list(cursor.fetchall()) assert len(metrics) == 4 assert metrics[0][str('metric_name')] == 'h.foo' assert metrics[0][str('metric_value')] == 1 assert metrics[1][str('metric_name')] == 'h.foo' assert metrics[1][str('metric_value')] == 3 assert metrics[2][str('metric_name')] == 'h.foo' assert metrics[2][str('metric_value')] == 2 assert metrics[3][str('metric_name')] == 'h.foo' assert metrics[3][str('metric_value')] == 8 finally: cursor.close() cursor = connection.cursor() try: # noinspection PyTypeChecker cursor.execute( "SELECT * FROM pymetrics_histograms WHERE metric_name = 'h.bar';" ) metrics = list(cursor.fetchall()) assert len(metrics) == 1 assert metrics[0][str('metric_name')] == 'h.bar' assert metrics[0][str('metric_value')] == 77 finally: cursor.close()
def test_meta_metrics(self, mock_logging): """ Test that meta metrics work properly This test confirms that, when enabled, meta-metrics are sent, informing about the performance of PyMetrics itself. """ port = self._start_udp_socket() metrics = [ Counter(u'test_meta_metrics.counter', initial_value=1), Gauge(u'test_meta_metrics.gauge', initial_value=9), Histogram(u'test_meta_metrics.histogram', initial_value=27), Timer(u'test_meta_metrics.timer', initial_value=1), ] # type: List[Metric] self.all_received = False publisher = StatsdPublisher('localhost', port) publisher.publish(metrics, enable_meta_metrics=True) # We want to make sure that no logging was called at all # We test it this way so that any unexpected calls are printed to the output if mock_logging.getLogger.return_value.error.called: raise AssertionError( 'No errors should have been logged. Instead got: {}'.format( mock_logging.getLogger.return_value.error.call_args_list)) assert self.sock is not None received = self.sock.recv(2048) assert received is not None msg = received.decode('utf-8') received_regex = re.compile( br'^pymetrics\.meta\.publish\.statsd\.format_metrics:[0-9]+\|ms\n' br'test_meta_metrics\.counter:1\|c\n' br'test_meta_metrics\.gauge:9\|g\n' br'test_meta_metrics\.histogram:27|ms\n' br'test_meta_metrics\.timer:0\|ms$') assert received_regex.match(received), msg self.all_received = True received = self.sock.recv(2048) assert received is not None assert re.compile(br'^pymetrics\.meta\.publish\.statsd\.send:1\|c$', re.MULTILINE).search(received), msg assert re.compile(br'^pymetrics\.meta\.publish\.statsd\.send\.num_metrics:5\|ms$', re.MULTILINE)\ .search(received), msg assert re.compile(br'^pymetrics\.meta\.publish\.statsd\.send\.timer:[0-9]+\|ms$', re.MULTILINE)\ .search(received), msg assert not re.compile(br'^pymetrics\.meta\.publish\.statsd\.send\.error\.max_packet:1\|c$', re.MULTILINE)\ .search(received), msg assert not re.compile(br'^pymetrics\.meta\.publish\.statsd\.send\.error\.unknown:1\|c$', re.MULTILINE)\ .search(received), msg assert not re.compile(br'^pymetrics\.meta\.publish\.statsd\.send\.exceeds_max_packet:1\|c$', re.MULTILINE)\ .search(received), msg assert not re.compile(br'^pymetrics\.meta\.publish\.statsd\.send\.exceeds_max_gig_e:1\|c$', re.MULTILINE)\ .search(received), msg assert not re.compile(br'^pymetrics\.meta\.publish\.statsd\.send\.exceeds_max_fast_e:1\|c$', re.MULTILINE)\ .search(received), msg
def _send_chunked_payload(self, payload, number_of_metrics, error_logger=None, enable_meta_metrics=False): # type: (six.binary_type, int, six.text_type, bool) -> None meta_timer = None error = error_max_packet = False sock = None try: if enable_meta_metrics: meta_timer = Timer('', resolution=TimerResolution.MICROSECONDS) sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.settimeout(self.timeout) sock.connect((self.host, self.port)) sock.sendall(payload) except Exception as e: error = True if isinstance(e, socket.error) and e.errno == errno.EMSGSIZE: error_max_packet = True if error_logger: extra = {'data': { 'payload_length': len(payload), 'num_metrics': number_of_metrics, 'enable_meta_metrics': enable_meta_metrics, }} if error_max_packet: logging.getLogger(error_logger).error( 'Failed to send metrics to statsd because UDP packet too big', extra=extra, ) else: logging.getLogger(error_logger).exception( 'Failed to send metrics to statsd {}:{}'.format(self.host, self.port), extra=extra, ) finally: if sock: # noinspection PyBroadException try: sock.close() except Exception: pass if meta_timer: meta_timer.stop() if enable_meta_metrics: num_bytes = len(payload) # TODO temporary; the length of the packet that we tried to send payload = b'pymetrics.meta.publish.statsd.send:1|%s' % self.METRIC_TYPE_COUNTER payload += b'\npymetrics.meta.publish.statsd.send.num_metrics:%d|%s' % ( number_of_metrics, self._metric_type_histogram ) if meta_timer: payload += b'\npymetrics.meta.publish.statsd.send.timer:%d|%s' % ( cast(int, meta_timer.value), self._metric_type_timer, ) if error: if error_max_packet: payload += b'\npymetrics.meta.publish.statsd.send.error.max_packet:1|%s' % self.METRIC_TYPE_COUNTER else: payload += b'\npymetrics.meta.publish.statsd.send.error.unknown:1|%s' % self.METRIC_TYPE_COUNTER # TODO The following three stats are temporary, to test out potential MTU problems noted above if num_bytes >= MAX_IPV4_PAYLOAD_SIZE_BYTES: payload += b'\npymetrics.meta.publish.statsd.send.exceeds_max_packet:1|%s' % self.METRIC_TYPE_COUNTER if num_bytes >= MAX_GIG_E_PAYLOAD_SIZE_BYTES: payload += b'\npymetrics.meta.publish.statsd.send.exceeds_max_gig_e:1|%s' % self.METRIC_TYPE_COUNTER if num_bytes >= MAX_FAST_E_PAYLOAD_SIZE_BYTES: payload += b'\npymetrics.meta.publish.statsd.send.exceeds_max_fast_e:1|%s' % self.METRIC_TYPE_COUNTER sock = None # noinspection PyBroadException try: sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock.settimeout(self.timeout) sock.connect((self.host, self.port)) sock.sendall(payload) except Exception: if error_logger: logging.getLogger(error_logger).exception( 'Failed to send meta metrics to statsd {}:{}'.format(self.host, self.port), ) finally: if sock: # noinspection PyBroadException try: sock.close() except Exception: pass
def reset(self): self.recorder.reset_mock() self.recorder.counter.return_value = Counter('') self.recorder.timer.return_value = Timer('')
def test_timer(): timer = Timer('test.timer.1', tag_4='value_4') assert timer.name == 'test.timer.1' assert timer.tags['tag_4'] == 'value_4' assert timer.value is None timer = Timer('test.timer.2', initial_value=3.75) assert timer.name == 'test.timer.2' assert not timer.tags assert timer.value == 4 with freezegun.freeze_time() as frozen_time: timer.start() frozen_time.tick(_milliseconds(15)) timer.stop() assert timer.value == 15 timer.start() frozen_time.tick(_milliseconds(27)) timer.stop() assert timer.value == 42 with timer: frozen_time.tick(_milliseconds(8)) frozen_time.tick(_microseconds(100)) assert timer.value == 50 def around(foo, bar): assert foo == 'baz' assert bar == 'qux' frozen_time.tick(_milliseconds(11)) frozen_time.tick(_microseconds(600)) return 'Ipsum' assert timer.record_over_function(around, 'baz', bar='qux') == 'Ipsum' assert timer.value == 62 timer = Timer('test.timer.3', resolution=TimerResolution.MICROSECONDS) with freezegun.freeze_time() as frozen_time: timer.start() frozen_time.tick(_milliseconds(1)) frozen_time.tick(_microseconds(103)) timer.stop() assert timer.value == 1103 with timer: frozen_time.tick(_microseconds(209)) assert timer.value == 1312 timer.start() assert timer.value is None frozen_time.tick(_milliseconds(2)) timer.stop() assert timer.value == 3312 assert repr(timer) == 'Timer(name="test.timer.3", value=3312)' # make sure no error is raised, nothing happens timer.stop() timer.stop() timer.stop() assert repr(timer) == 'Timer(name="test.timer.3", value=3312)'
def test_no_metric_values_does_nothing(self): publisher = DogStatsdPublisher('127.0.0.1', 8125) assert publisher.get_formatted_metrics([Timer(u'hello')]) == []