Exemplo n.º 1
0
 def test_flush_and_delete_counter(self):
     """
     If a counter is present, flushing it will generate a counter message
     normalized to the default interval. If delete_idle_counters is true,
     then the counter key will no longer exist after the flush.
     """
     del_processor = MessageProcessor(time_function=lambda: 42,
                                       plugins=getPlugins(IMetricFactory),
                                       delete_idle_counters=1)
     del_processor.counter_metrics["gorets"] = 42
     messages = list(del_processor.flush())
     self.assertEqual(("stats.gorets", 4, 42), messages[0])
     self.assertEqual(("stats_counts.gorets", 42, 42), messages[1])
     self.assertEqual(("statsd.numStats", 1, 42), messages[2])
     try:
         self.assertEqual(0, del_processor.counter_metrics["gorets"])
         self.fail("key 'gorets' should not exist after flush.")
     except KeyError:
         pass
class FlushMeterMetricMessagesTest(TestCase):

    def setUp(self):
        self.processor = MessageProcessor(time_function=self.wall_clock_time)
        self.time_now = int(time.time())

    def wall_clock_time(self):
        return self.time_now

    def test_flush_meter_metric(self):
        """
        Test the correct rendering of the Graphite report for
        a meter metric.
        """
        self.processor.process("gorets:3.0|m")

        self.time_now += 1
        messages = list(self.processor.flush())
        self.assertEqual(
            ("stats.meter.gorets.count", 3.0, self.time_now),
            messages[0])
        self.assertEqual(
            ("stats.meter.gorets.rate", 3.0, self.time_now),
            messages[1])
        self.assertEqual(
            ("statsd.numStats", 1, self.time_now),
            messages[2])

        self.time_now += 60
        messages = list(self.processor.flush())
        self.assertEqual(
            ("stats.meter.gorets.count", 3.0, self.time_now),
            messages[0])
        self.assertEqual(
            ("stats.meter.gorets.rate", 0.0, self.time_now),
            messages[1])
        self.assertEqual(
            ("statsd.numStats", 1, self.time_now), messages[2])
Exemplo n.º 3
0
class ProcessorStatsTest(TestCase):

    def setUp(self):
        self.timer = Timer()
        self.processor = MessageProcessor(time_function=self.timer)

    def test_process_keeps_processing_time(self):
        """
        When a message is processed, we keep the time it took to process it for
        later reporting.
        """
        self.timer.set([0, 5])
        self.processor.process("gorets:1|c")
        self.assertEqual(5, self.processor.process_timings["c"])
        self.assertEquals(1, self.processor.by_type["c"])

    def test_flush_tracks_flushing_time(self):
        """
        When flushing metrics, we track the time each metric type took to be
        flushed.
        """
        self.timer.set([0,
                        0, 1, # counter
                        1, 3, # timer
                        3, 6, # gauge
                        6, 10, # meter
                        10, 15, # plugin
                        ])
        def flush_metrics_summary(messages, num_stats, per_metric, timestamp):
            self.assertEqual((0, 1), per_metric["counter"])
            self.assertEqual((0, 2), per_metric["timer"])
            self.assertEqual((0, 3), per_metric["gauge"])
            self.assertEqual((0, 4), per_metric["meter"])
            self.assertEqual((0, 5), per_metric["plugin"])
        self.addCleanup(setattr, self.processor, "flush_metrics_summary",
                        self.processor.flush_metrics_summary)
        self.processor.flush_metrics_summary = flush_metrics_summary
        self.processor.flush()

    def test_flush_metrics_summary(self):
        """
        When flushing the metrics summary, we report duration and count of
        flushing each different type of metric as well as processing time.
        """
        per_metric = {"counter": (10, 1)}
        self.processor.process_timings = {"c": 1}
        self.processor.by_type = {"c": 42}
        messages = []
        self.processor.flush_metrics_summary(messages, 1, per_metric, 42)
        self.assertEqual(5, len(messages))
        self.assertEqual([('statsd.numStats', 1, 42),
                          ('statsd.flush.counter.count', 10, 42),
                          ('statsd.flush.counter.duration', 1000, 42),
                          ('statsd.receive.c.count', 42, 42),
                          ('statsd.receive.c.duration', 1000, 42)],
                          messages)
        self.assertEquals({}, self.processor.process_timings)
        self.assertEquals({}, self.processor.by_type)
    def test_monitor_response(self):
        """
        The StatsD service messages the expected response to the
        monitoring agent.
        """
        from twisted.internet import reactor

        options = service.StatsDOptions()
        processor = MessageProcessor()
        statsd_server_protocol = StatsDServerProtocol(
            processor,
            monitor_message=options["monitor-message"],
            monitor_response=options["monitor-response"])
        reactor.listenUDP(options["listen-port"], statsd_server_protocol)

        agent = Agent()
        reactor.listenUDP(0, agent)

        @inlineCallbacks
        def exercise():
            def monitor_send():
                agent.transport.write(
                    options["monitor-message"],
                    ("127.0.0.1", options["listen-port"]))

            def statsd_response(result):
                self.assertEqual(options["monitor-response"],
                                 agent.monitor_response)

            yield monitor_send()

            d = Deferred()
            d.addCallback(statsd_response)
            reactor.callLater(.1, d.callback, None)
            try:
                yield d
            except:
                raise
            finally:
                reactor.stop()

        reactor.callWhenRunning(exercise)
        reactor.run()
 def setUp(self):
     self.processor = MessageProcessor(time_function=self.wall_clock_time)
     self.time_now = int(time.time())
 def setUp(self):
     self.processor = MessageProcessor(time_function=lambda: 42,
                                       plugins=getPlugins(IMetricFactory))
class FlushMessagesTest(TestCase):

    def setUp(self):
        self.processor = MessageProcessor(time_function=lambda: 42,
                                          plugins=getPlugins(IMetricFactory))

    def test_flush_no_stats(self):
        """
        Flushing the message processor when there are no stats available should
        still produce one message where C{statsd.numStats} is set to zero.
        """
        self.assertEqual(("statsd.numStats", 0, 42),
                         list(self.processor.flush())[0])

    def test_flush_counter(self):
        """
        If a counter is present, flushing it will generate a counter message
        normalized to the default interval.
        """
        self.processor.counter_metrics["gorets"] = 42
        messages = list(self.processor.flush())
        self.assertEqual(("stats.gorets", 4, 42), messages[0])
        self.assertEqual(("stats_counts.gorets", 42, 42), messages[1])
        self.assertEqual(("statsd.numStats", 1, 42), messages[2])
        self.assertEqual(0, self.processor.counter_metrics["gorets"])

    def test_flush_counter_one_second_interval(self):
        """
        It is possible to flush counters with a one-second interval, in which
        case the counter value will be unchanged.
        """
        self.processor.counter_metrics["gorets"] = 42
        messages = list(self.processor.flush(interval=1000))
        self.assertEqual(("stats.gorets", 42, 42), messages[0])
        self.assertEqual(("stats_counts.gorets", 42, 42), messages[1])
        self.assertEqual(("statsd.numStats", 1, 42), messages[2])
        self.assertEqual(0, self.processor.counter_metrics["gorets"])

    def test_flush_single_timer_single_time(self):
        """
        If a single timer with a single data point is present, all of upper,
        threshold_upper, lower, mean will be set to the same value. Timer is
        reset after flush is called.
        """
        self.processor.timer_metrics["glork"] = [24]
        messages = list(self.processor.flush())
        self.assertEqual(("stats.timers.glork.count", 1, 42), messages[0])
        self.assertEqual(("stats.timers.glork.lower", 24, 42), messages[1])
        self.assertEqual(("stats.timers.glork.mean", 24, 42), messages[2])
        self.assertEqual(("stats.timers.glork.upper", 24, 42), messages[3])
        self.assertEqual(("stats.timers.glork.upper_90", 24, 42), messages[4])
        self.assertEqual(("statsd.numStats", 1, 42), messages[5])
        self.assertEqual([], self.processor.timer_metrics["glork"])

    def test_flush_single_timer_multiple_times(self):
        """
        If a single timer with multiple data points is present:
        - lower will be set to the smallest value
        - upper will be set to the largest value
        - upper_90 will be set to the 90th percentile
        - count will be the count of data points
        - mean will be the mean value within the 90th percentile
        """
        self.processor.timer_metrics["glork"] = [4, 8, 15, 16, 23, 42]
        messages = list(self.processor.flush())
        self.assertEqual(("stats.timers.glork.count", 6, 42), messages[0])
        self.assertEqual(("stats.timers.glork.lower", 4, 42), messages[1])
        self.assertEqual(("stats.timers.glork.mean", 13, 42), messages[2])
        self.assertEqual(("stats.timers.glork.upper", 42, 42), messages[3])
        self.assertEqual(("stats.timers.glork.upper_90", 23, 42), messages[4])
        self.assertEqual(("statsd.numStats", 1, 42), messages[5])
        self.assertEqual([], self.processor.timer_metrics["glork"])

    def test_flush_single_timer_50th_percentile(self):
        """
        It is possible to flush the timers with a different percentile, in this
        example, 50%.

        If a single timer with multiple data points is present:
        - lower will be set to the smallest value
        - upper will be set to the largest value
        - upper_50 will be set to the 50th percentile
        - count will be the count of data points
        - mean will be the mean value within the 50th percentile
        """
        self.processor.timer_metrics["glork"] = [4, 8, 15, 16, 23, 42]
        messages = list(self.processor.flush(percent=50))
        self.assertEqual(("stats.timers.glork.count", 6, 42), messages[0])
        self.assertEqual(("stats.timers.glork.lower", 4, 42), messages[1])
        self.assertEqual(("stats.timers.glork.mean", 9, 42), messages[2])
        self.assertEqual(("stats.timers.glork.upper", 42, 42), messages[3])
        self.assertEqual(("stats.timers.glork.upper_50", 15, 42), messages[4])
        self.assertEqual(("statsd.numStats", 1, 42), messages[5])
        self.assertEqual([], self.processor.timer_metrics["glork"])

    def test_flush_gauge_metric(self):
        """
        Test the correct rendering of the Graphite report for
        a gauge metric.
        """

        self.processor.process("gorets:9.6|g")

        messages = list(self.processor.flush())
        self.assertEqual(
            ("stats.gauge.gorets.value", 9.6, 42), messages[0])
        self.assertEqual(
            ("statsd.numStats", 1, 42), messages[1])

        # ensure the gauge value remains after the flush.
        self.assertEqual(1, len(self.processor.gauge_metrics))

        # ensure that subsequent flushes continue to report the gauge value
        messages = list(self.processor.flush())
        self.assertEqual(
            ("stats.gauge.gorets.value", 9.6, 42), messages[0])
        self.assertEqual(
            ("statsd.numStats", 1, 42), messages[1])

    def test_flush_distinct_metric(self):
        """
        Test the correct rendering of the Graphite report for
        a distinct metric.
        """

        self.processor.process("gorets:item|pd")

        messages = list(self.processor.flush())
        self.assertEqual(("stats.pdistinct.gorets.count", 1, 42), messages[0])
        self.assertEqual(("stats.pdistinct.gorets.count_1day",
                          5552568545, 42), messages[1])
        self.assertEqual(("stats.pdistinct.gorets.count_1hour",
                          5552568545, 42), messages[2])
        self.assertEqual(("stats.pdistinct.gorets.count_1min",
                          5552568545, 42), messages[3])

    def test_flush_plugin_arguments(self):
        """Test the passing of arguments for flush."""

        class FakeMetric(object):
            def flush(self, interval, timestamp):
                self.data = interval, timestamp
                return []

        self.processor.plugin_metrics["somemetric"] = FakeMetric()
        list(self.processor.flush(41000))
        self.assertEquals(
            (41, 42), self.processor.plugin_metrics["somemetric"].data)
 def setUp(self):
     self.timer = Timer()
     self.processor = MessageProcessor(time_function=self.timer)
class ProcessorStatsTest(TestCase):

    def setUp(self):
        self.timer = Timer()
        self.processor = MessageProcessor(time_function=self.timer)

    def test_process_keeps_processing_time(self):
        """
        When a message is processed, we keep the time it took to process it for
        later reporting.
        """
        self.timer.set([0, 5])
        self.processor.process("gorets:1|c")
        self.assertEqual(5, self.processor.process_timings["c"])
        self.assertEquals(1, self.processor.by_type["c"])

    def test_flush_tracks_flushing_time(self):
        """
        When flushing metrics, we track the time each metric type took to be
        flushed.
        """
        self.timer.set([0,
                        0, 1,  # counter
                        1, 3,  # timer
                        3, 6,  # gauge
                        6, 10,  # meter
                        10, 15,  # plugin
                        ])

        def flush_metrics_summary(num_stats, per_metric, timestamp):
            self.assertEqual((0, 1), per_metric["counter"])
            self.assertEqual((0, 2), per_metric["timer"])
            self.assertEqual((0, 3), per_metric["gauge"])
            self.assertEqual((0, 4), per_metric["meter"])
            self.assertEqual((0, 5), per_metric["plugin"])
            yield ()

        self.addCleanup(setattr, self.processor, "flush_metrics_summary",
                        self.processor.flush_metrics_summary)
        self.processor.flush_metrics_summary = flush_metrics_summary
        list(self.processor.flush())

    def test_flush_metrics_summary(self):
        """
        When flushing the metrics summary, we report duration and count of
        flushing each different type of metric as well as processing time.
        """
        per_metric = {"counter": (10, 1)}
        self.processor.process_timings = {"c": 1}
        self.processor.by_type = {"c": 42}
        messages = []
        map(messages.extend, self.processor.flush_metrics_summary(
            1, per_metric, 42))
        self.assertEqual(5, len(messages))
        self.assertEqual([('statsd.numStats', 1, 42),
                          ('statsd.flush.counter.count', 10, 42),
                          ('statsd.flush.counter.duration', 1000, 42),
                          ('statsd.receive.c.count', 42, 42),
                          ('statsd.receive.c.duration', 1000, 42)],
                         messages)
        self.assertEquals({}, self.processor.process_timings)
        self.assertEquals({}, self.processor.by_type)