def test_start_stop_with_no_tasks(self): """Service can be started/stopped even if there are no tasks.""" service = ReportingService() self.assertEqual(0, len(service.tasks)) service.startService() self.assertTrue(service.running) service.stopService() self.assertFalse(service.running)
def makeService(host='127.0.0.1', port=8125, sample_rate=1.0, prefix=''): client = TwistedStatsDClient(host, port) metrics = Metrics(connection=client, namespace=prefix) reporting = ReportingService() for report in PROCESS_STATS: reporting.schedule(report, sample_rate, metrics.gauge) for report in NET_STATS: reporting.schedule(report, sample_rate, metrics.gauge) for report in COUNTER_STATS: reporting.schedule(report, sample_rate, metrics.gauge) # Attach statsd log observer metric_collector = StatsdMetricCollector(metrics) reporting.schedule(metric_collector.report_metrics, sample_rate, None) log.addObserver(metric_collector.emit) protocol = StatsDClientProtocol(client) reactor.listenUDP(0, protocol) return reporting
def test_schedule_when_running(self): """Schedule after service is running runs the task immediately.""" clock = Clock() service = ReportingService(clock=clock) service.startService() called = [] def foo(): called.append(("foo", 1)) service.schedule(foo, 1, None) self.assertEqual(1, len(service.tasks)) self.assertEquals([("foo", 1)], called)
def makeService(host='127.0.0.1', port=8125, sample_rate=1.0, prefix=''): client = TwistedStatsDClient(host, port) metrics = Metrics(connection=client, namespace=prefix) reporting = ReportingService() for report in PROCESS_STATS: reporting.schedule(report, sample_rate, metrics.gauge) for report in COUNTER_STATS: reporting.schedule(report, sample_rate, metrics.gauge) # Attach log observer to collect metrics for us metric_collector = MetricCollector() metric_collector.start() metric_reporter = MetricReporter(metrics, metric_collector) reporting.schedule(metric_reporter.report_metrics, sample_rate, None) protocol = StatsDClientProtocol(client) reactor.listenUDP(0, protocol) return reporting
def test_schedule_without_report_function(self): """Scheduling without a report function calls original function.""" clock = Clock() service = ReportingService(clock=clock) called = [] def foo(): called.append(("foo", 1)) service.schedule(foo, 1, None) self.assertEqual(1, len(service.tasks)) service.startService() clock.advance(1) self.assertEquals([("foo", 1)], called)
def test_schedule_with_report_function(self): """Scheduling with a report function wraps original function.""" clock = Clock() service = ReportingService(clock=clock) def foo(): return {"foo": 1} called = [] def report(name, value): called.append((name, value)) service.schedule(foo, 1, report) self.assertEqual(1, len(service.tasks)) service.startService() clock.advance(1) self.assertEquals([("foo", 1)], called)
def makeService(host='127.0.0.1', port=8125, sample_rate=1.0, prefix=''): client = TwistedStatsDClient(host, port) metrics = Metrics(connection=client, namespace=prefix) reporting = ReportingService() if "bsd" not in sys.platform: for report in PROCESS_STATS: reporting.schedule(report, sample_rate, metrics.gauge) for report in COUNTER_STATS: reporting.schedule(report, sample_rate, metrics.gauge) # Attach log observer to collect metrics for us metric_collector = MetricCollector() metric_collector.start() metric_reporter = MetricReporter(metrics, metric_collector) reporting.schedule(metric_reporter.report_metrics, sample_rate, None) protocol = StatsDClientProtocol(client) reactor.listenUDP(0, protocol) return reporting
def test_report_with_instance_name(self): """ If an instance_name was provided when creating the ReportingService, it gets prepended to the metric name when reporting. """ clock = Clock() service = ReportingService(instance_name="instance-1", clock=clock) def foo(): return {"foo": 1} called = [] def report(name, value): called.append((name, value)) service.schedule(foo, 1, report) self.assertEqual(1, len(service.tasks)) service.startService() clock.advance(1) self.assertEquals([("instance-1.foo", 1)], called)
def createService(options): """Create a txStatsD service.""" from carbon.routers import ConsistentHashingRouter from carbon.client import CarbonClientManager from carbon.conf import settings settings.MAX_QUEUE_SIZE = options["max-queue-size"] settings.MAX_DATAPOINTS_PER_MESSAGE = options["max-datapoints-per-message"] root_service = MultiService() root_service.setName("statsd") prefix = options["prefix"] if prefix is None: prefix = "statsd" instance_name = options["instance-name"] if not instance_name: instance_name = platform.node() # initialize plugins plugin_metrics = [] for plugin in getPlugins(IMetricFactory): plugin.configure(options) plugin_metrics.append(plugin) processor = None if options["dump-mode"]: # LoggingMessageProcessor supersedes # any other processor class in "dump-mode" assert not hasattr(log, 'info') log.info = log.msg # for compatibility with LMP logger interface processor = functools.partial(LoggingMessageProcessor, logger=log) if options["statsd-compliance"]: processor = (processor or MessageProcessor)(plugins=plugin_metrics) input_router = Router(processor, options['routing'], root_service) connection = InternalClient(input_router) metrics = Metrics(connection) else: processor = (processor or ConfigurableMessageProcessor)( message_prefix=prefix, internal_metrics_prefix=prefix + "." + instance_name + ".", plugins=plugin_metrics) input_router = Router(processor, options['routing'], root_service) connection = InternalClient(input_router) metrics = ExtendedMetrics(connection) if not options["carbon-cache-host"]: options["carbon-cache-host"].append("127.0.0.1") if not options["carbon-cache-port"]: options["carbon-cache-port"].append(2004) if not options["carbon-cache-name"]: options["carbon-cache-name"].append(None) reporting = ReportingService(instance_name) reporting.setServiceParent(root_service) reporting.schedule(report_client_manager_stats, options["flush-interval"] / 1000, metrics.gauge) if options["report"] is not None: from txstatsd import process from twisted.internet import reactor reporting.schedule(process.report_reactor_stats(reactor), 60, metrics.gauge) reports = [name.strip() for name in options["report"].split(",")] for report_name in reports: if report_name == "reactor": inspector = ReactorInspectorService(reactor, metrics, loop_time=0.05) inspector.setServiceParent(root_service) for reporter in getattr(process, "%s_STATS" % report_name.upper(), ()): reporting.schedule(reporter, 60, metrics.gauge) # XXX Make this configurable. router = ConsistentHashingRouter() carbon_client = CarbonClientManager(router) carbon_client.setServiceParent(root_service) for host, port, name in zip(options["carbon-cache-host"], options["carbon-cache-port"], options["carbon-cache-name"]): carbon_client.startClient((host, port, name)) statsd_service = StatsDService(carbon_client, input_router, options["flush-interval"]) statsd_service.setServiceParent(root_service) statsd_server_protocol = StatsDServerProtocol( input_router, monitor_message=options["monitor-message"], monitor_response=options["monitor-response"]) listener = UDPServer(options["listen-port"], statsd_server_protocol) listener.setServiceParent(root_service) if options["listen-tcp-port"] is not None: statsd_tcp_server_factory = StatsDTCPServerFactory( input_router, monitor_message=options["monitor-message"], monitor_response=options["monitor-response"]) listener = TCPServer(options["listen-tcp-port"], statsd_tcp_server_factory) listener.setServiceParent(root_service) httpinfo_service = httpinfo.makeService(options, processor, statsd_service) httpinfo_service.setServiceParent(root_service) return root_service
def createService(options): """Create a txStatsD service.""" from carbon.routers import ConsistentHashingRouter from carbon.client import CarbonClientManager from carbon.conf import settings settings.MAX_QUEUE_SIZE = options["max-queue-size"] settings.MAX_DATAPOINTS_PER_MESSAGE = options["max-datapoints-per-message"] root_service = MultiService() root_service.setName("statsd") prefix = options["prefix"] if prefix is None: prefix = "statsd" instance_name = options["instance-name"] if not instance_name: instance_name = platform.node() # initialize plugins plugin_metrics = [] for plugin in getPlugins(IMetricFactory): plugin.configure(options) plugin_metrics.append(plugin) processor = None if options["dump-mode"]: # LoggingMessageProcessor supersedes # any other processor class in "dump-mode" assert not hasattr(log, 'info') log.info = log.msg # for compatibility with LMP logger interface processor = functools.partial(LoggingMessageProcessor, logger=log) if options["statsd-compliance"]: processor = (processor or MessageProcessor)(plugins=plugin_metrics) input_router = Router(processor, options['routing'], root_service) connection = InternalClient(input_router) metrics = Metrics(connection) else: processor = (processor or ConfigurableMessageProcessor)( message_prefix=prefix, internal_metrics_prefix=prefix + "." + instance_name + ".", plugins=plugin_metrics) input_router = Router(processor, options['routing'], root_service) connection = InternalClient(input_router) metrics = ExtendedMetrics(connection) if not options["carbon-cache-host"]: options["carbon-cache-host"].append("127.0.0.1") if not options["carbon-cache-port"]: options["carbon-cache-port"].append(2004) if not options["carbon-cache-name"]: options["carbon-cache-name"].append(None) reporting = ReportingService(instance_name) reporting.setServiceParent(root_service) reporting.schedule(report_client_manager_stats, options["flush-interval"] / 1000, metrics.gauge) if options["report"] is not None: from txstatsd import process from twisted.internet import reactor reporting.schedule( process.report_reactor_stats(reactor), 60, metrics.gauge) reports = [name.strip() for name in options["report"].split(",")] for report_name in reports: if report_name == "reactor": inspector = ReactorInspectorService(reactor, metrics, loop_time=0.05) inspector.setServiceParent(root_service) for reporter in getattr(process, "%s_STATS" % report_name.upper(), ()): reporting.schedule(reporter, 60, metrics.gauge) # XXX Make this configurable. router = ConsistentHashingRouter() carbon_client = CarbonClientManager(router) carbon_client.setServiceParent(root_service) for host, port, name in zip(options["carbon-cache-host"], options["carbon-cache-port"], options["carbon-cache-name"]): carbon_client.startClient((host, port, name)) statsd_service = StatsDService(carbon_client, input_router, options["flush-interval"]) statsd_service.setServiceParent(root_service) statsd_server_protocol = StatsDServerProtocol( input_router, monitor_message=options["monitor-message"], monitor_response=options["monitor-response"]) listener = UDPServer(options["listen-port"], statsd_server_protocol) listener.setServiceParent(root_service) if options["listen-tcp-port"] is not None: statsd_tcp_server_factory = StatsDTCPServerFactory( input_router, monitor_message=options["monitor-message"], monitor_response=options["monitor-response"]) listener = TCPServer(options["listen-tcp-port"], statsd_tcp_server_factory) listener.setServiceParent(root_service) httpinfo_service = httpinfo.makeService(options, processor, statsd_service) httpinfo_service.setServiceParent(root_service) return root_service