def test_start_stop_with_no_tasks(self):
"""Service can be started/stopped even if there are no tasks."""
service = ReportingService()
self.assertEqual(0, len(service.tasks))
service.startService()
self.assertTrue(service.running)
service.stopService()
self.assertFalse(service.running)
def makeService(host='127.0.0.1', port=8125, sample_rate=1.0, prefix=''):
client = TwistedStatsDClient(host, port)
metrics = Metrics(connection=client, namespace=prefix)
reporting = ReportingService()
for report in PROCESS_STATS:
reporting.schedule(report, sample_rate, metrics.gauge)
for report in NET_STATS:
reporting.schedule(report, sample_rate, metrics.gauge)
for report in COUNTER_STATS:
reporting.schedule(report, sample_rate, metrics.gauge)
# Attach statsd log observer
metric_collector = StatsdMetricCollector(metrics)
reporting.schedule(metric_collector.report_metrics, sample_rate, None)
log.addObserver(metric_collector.emit)
protocol = StatsDClientProtocol(client)
reactor.listenUDP(0, protocol)
return reporting
def test_schedule_when_running(self):
"""Schedule after service is running runs the task immediately."""
clock = Clock()
service = ReportingService(clock=clock)
service.startService()
called = []
def foo():
called.append(("foo", 1))
service.schedule(foo, 1, None)
self.assertEqual(1, len(service.tasks))
self.assertEquals([("foo", 1)], called)
def makeService(host='127.0.0.1', port=8125, sample_rate=1.0, prefix=''):
client = TwistedStatsDClient(host, port)
metrics = Metrics(connection=client, namespace=prefix)
reporting = ReportingService()
for report in PROCESS_STATS:
reporting.schedule(report, sample_rate, metrics.gauge)
for report in COUNTER_STATS:
reporting.schedule(report, sample_rate, metrics.gauge)
# Attach log observer to collect metrics for us
metric_collector = MetricCollector()
metric_collector.start()
metric_reporter = MetricReporter(metrics, metric_collector)
reporting.schedule(metric_reporter.report_metrics, sample_rate, None)
protocol = StatsDClientProtocol(client)
reactor.listenUDP(0, protocol)
return reporting
def test_schedule_without_report_function(self):
"""Scheduling without a report function calls original function."""
clock = Clock()
service = ReportingService(clock=clock)
called = []
def foo():
called.append(("foo", 1))
service.schedule(foo, 1, None)
self.assertEqual(1, len(service.tasks))
service.startService()
clock.advance(1)
self.assertEquals([("foo", 1)], called)
def test_schedule_with_report_function(self):
"""Scheduling with a report function wraps original function."""
clock = Clock()
service = ReportingService(clock=clock)
def foo():
return {"foo": 1}
called = []
def report(name, value):
called.append((name, value))
service.schedule(foo, 1, report)
self.assertEqual(1, len(service.tasks))
service.startService()
clock.advance(1)
self.assertEquals([("foo", 1)], called)
def makeService(host='127.0.0.1', port=8125, sample_rate=1.0, prefix=''):
client = TwistedStatsDClient(host, port)
metrics = Metrics(connection=client, namespace=prefix)
reporting = ReportingService()
if "bsd" not in sys.platform:
for report in PROCESS_STATS:
reporting.schedule(report, sample_rate, metrics.gauge)
for report in COUNTER_STATS:
reporting.schedule(report, sample_rate, metrics.gauge)
# Attach log observer to collect metrics for us
metric_collector = MetricCollector()
metric_collector.start()
metric_reporter = MetricReporter(metrics, metric_collector)
reporting.schedule(metric_reporter.report_metrics, sample_rate, None)
protocol = StatsDClientProtocol(client)
reactor.listenUDP(0, protocol)
return reporting
def test_report_with_instance_name(self):
"""
If an instance_name was provided when creating the ReportingService, it
gets prepended to the metric name when reporting.
"""
clock = Clock()
service = ReportingService(instance_name="instance-1", clock=clock)
def foo():
return {"foo": 1}
called = []
def report(name, value):
called.append((name, value))
service.schedule(foo, 1, report)
self.assertEqual(1, len(service.tasks))
service.startService()
clock.advance(1)
self.assertEquals([("instance-1.foo", 1)], called)
def createService(options):
"""Create a txStatsD service."""
from carbon.routers import ConsistentHashingRouter
from carbon.client import CarbonClientManager
from carbon.conf import settings
settings.MAX_QUEUE_SIZE = options["max-queue-size"]
settings.MAX_DATAPOINTS_PER_MESSAGE = options["max-datapoints-per-message"]
root_service = MultiService()
root_service.setName("statsd")
prefix = options["prefix"]
if prefix is None:
prefix = "statsd"
instance_name = options["instance-name"]
if not instance_name:
instance_name = platform.node()
# initialize plugins
plugin_metrics = []
for plugin in getPlugins(IMetricFactory):
plugin.configure(options)
plugin_metrics.append(plugin)
processor = None
if options["dump-mode"]:
# LoggingMessageProcessor supersedes
# any other processor class in "dump-mode"
assert not hasattr(log, 'info')
log.info = log.msg # for compatibility with LMP logger interface
processor = functools.partial(LoggingMessageProcessor, logger=log)
if options["statsd-compliance"]:
processor = (processor or MessageProcessor)(plugins=plugin_metrics)
input_router = Router(processor, options['routing'], root_service)
connection = InternalClient(input_router)
metrics = Metrics(connection)
else:
processor = (processor or ConfigurableMessageProcessor)(
message_prefix=prefix,
internal_metrics_prefix=prefix + "." + instance_name + ".",
plugins=plugin_metrics)
input_router = Router(processor, options['routing'], root_service)
connection = InternalClient(input_router)
metrics = ExtendedMetrics(connection)
if not options["carbon-cache-host"]:
options["carbon-cache-host"].append("127.0.0.1")
if not options["carbon-cache-port"]:
options["carbon-cache-port"].append(2004)
if not options["carbon-cache-name"]:
options["carbon-cache-name"].append(None)
reporting = ReportingService(instance_name)
reporting.setServiceParent(root_service)
reporting.schedule(report_client_manager_stats,
options["flush-interval"] / 1000, metrics.gauge)
if options["report"] is not None:
from txstatsd import process
from twisted.internet import reactor
reporting.schedule(process.report_reactor_stats(reactor), 60,
metrics.gauge)
reports = [name.strip() for name in options["report"].split(",")]
for report_name in reports:
if report_name == "reactor":
inspector = ReactorInspectorService(reactor,
metrics,
loop_time=0.05)
inspector.setServiceParent(root_service)
for reporter in getattr(process, "%s_STATS" % report_name.upper(),
()):
reporting.schedule(reporter, 60, metrics.gauge)
# XXX Make this configurable.
router = ConsistentHashingRouter()
carbon_client = CarbonClientManager(router)
carbon_client.setServiceParent(root_service)
for host, port, name in zip(options["carbon-cache-host"],
options["carbon-cache-port"],
options["carbon-cache-name"]):
carbon_client.startClient((host, port, name))
statsd_service = StatsDService(carbon_client, input_router,
options["flush-interval"])
statsd_service.setServiceParent(root_service)
statsd_server_protocol = StatsDServerProtocol(
input_router,
monitor_message=options["monitor-message"],
monitor_response=options["monitor-response"])
listener = UDPServer(options["listen-port"], statsd_server_protocol)
listener.setServiceParent(root_service)
if options["listen-tcp-port"] is not None:
statsd_tcp_server_factory = StatsDTCPServerFactory(
input_router,
monitor_message=options["monitor-message"],
monitor_response=options["monitor-response"])
listener = TCPServer(options["listen-tcp-port"],
statsd_tcp_server_factory)
listener.setServiceParent(root_service)
httpinfo_service = httpinfo.makeService(options, processor, statsd_service)
httpinfo_service.setServiceParent(root_service)
return root_service
def createService(options):
"""Create a txStatsD service."""
from carbon.routers import ConsistentHashingRouter
from carbon.client import CarbonClientManager
from carbon.conf import settings
settings.MAX_QUEUE_SIZE = options["max-queue-size"]
settings.MAX_DATAPOINTS_PER_MESSAGE = options["max-datapoints-per-message"]
root_service = MultiService()
root_service.setName("statsd")
prefix = options["prefix"]
if prefix is None:
prefix = "statsd"
instance_name = options["instance-name"]
if not instance_name:
instance_name = platform.node()
# initialize plugins
plugin_metrics = []
for plugin in getPlugins(IMetricFactory):
plugin.configure(options)
plugin_metrics.append(plugin)
processor = None
if options["dump-mode"]:
# LoggingMessageProcessor supersedes
# any other processor class in "dump-mode"
assert not hasattr(log, 'info')
log.info = log.msg # for compatibility with LMP logger interface
processor = functools.partial(LoggingMessageProcessor, logger=log)
if options["statsd-compliance"]:
processor = (processor or MessageProcessor)(plugins=plugin_metrics)
input_router = Router(processor, options['routing'], root_service)
connection = InternalClient(input_router)
metrics = Metrics(connection)
else:
processor = (processor or ConfigurableMessageProcessor)(
message_prefix=prefix,
internal_metrics_prefix=prefix + "." + instance_name + ".",
plugins=plugin_metrics)
input_router = Router(processor, options['routing'], root_service)
connection = InternalClient(input_router)
metrics = ExtendedMetrics(connection)
if not options["carbon-cache-host"]:
options["carbon-cache-host"].append("127.0.0.1")
if not options["carbon-cache-port"]:
options["carbon-cache-port"].append(2004)
if not options["carbon-cache-name"]:
options["carbon-cache-name"].append(None)
reporting = ReportingService(instance_name)
reporting.setServiceParent(root_service)
reporting.schedule(report_client_manager_stats,
options["flush-interval"] / 1000,
metrics.gauge)
if options["report"] is not None:
from txstatsd import process
from twisted.internet import reactor
reporting.schedule(
process.report_reactor_stats(reactor), 60, metrics.gauge)
reports = [name.strip() for name in options["report"].split(",")]
for report_name in reports:
if report_name == "reactor":
inspector = ReactorInspectorService(reactor, metrics,
loop_time=0.05)
inspector.setServiceParent(root_service)
for reporter in getattr(process, "%s_STATS" %
report_name.upper(), ()):
reporting.schedule(reporter, 60, metrics.gauge)
# XXX Make this configurable.
router = ConsistentHashingRouter()
carbon_client = CarbonClientManager(router)
carbon_client.setServiceParent(root_service)
for host, port, name in zip(options["carbon-cache-host"],
options["carbon-cache-port"],
options["carbon-cache-name"]):
carbon_client.startClient((host, port, name))
statsd_service = StatsDService(carbon_client, input_router,
options["flush-interval"])
statsd_service.setServiceParent(root_service)
statsd_server_protocol = StatsDServerProtocol(
input_router,
monitor_message=options["monitor-message"],
monitor_response=options["monitor-response"])
listener = UDPServer(options["listen-port"], statsd_server_protocol)
listener.setServiceParent(root_service)
if options["listen-tcp-port"] is not None:
statsd_tcp_server_factory = StatsDTCPServerFactory(
input_router,
monitor_message=options["monitor-message"],
monitor_response=options["monitor-response"])
listener = TCPServer(options["listen-tcp-port"],
statsd_tcp_server_factory)
listener.setServiceParent(root_service)
httpinfo_service = httpinfo.makeService(options, processor, statsd_service)
httpinfo_service.setServiceParent(root_service)
return root_service
