def __init__(self,
statsd_host="localhost",
statsd_port=8125,
namespace="aplt"):
self.client = TwistedStatsDClient.create(statsd_host, statsd_port)
self._metric = Metrics(connection=self.client, namespace=namespace)
self._stat_protocol = None
def get_metrics(namespace='daisy'):
global METRICS
if METRICS is None:
connection = UdpStatsDClient(host=config.statsd_host,
port=config.statsd_port)
connection.connect()
namespace = 'whoopsie-daisy.' + namespace
METRICS = Metrics(connection=connection, namespace=namespace)
return METRICS
def setup_statsd(config):
from txstatsd.client import TwistedStatsDClient, StatsDClientProtocol
from txstatsd.metrics.metrics import Metrics
global metrics
statsd = TwistedStatsDClient(config['statsd']['host'],
config['statsd']['port'])
metrics = Metrics(connection=statsd,
namespace='smap-archiver.' + config['statsd']['prefix'])
protocol = StatsDClientProtocol(statsd)
reactor.listenUDP(0, protocol)
class TwistedMetrics(object):
"""Twisted implementation of statsd output"""
def __init__(self, statsd_host="localhost", statsd_port=8125,
namespace="aplt"):
self.client = TwistedStatsDClient(statsd_host, statsd_port)
self._metric = Metrics(connection=self.client, namespace=namespace)
self._stat_protocol = None
def start(self):
protocol = StatsDClientProtocol(self.client)
self._stat_protocol = reactor.listenUDP(0, protocol)
def stop(self):
if self._stat_protocol: # pragma: nocover
self._stat_protocol.stopListening()
self._stat_protocol = None
def increment(self, name, count=1, **kwargs):
self._metric.increment(name, count)
def timing(self, name, duration, **kwargs):
self._metric.timing(name, duration)
class TwistedMetrics(object):
"""Twisted implementation of statsd output"""
def __init__(self,
statsd_host="localhost",
statsd_port=8125,
namespace="aplt"):
self.client = TwistedStatsDClient.create(statsd_host, statsd_port)
self._metric = Metrics(connection=self.client, namespace=namespace)
self._stat_protocol = None
def start(self):
protocol = StatsDClientProtocol(self.client)
self._stat_protocol = reactor.listenUDP(0, protocol)
def stop(self):
if self._stat_protocol: # pragma: nocover
self._stat_protocol.stopListening()
self._stat_protocol = None
def increment(self, name, count=1, **kwargs):
self._metric.increment(name, count)
def timing(self, name, duration, **kwargs):
self._metric.timing(name, duration)
def makeService(host='127.0.0.1', port=8125, sample_rate=1.0, prefix=''):
client = TwistedStatsDClient(host, port)
metrics = Metrics(connection=client, namespace=prefix)
reporting = ReportingService()
for report in PROCESS_STATS:
reporting.schedule(report, sample_rate, metrics.gauge)
for report in COUNTER_STATS:
reporting.schedule(report, sample_rate, metrics.gauge)
# Attach log observer to collect metrics for us
metric_collector = MetricCollector()
metric_collector.start()
metric_reporter = MetricReporter(metrics, metric_collector)
reporting.schedule(metric_reporter.report_metrics, sample_rate, None)
protocol = StatsDClientProtocol(client)
reactor.listenUDP(0, protocol)
return reporting
class TwistedMetrics(object):
"""Twisted implementation of statsd output"""
def __init__(self, statsd_host="localhost", statsd_port=8125):
self.client = TwistedStatsDClient(statsd_host, statsd_port)
self._metric = Metrics(connection=self.client, namespace="autopush")
def start(self):
protocol = StatsDClientProtocol(self.client)
reactor.listenUDP(0, protocol)
def increment(self, name, count=1, **kwargs):
self._metric.increment(name, count)
def gauge(self, name, count, **kwargs):
self._metric.gauge(name, count)
def timing(self, name, duration, **kwargs):
self._metric.timing(name, duration)
class TwistedMetrics(object):
"""Twisted implementation of statsd output"""
def __init__(self, statsd_host="localhost", statsd_port=8125):
self.client = TwistedStatsDClient(statsd_host, statsd_port)
self._metric = Metrics(connection=self.client, namespace="autopush")
def start(self):
protocol = StatsDClientProtocol(self.client)
reactor.listenUDP(0, protocol)
def increment(self, name, count=1, **kwargs):
self._metric.increment(name, count)
def gauge(self, name, count, **kwargs):
self._metric.gauge(name, count)
def timing(self, name, duration, **kwargs):
self._metric.timing(name, duration)
def __init__(self, statsd_host="localhost", statsd_port=8125,
namespace="aplt"):
self.client = TwistedStatsDClient(statsd_host, statsd_port)
self._metric = Metrics(connection=self.client, namespace=namespace)
self._stat_protocol = None
def __init__(self, statsd_host="localhost", statsd_port=8125):
self.client = TwistedStatsDClient(statsd_host, statsd_port)
self._metric = Metrics(connection=self.client, namespace="autopush")
def createService(options):
"""Create a txStatsD service."""
from carbon.routers import ConsistentHashingRouter
from carbon.client import CarbonClientManager
from carbon.conf import settings
settings.MAX_QUEUE_SIZE = options["max-queue-size"]
settings.MAX_DATAPOINTS_PER_MESSAGE = options["max-datapoints-per-message"]
root_service = MultiService()
root_service.setName("statsd")
prefix = options["prefix"]
if prefix is None:
prefix = "statsd"
instance_name = options["instance-name"]
if not instance_name:
instance_name = platform.node()
# initialize plugins
plugin_metrics = []
for plugin in getPlugins(IMetricFactory):
plugin.configure(options)
plugin_metrics.append(plugin)
processor = None
if options["dump-mode"]:
# LoggingMessageProcessor supersedes
# any other processor class in "dump-mode"
assert not hasattr(log, 'info')
log.info = log.msg # for compatibility with LMP logger interface
processor = functools.partial(LoggingMessageProcessor, logger=log)
if options["statsd-compliance"]:
processor = (processor or MessageProcessor)(plugins=plugin_metrics)
input_router = Router(processor, options['routing'], root_service)
connection = InternalClient(input_router)
metrics = Metrics(connection)
else:
processor = (processor or ConfigurableMessageProcessor)(
message_prefix=prefix,
internal_metrics_prefix=prefix + "." + instance_name + ".",
plugins=plugin_metrics)
input_router = Router(processor, options['routing'], root_service)
connection = InternalClient(input_router)
metrics = ExtendedMetrics(connection)
if not options["carbon-cache-host"]:
options["carbon-cache-host"].append("127.0.0.1")
if not options["carbon-cache-port"]:
options["carbon-cache-port"].append(2004)
if not options["carbon-cache-name"]:
options["carbon-cache-name"].append(None)
reporting = ReportingService(instance_name)
reporting.setServiceParent(root_service)
reporting.schedule(report_client_manager_stats,
options["flush-interval"] / 1000, metrics.gauge)
if options["report"] is not None:
from txstatsd import process
from twisted.internet import reactor
reporting.schedule(process.report_reactor_stats(reactor), 60,
metrics.gauge)
reports = [name.strip() for name in options["report"].split(",")]
for report_name in reports:
if report_name == "reactor":
inspector = ReactorInspectorService(reactor,
metrics,
loop_time=0.05)
inspector.setServiceParent(root_service)
for reporter in getattr(process, "%s_STATS" % report_name.upper(),
()):
reporting.schedule(reporter, 60, metrics.gauge)
# XXX Make this configurable.
router = ConsistentHashingRouter()
carbon_client = CarbonClientManager(router)
carbon_client.setServiceParent(root_service)
for host, port, name in zip(options["carbon-cache-host"],
options["carbon-cache-port"],
options["carbon-cache-name"]):
carbon_client.startClient((host, port, name))
statsd_service = StatsDService(carbon_client, input_router,
options["flush-interval"])
statsd_service.setServiceParent(root_service)
statsd_server_protocol = StatsDServerProtocol(
input_router,
monitor_message=options["monitor-message"],
monitor_response=options["monitor-response"])
listener = UDPServer(options["listen-port"], statsd_server_protocol)
listener.setServiceParent(root_service)
if options["listen-tcp-port"] is not None:
statsd_tcp_server_factory = StatsDTCPServerFactory(
input_router,
monitor_message=options["monitor-message"],
monitor_response=options["monitor-response"])
listener = TCPServer(options["listen-tcp-port"],
statsd_tcp_server_factory)
listener.setServiceParent(root_service)
httpinfo_service = httpinfo.makeService(options, processor, statsd_service)
httpinfo_service.setServiceParent(root_service)
return root_service
def setUp(self):
self.connection = FakeStatsDClient()
self.metrics = Metrics(self.connection, 'txstatsd.tests')
class TestMetrics(TestCase):
def setUp(self):
self.connection = FakeStatsDClient()
self.metrics = Metrics(self.connection, 'txstatsd.tests')
def test_gauge(self):
"""Test reporting of a gauge metric sample."""
self.metrics.gauge('gauge', 102)
self.assertEqual(self.connection.data,
'txstatsd.tests.gauge:102|g')
def test_meter(self):
"""Test reporting of a meter metric sample."""
self.metrics.meter('meter', 3)
self.assertEqual(self.connection.data,
'txstatsd.tests.meter:3|m')
def test_counter(self):
"""Test the increment and decrement operations."""
self.metrics.increment('counter', 18)
self.assertEqual(self.connection.data,
'txstatsd.tests.counter:18|c')
self.metrics.decrement('counter', 9)
self.assertEqual(self.connection.data,
'txstatsd.tests.counter:-9|c')
def test_timing(self):
"""Test the timing operation."""
self.metrics.timing('timing', 101.1234)
self.assertEqual(self.connection.data,
'txstatsd.tests.timing:101123.4|ms')
def test_timing_automatic(self):
"""Test the automatic timing operation with explicit reset"""
start_time = time.time()
self.metrics.reset_timing()
time.sleep(.1)
self.metrics.timing('timing')
elapsed = time.time() - start_time
label, val, units = re.split(":|\|", self.connection.data)
self.assertEqual(label, 'txstatsd.tests.timing')
self.assertEqual(units, 'ms')
self.assertTrue(100 <= float(val) <= elapsed * 1000)
def test_timing_automatic_implicit_reset(self):
"""Test the automatic timing operation with implicit reset"""
start_time = time.time()
self.metrics.timing('something_else')
time.sleep(.1)
self.metrics.timing('timing')
elapsed = time.time() - start_time
label, val, units = re.split(":|\|", self.connection.data)
self.assertEqual(label, 'txstatsd.tests.timing')
self.assertEqual(units, 'ms')
self.assertTrue(100 <= float(val) <= elapsed * 1000)
def test_generic(self):
"""Test the GenericMetric class."""
self.metrics.report('users', "pepe", "pd")
self.assertEqual(self.connection.data,
'txstatsd.tests.users:pepe|pd')
def test_generic_extra(self):
"""Test the GenericMetric class."""
self.metrics.report('users', "pepe", "pd", 100)
self.assertEqual(self.connection.data,
'txstatsd.tests.users:pepe|pd|100')
def test_empty_namespace(self):
"""Test reporting of an empty namespace."""
self.metrics.namespace = None
self.metrics.gauge('gauge', 213)
self.assertEqual(self.connection.data,
'gauge:213|g')
self.metrics.namespace = ''
self.metrics.gauge('gauge', 413)
self.assertEqual(self.connection.data,
'gauge:413|g')
def setUp(self):
self.connection = FakeStatsDClient()
self.metrics = Metrics(self.connection, 'txstatsd.tests')
class TestMetrics(TestCase):
def setUp(self):
self.connection = FakeStatsDClient()
self.metrics = Metrics(self.connection, 'txstatsd.tests')
def test_gauge(self):
"""Test reporting of a gauge metric sample."""
self.metrics.gauge('gauge', 102)
self.assertEqual(self.connection.data, b'txstatsd.tests.gauge:102|g')
def test_gauge_with_tags(self):
"""Test reporting of a gauge metric sample w/ tags."""
self.metrics.gauge('gauge', 102, tags=["foo:bar"])
self.assertEqual(self.connection.data,
b'txstatsd.tests.gauge:102|g|#foo:bar')
def test_meter(self):
"""Test reporting of a meter metric sample."""
self.metrics.meter('meter', 3)
self.assertEqual(self.connection.data, b'txstatsd.tests.meter:3|m')
def test_counter(self):
"""Test the increment and decrement operations."""
self.metrics.increment('counter', 18)
self.assertEqual(self.connection.data, b'txstatsd.tests.counter:18|c')
self.metrics.decrement('counter', 9)
self.assertEqual(self.connection.data, b'txstatsd.tests.counter:-9|c')
def test_counter_with_tags(self):
"""Test the increment and decrement operations w/ tags."""
self.metrics.increment('counter', 18, tags=["foo:bar"])
self.assertEqual(self.connection.data,
b'txstatsd.tests.counter:18|c|#foo:bar')
self.metrics.decrement('counter', 9, tags=["foo:bar", "baz:quux"])
self.assertEqual(self.connection.data,
b'txstatsd.tests.counter:-9|c|#foo:bar,baz:quux')
def test_timing(self):
"""Test the timing operation."""
self.metrics.timing('timing', 101.1234)
# to support both 2 and 3, can't use assertRegex
match = re.match(b'txstatsd.tests.timing:101123.4[0-9]*|ms',
self.connection.data)
self.assertFalse(match is None)
def test_timing_with_tags(self):
"""Test the timing operation w/ tags."""
self.metrics.timing('timing', 101.1234, tags=["foo:bar"])
# to support both 2 and 3, can't use assertRegex
match = re.match(b'txstatsd.tests.timing:101123.4[0-9]*|ms|#foo:bar',
self.connection.data)
self.assertFalse(match is None)
def test_timing_automatic(self):
"""Test the automatic timing operation with explicit reset"""
start_time = time.time()
self.metrics.reset_timing()
time.sleep(.1)
self.metrics.timing('timing')
elapsed = time.time() - start_time
label, val, units = re.split(b":|\|", self.connection.data)
self.assertEqual(label, b'txstatsd.tests.timing')
self.assertEqual(units, b'ms')
self.assertTrue(100 <= float(val) <= elapsed * 1000)
def test_timing_automatic_implicit_reset(self):
"""Test the automatic timing operation with implicit reset"""
start_time = time.time()
self.metrics.timing('something_else')
time.sleep(.1)
self.metrics.timing('timing')
elapsed = time.time() - start_time
label, val, units = re.split(b":|\|", self.connection.data)
self.assertEqual(label, b'txstatsd.tests.timing')
self.assertEqual(units, b'ms')
self.assertTrue(100 <= float(val) <= elapsed * 1000)
def test_generic(self):
"""Test the GenericMetric class."""
self.metrics.report('users', "pepe", "pd")
self.assertEqual(self.connection.data, b'txstatsd.tests.users:pepe|pd')
def test_generic_extra(self):
"""Test the GenericMetric class."""
self.metrics.report('users', "pepe", "pd", 100)
self.assertEqual(self.connection.data,
b'txstatsd.tests.users:pepe|pd|100')
def test_empty_namespace(self):
"""Test reporting of an empty namespace."""
self.metrics.namespace = None
self.metrics.gauge('gauge', 213)
self.assertEqual(self.connection.data, b'gauge:213|g')
self.metrics.namespace = ''
self.metrics.gauge('gauge', 413)
self.assertEqual(self.connection.data, b'gauge:413|g')
class DummyRequest(object):
"""Dummy request object that imitates twisted.web.http.Request's
interfaces."""
finished = 0
response_code = 200
response_msg = None
metrics = Metrics(FakeStatsDClient(), 'webprotectme.null')
def __init__(self, api_mode="test", api_version="0.1", api_name="TestAPI", uri="",
method="GET", user="", password=""):
self.headers = {}
self.args = {}
self.cookies = []
self.received_cookies = {}
self.client_ip = "4.3.2.1"
self.content = StringIO()
self._finishedDeferreds = []
self.api_mode = api_mode
self.api_version = api_version
self.api_name = api_name
self.uri = uri
self.user = user
self.password = password
self.method = method
self.ignore_auth = True
self.ignore_secure_cookies = True
x = self.uri.split(b'?', 1)
if len(x) == 1:
self.path = self.uri
else:
self.path, argstring = x
self.args = parse_qs(argstring, 1)
def _reset_body(self):
self.content = StringIO()
def setHeader(self, name, value):
self.headers[name] = value
def getAllHeaders(self):
return self.headers
def getHeader(self, header):
try:
return self.headers[header]
except KeyError:
return
def addCookie(self, k, v, expires=None, domain=None, path=None, max_age=None, comment=None, secure=None):
"""
Set an outgoing HTTP cookie.
In general, you should consider using sessions instead of cookies, see
L{twisted.web.server.Request.getSession} and the
L{twisted.web.server.Session} class for details.
"""
cookie = '%s=%s' % (k, v)
if expires is not None:
cookie = cookie +"; Expires=%s" % expires
if domain is not None:
cookie = cookie +"; Domain=%s" % domain
if path is not None:
cookie = cookie +"; Path=%s" % path
if max_age is not None:
cookie = cookie +"; Max-Age=%s" % max_age
if comment is not None:
cookie = cookie +"; Comment=%s" % comment
if secure:
cookie = cookie +"; Secure"
self.cookies.append(cookie)
def getCookie(self, key):
"""
Get a cookie that was sent from the network.
"""
return self.received_cookies.get(key)
def write(self, data):
cookie_data = ""
for cookie in self.cookies:
cookie_data = cookie_data + ('%s: %s\r\n' % ("Set-Cookie", cookie))
if cookie_data != "":
cookie_data = cookie_data + "\r\n"
self.content = StringIO(cookie_data + self.content.getvalue() + data)
self.cookies = []
else:
self.content = StringIO(self.content.getvalue() + data)
def notifyFinish(self):
"""
Return a L{Deferred} which is called back with C{None} when the request
is finished. This will probably only work if you haven't called
C{finish} yet.
"""
finished = Deferred()
self._finishedDeferreds.append(finished)
return finished
def getClientIP(self, honor_xrealip=False):
"""
Return request IP
"""
if honor_xrealip and self.getHeader("X-Real-IP"):
return self.getHeader("X-Real-IP")
return self.client_ip
def finish(self):
self.finished = self.finished + 1
if self._finishedDeferreds is not None:
observers = self._finishedDeferreds
self._finishedDeferreds = None
for obs in observers:
obs.callback(None)
def setResponseCode(self, code, message=None):
self.response_code = code
self.response_msg = message
def getUser(self):
return self.user
def getPassword(self):
return self.password
def __init__(self, statsd_host="localhost", statsd_port=8125):
self.client = TwistedStatsDClient(statsd_host, statsd_port)
self._metric = Metrics(connection=self.client, namespace="autopush")
def install_stats(host, port, scheme):
global metrics
statsd_client = TwistedStatsDClient(host, port)
metrics = Metrics(connection=statsd_client, namespace=scheme)
return StatsDClientProtocol(statsd_client)
sys.path.append(vendor_dir)
from txstatsd.client import (TwistedStatsDClient, StatsDClientProtocol)
from txstatsd.metrics.metrics import Metrics
class FakeStatsDClient(object):
def connect(self):
"""Connect to the StatsD server."""
pass
def disconnect(self):
"""Disconnect from the StatsD server."""
pass
def write(self, data):
"""Send the metric to the StatsD server."""
self.data = data
# Default metrics to using a fake provider.
metrics = Metrics(FakeStatsDClient(), 'webprotectme.null')
def install_stats(host, port, scheme):
global metrics
statsd_client = TwistedStatsDClient(host, port)
metrics = Metrics(connection=statsd_client, namespace=scheme)
return StatsDClientProtocol(statsd_client)
def __init__(self, namespace=""):
Metrics.__init__(self, FakeStatsDClient(self), namespace=namespace)
self.data = []
def __init__(self, namespace=""):
Metrics.__init__(self, FakeStatsDClient(self), namespace=namespace)
self.data = []
