def processMessage(self, message):
"""Parse a message and post it as a metric."""
if self.factory.verbose:
log.listener("Message received: %s" % (message,))
metric = message.routing_key
for line in message.content.body.split("\n"):
try:
if settings.get("AMQP_METRIC_NAME_IN_BODY", False):
metric, value, timestamp = line.strip().split()
else:
value, timestamp = line.strip().split()
datapoint = ( float(timestamp), float(value) )
except ValueError:
log.listener("invalid message line: %s" % (line,))
continue
increment('metricsReceived')
metricReceived(metric, datapoint)
if self.factory.verbose:
log.listener("Metric posted: %s %s %s" %
(metric, value, timestamp,))
def build(cls, root_service):
plugin_up = cls.plugin_name.upper()
interface = settings.get('%s_RECEIVER_INTERFACE' % plugin_up, None)
port = int(settings.get('%s_RECEIVER_PORT' % plugin_up, 0))
protocol = cls
if not port:
return
if hasattr(protocol, 'datagramReceived'):
service = CarbonService(interface, port, protocol, None)
else:
factory = CarbonReceiverFactory()
factory.protocol = protocol
service = CarbonService(interface, port, protocol, factory)
service.setServiceParent(root_service)
def processMessage(self, message):
"""Parse a message and post it as a metric."""
if self.factory.verbose:
log.listener("Message received: %s" % (message, ))
metric = message.routing_key
for line in message.content.body.split("\n"):
line = line.strip()
if not line:
continue
try:
if settings.get("AMQP_METRIC_NAME_IN_BODY", False):
metric, value, timestamp = line.split()
else:
value, timestamp = line.split()
datapoint = (float(timestamp), float(value))
if datapoint[1] != datapoint[1]: # filter out NaN values
continue
except ValueError:
log.listener("invalid message line: %s" % (line, ))
continue
events.metricReceived(metric, datapoint)
if self.factory.verbose:
log.listener("Metric posted: %s %s %s" % (
metric,
value,
timestamp,
))
def __init__(self, input_pattern, output_pattern, method, frequency):
self.input_pattern = input_pattern
self.output_pattern = output_pattern
self.method = method
self.frequency = int(frequency)
if method not in AGGREGATION_METHODS:
raise ValueError("Invalid aggregation method '%s'" % method)
self.aggregation_func = AGGREGATION_METHODS[method]
self.build_regex()
self.build_template()
self.cache = LRUCache(settings.get("AGGREGATION_CACHE_SIZE", 10000))
def __init__(self, input_pattern, output_pattern, method, frequency):
self.input_pattern = input_pattern
self.output_pattern = output_pattern
self.method = method
self.frequency = int(frequency)
if method not in AGGREGATION_METHODS:
raise ValueError("Invalid aggregation method '%s'" % method)
self.aggregation_func = AGGREGATION_METHODS[method]
self.build_regex()
self.build_template()
self.cache = LRUCache(settings.get("AGGREGATION_CACHE_SIZE", 10000))
def createBaseService(config):
from carbon.conf import settings
from carbon.protocols import (MetricLineReceiver, MetricPickleReceiver,
MetricDatagramReceiver)
root_service = CarbonRootService()
root_service.setName(settings.program)
use_amqp = settings.get("ENABLE_AMQP", False)
if use_amqp:
from carbon import amqp_listener
amqp_host = settings.get("AMQP_HOST", "localhost")
amqp_port = settings.get("AMQP_PORT", 5672)
amqp_user = settings.get("AMQP_USER", "guest")
amqp_password = settings.get("AMQP_PASSWORD", "guest")
amqp_verbose = settings.get("AMQP_VERBOSE", False)
amqp_vhost = settings.get("AMQP_VHOST", "/")
amqp_spec = settings.get("AMQP_SPEC", None)
amqp_exchange_name = settings.get("AMQP_EXCHANGE", "graphite")
for interface, port, backlog, protocol in (
(settings.LINE_RECEIVER_INTERFACE, settings.LINE_RECEIVER_PORT,
settings.LINE_RECEIVER_BACKLOG, MetricLineReceiver),
(settings.PICKLE_RECEIVER_INTERFACE, settings.PICKLE_RECEIVER_PORT,
settings.PICKLE_RECEIVER_BACKLOG, MetricPickleReceiver)):
if port:
factory = ServerFactory()
factory.protocol = protocol
service = TCPServer(int(port),
factory,
interface=interface,
backlog=backlog)
service.setServiceParent(root_service)
if settings.ENABLE_UDP_LISTENER:
service = UDPServer(int(settings.UDP_RECEIVER_PORT),
MetricDatagramReceiver(),
interface=settings.UDP_RECEIVER_INTERFACE)
service.setServiceParent(root_service)
if use_amqp:
factory = amqp_listener.createAMQPListener(
amqp_user,
amqp_password,
vhost=amqp_vhost,
spec=amqp_spec,
exchange_name=amqp_exchange_name,
verbose=amqp_verbose)
service = TCPClient(amqp_host, int(amqp_port), factory)
service.setServiceParent(root_service)
if settings.ENABLE_MANHOLE:
from carbon import manhole
factory = manhole.createManholeListener()
service = TCPServer(int(settings.MANHOLE_PORT),
factory,
interface=settings.MANHOLE_INTERFACE)
service.setServiceParent(root_service)
if settings.USE_WHITELIST:
from carbon.regexlist import WhiteList, BlackList
WhiteList.read_from(settings["whitelist"])
BlackList.read_from(settings["blacklist"])
# Instantiate an instrumentation service that will record metrics about
# this service.
from carbon.instrumentation import InstrumentationService
service = InstrumentationService()
service.setServiceParent(root_service)
return root_service
def createBaseService(config):
from carbon.conf import settings
from carbon.protocols import (MetricLineReceiver, MetricPickleReceiver,
MetricDatagramReceiver, CmdLineReceiver)
root_service = CarbonRootService()
root_service.setName(settings.program)
use_amqp = settings.get("ENABLE_AMQP", False)
if use_amqp:
from carbon import amqp_listener
amqp_host = settings.get("AMQP_HOST", "localhost")
amqp_port = settings.get("AMQP_PORT", 5672)
amqp_user = settings.get("AMQP_USER", "guest")
amqp_password = settings.get("AMQP_PASSWORD", "guest")
amqp_verbose = settings.get("AMQP_VERBOSE", False)
amqp_vhost = settings.get("AMQP_VHOST", "/")
amqp_spec = settings.get("AMQP_SPEC", None)
amqp_exchange_name = settings.get("AMQP_EXCHANGE", "graphite")
for interface, port, protocol in ((settings.LINE_RECEIVER_INTERFACE,
settings.LINE_RECEIVER_PORT,
MetricLineReceiver),
(settings.PICKLE_RECEIVER_INTERFACE,
settings.PICKLE_RECEIVER_PORT,
MetricPickleReceiver),
(settings.CMD_LINE_RECEIVER_INTERFACE,
settings.CMD_LINE_RECEIVER_PORT,
CmdLineReceiver)):
if port:
factory = ServerFactory()
factory.protocol = protocol
service = TCPServer(int(port), factory, interface=interface)
service.setServiceParent(root_service)
if settings.ENABLE_UDP_LISTENER:
service = UDPServer(int(settings.UDP_RECEIVER_PORT),
MetricDatagramReceiver(),
interface=settings.UDP_RECEIVER_INTERFACE)
service.setServiceParent(root_service)
if use_amqp:
factory = amqp_listener.createAMQPListener(
amqp_user, amqp_password,
vhost=amqp_vhost, spec=amqp_spec,
exchange_name=amqp_exchange_name,
verbose=amqp_verbose)
service = TCPClient(amqp_host, int(amqp_port), factory)
service.setServiceParent(root_service)
if settings.ENABLE_MANHOLE:
from carbon import manhole
factory = manhole.createManholeListener()
service = TCPServer(int(settings.MANHOLE_PORT), factory,
interface=settings.MANHOLE_INTERFACE)
service.setServiceParent(root_service)
if settings.USE_WHITELIST:
from carbon.regexlist import WhiteList,BlackList
WhiteList.read_from(settings["whitelist"])
BlackList.read_from(settings["blacklist"])
# Instantiate an instrumentation service that will record metrics about
# this service.
from carbon.instrumentation import InstrumentationService
service = InstrumentationService()
service.setServiceParent(root_service)
return root_service
if instance:
settings.readFrom(config, 'cache:%s' % instance)
# Import application components
from carbon.log import logToStdout, logToDir
from carbon.listeners import MetricLineReceiver, MetricPickleReceiver, CacheQueryHandler, startListener
from carbon.cache import MetricCache
from carbon.instrumentation import startRecording
from carbon.events import metricReceived
storage_schemas = join(CONF_DIR, 'storage-schemas.conf')
if not exists(storage_schemas):
print "Error: missing required config %s" % storage_schemas
sys.exit(1)
use_amqp = settings.get("ENABLE_AMQP", False)
if use_amqp:
from carbon import amqp_listener
amqp_host = settings.get("AMQP_HOST", "localhost")
amqp_port = settings.get("AMQP_PORT", 5672)
amqp_user = settings.get("AMQP_USER", "guest")
amqp_password = settings.get("AMQP_PASSWORD", "guest")
amqp_verbose = settings.get("AMQP_VERBOSE", False)
amqp_vhost = settings.get("AMQP_VHOST", "/")
amqp_spec = settings.get("AMQP_SPEC", None)
amqp_exchange_name = settings.get("AMQP_EXCHANGE", "graphite")
# --debug
if options.debug:
logToStdout()
if instance:
settings.readFrom(config, 'cache:%s' % instance)
# Import application components
from carbon.log import logToStdout, logToDir
from carbon.listeners import MetricLineReceiver, MetricPickleReceiver, CacheQueryHandler, startListener
from carbon.cache import MetricCache
from carbon.instrumentation import startRecording
from carbon.events import metricReceived
storage_schemas = join(CONF_DIR, 'storage-schemas.conf')
if not exists(storage_schemas):
print "Error: missing required config %s" % storage_schemas
sys.exit(1)
use_amqp = settings.get("ENABLE_AMQP", False)
if use_amqp:
from carbon import amqp_listener
amqp_host = settings.get("AMQP_HOST", "localhost")
amqp_port = settings.get("AMQP_PORT", 5672)
amqp_user = settings.get("AMQP_USER", "guest")
amqp_password = settings.get("AMQP_PASSWORD", "guest")
amqp_verbose = settings.get("AMQP_VERBOSE", False)
amqp_vhost = settings.get("AMQP_VHOST", "/")
amqp_spec = settings.get("AMQP_SPEC", None)
amqp_exchange_name = settings.get("AMQP_EXCHANGE", "graphite")
# --debug
if options.debug:
logToStdout()
