def setupReceivers(root_service, settings):
from carbon.protocols import MetricLineReceiver, MetricPickleReceiver, MetricDatagramReceiver
for protocol, interface, port in [
(MetricLineReceiver, settings.LINE_RECEIVER_INTERFACE, settings.LINE_RECEIVER_PORT),
(MetricPickleReceiver, settings.PICKLE_RECEIVER_INTERFACE, settings.PICKLE_RECEIVER_PORT)
]:
if port:
factory = ServerFactory()
factory.protocol = protocol
service = TCPServer(port, factory, interface=interface)
service.setServiceParent(root_service)
if settings.ENABLE_UDP_LISTENER:
service = UDPServer(int(settings.UDP_RECEIVER_PORT),
MetricDatagramReceiver(),
interface=settings.UDP_RECEIVER_INTERFACE)
service.setServiceParent(root_service)
if settings.ENABLE_AMQP:
from carbon import amqp_listener
amqp_host = settings.AMQP_HOST
amqp_port = settings.AMQP_PORT
amqp_user = settings.AMQP_USER
amqp_password = settings.AMQP_PASSWORD
amqp_verbose = settings.AMQP_VERBOSE
amqp_vhost = settings.AMQP_VHOST
amqp_spec = settings.AMQP_SPEC
amqp_exchange_name = settings.AMQP_EXCHANGE
factory = amqp_listener.createAMQPListener(
amqp_user,
amqp_password,
vhost=amqp_vhost,
spec=amqp_spec,
exchange_name=amqp_exchange_name,
verbose=amqp_verbose)
service = TCPClient(amqp_host, amqp_port, factory)
service.setServiceParent(root_service)
if settings.ENABLE_MANHOLE:
from carbon import manhole
# Configure application components
if settings.RELAY_METHOD == 'rules':
router = RelayRulesRouter(settings["relay-rules"])
elif settings.RELAY_METHOD == 'consistent-hashing':
router = ConsistentHashingRouter(settings.REPLICATION_FACTOR)
elif settings.RELAY_METHOD == 'aggregated-consistent-hashing':
from carbon.aggregator.rules import RuleManager
RuleManager.read_from(settings["aggregation-rules"])
router = AggregatedConsistentHashingRouter(RuleManager, settings.REPLICATION_FACTOR)
elif settings.RELAY_METHOD == 'remove-node-consistent-hashing':
router = RemoveNodeConsistentHashingRouter(settings.REPLICATION_FACTOR, settings.REMOVE_NODE_INDEX)
factory = manhole.createManholeListener()
service = TCPServer(
settings.MANHOLE_PORT,
factory,
interface=settings.MANHOLE_INTERFACE)
service.setServiceParent(root_service)
def createAggregatorService(config):
from carbon.aggregator import receiver
from carbon.aggregator.rules import RuleManager
from carbon.routers import ConsistentHashingRouter
from carbon.client import CarbonClientManager
from carbon.rewrite import RewriteRuleManager
from carbon.conf import settings
from carbon import events
root_service = createBaseService(config)
# Configure application components
router = ConsistentHashingRouter()
client_manager = CarbonClientManager(router)
client_manager.setServiceParent(root_service)
events.metricReceived.addHandler(receiver.process)
events.metricGenerated.addHandler(client_manager.sendDatapoint)
RuleManager.read_from(settings["aggregation-rules"])
if exists(settings["rewrite-rules"]):
RewriteRuleManager.read_from(settings["rewrite-rules"])
if not settings.DESTINATIONS:
raise CarbonConfigException("Required setting DESTINATIONS is missing from carbon.conf")
for destination in util.parseDestinations(settings.DESTINATIONS):
client_manager.startClient(destination)
return root_service
def createAggregatorService(config):
from carbon.aggregator import receiver
from carbon.aggregator.rules import RuleManager
from carbon.routers import ConsistentHashingRouter
from carbon.client import CarbonClientManager
from carbon.rewrite import RewriteRuleManager
from carbon.conf import settings
from carbon import events
root_service = createBaseService(config)
# Configure application components
router = ConsistentHashingRouter(
settings.REPLICATION_FACTOR,
diverse_replicas=settings.DIVERSE_REPLICAS)
client_manager = CarbonClientManager(router)
client_manager.setServiceParent(root_service)
events.metricReceived.addHandler(receiver.process)
events.metricGenerated.addHandler(client_manager.sendDatapoint)
RuleManager.read_from(settings["aggregation-rules"])
if exists(settings["rewrite-rules"]):
RewriteRuleManager.read_from(settings["rewrite-rules"])
if not settings.DESTINATIONS:
raise CarbonConfigException(
"Required setting DESTINATIONS is missing from carbon.conf")
for destination in util.parseDestinations(settings.DESTINATIONS):
client_manager.startClient(destination)
return root_service
def createRelayService(config):
from carbon.routers import RelayRulesRouter, ConsistentHashingRouter, AggregatedConsistentHashingRouter
from carbon.client import CarbonClientManager
from carbon.conf import settings
from carbon import events
root_service = createBaseService(config)
# Configure application components
if settings.RELAY_METHOD == 'rules':
router = RelayRulesRouter(settings["relay-rules"])
elif settings.RELAY_METHOD == 'consistent-hashing':
router = ConsistentHashingRouter(
settings.REPLICATION_FACTOR,
diverse_replicas=settings.DIVERSE_REPLICAS)
elif settings.RELAY_METHOD == 'aggregated-consistent-hashing':
from carbon.aggregator.rules import RuleManager
RuleManager.read_from(settings["aggregation-rules"])
router = AggregatedConsistentHashingRouter(RuleManager,
settings.REPLICATION_FACTOR)
client_manager = CarbonClientManager(router)
client_manager.setServiceParent(root_service)
events.metricReceived.addHandler(client_manager.sendDatapoint)
events.metricGenerated.addHandler(client_manager.sendDatapoint)
if not settings.DESTINATIONS:
raise CarbonConfigException(
"Required setting DESTINATIONS is missing from carbon.conf")
for destination in util.parseDestinations(settings.DESTINATIONS):
client_manager.startClient(destination)
return root_service
def setupRelayProcessor(root_service, settings):
from carbon.routers import AggregatedConsistentHashingRouter, \
ConsistentHashingRouter, RelayRulesRouter
from carbon.client import CarbonClientManager
if settings.RELAY_METHOD == 'consistent-hashing':
router = ConsistentHashingRouter(
settings.REPLICATION_FACTOR,
diverse_replicas=settings.DIVERSE_REPLICAS)
elif settings.RELAY_METHOD == 'aggregated-consistent-hashing':
from carbon.aggregator.rules import RuleManager
aggregation_rules_path = settings["aggregation-rules"]
RuleManager.read_from(aggregation_rules_path)
router = AggregatedConsistentHashingRouter(
RuleManager,
settings.REPLICATION_FACTOR,
diverse_replicas=settings.DIVERSE_REPLICAS)
elif settings.RELAY_METHOD == 'rules':
router = RelayRulesRouter(settings["relay-rules"])
state.client_manager = CarbonClientManager(router)
state.client_manager.setServiceParent(root_service)
for destination in util.parseDestinations(settings.DESTINATIONS):
state.client_manager.startClient(destination)
def createRelayService(config):
from carbon.routers import RelayRulesRouter, ConsistentHashingRouter, AggregatedConsistentHashingRouter
from carbon.client import CarbonClientManager
from carbon.conf import settings
from carbon import events
root_service = createBaseService(config)
# Configure application components
if settings.RELAY_METHOD == 'rules':
router = RelayRulesRouter(settings["relay-rules"])
elif settings.RELAY_METHOD == 'consistent-hashing':
router = ConsistentHashingRouter(settings.REPLICATION_FACTOR)
elif settings.RELAY_METHOD == 'aggregated-consistent-hashing':
from carbon.aggregator.rules import RuleManager
RuleManager.read_from(settings["aggregation-rules"])
router = AggregatedConsistentHashingRouter(RuleManager, settings.REPLICATION_FACTOR)
client_manager = CarbonClientManager(router)
client_manager.setServiceParent(root_service)
events.metricReceived.addHandler(client_manager.sendDatapoint)
events.metricGenerated.addHandler(client_manager.sendDatapoint)
events.specialMetricReceived.addHandler(client_manager.sendHighPriorityDatapoint)
events.specialMetricGenerated.addHandler(client_manager.sendHighPriorityDatapoint)
if not settings.DESTINATIONS:
raise CarbonConfigException("Required setting DESTINATIONS is missing from carbon.conf")
for destination in util.parseDestinations(settings.DESTINATIONS):
client_manager.startClient(destination)
return root_service
def __init__(self, settings):
from carbon.aggregator.rules import RuleManager
aggregation_rules_path = settings["aggregation-rules"]
if aggregation_rules_path:
RuleManager.read_from(aggregation_rules_path)
self.hash_router = ConsistentHashingRouter(settings)
self.agg_rules_manager = RuleManager
def setupAggregatorProcessor(root_service, settings):
from carbon.aggregator.processor import AggregationProcessor # Register the plugin class
from carbon.aggregator.rules import RuleManager
aggregation_rules_path = settings["aggregation-rules"]
if not exists(aggregation_rules_path):
raise CarbonConfigException("aggregation processor: file does not exist {0}".format(aggregation_rules_path))
RuleManager.read_from(aggregation_rules_path)
def setupAggregatorProcessor(root_service, settings):
from carbon.aggregator.processor import AggregationProcessor # Register the plugin class
from carbon.aggregator.rules import RuleManager
aggregation_rules_path = settings["aggregation-rules"]
if not exists(aggregation_rules_path):
raise CarbonConfigException("aggregation processor: file does not exist {0}".format(aggregation_rules_path))
RuleManager.read_from(aggregation_rules_path)
def __init__(self, config, cluster='main', aggregation_rules=None):
relay_method = config.relay_method(cluster=cluster)
if relay_method == "consistent-hashing":
self.ring = ConsistentHashingRouter(config.replication_factor(cluster))
elif relay_method == "aggregated-consistent-hashing":
if aggregation_rules:
RuleManager.read_from(aggregation_rules)
self.ring = AggregatedConsistentHashingRouter(RuleManager, config.replication_factor(cluster))
try:
dest_list = config.destinations(cluster)
self.destinations = util.parseDestinations(dest_list)
except ValueError as e:
raise SystemExit("Unable to parse destinations!" + str(e))
for d in self.destinations:
self.ring.addDestination(d)
def setupRelayProcessor(root_service, settings):
from carbon.routers import AggregatedConsistentHashingRouter, \
ConsistentHashingRouter, RelayRulesRouter
from carbon.client import CarbonClientManager
if settings.RELAY_METHOD == 'consistent-hashing':
router = ConsistentHashingRouter(settings.REPLICATION_FACTOR)
elif settings.RELAY_METHOD == 'aggregated-consistent-hashing':
from carbon.aggregator.rules import RuleManager
aggregation_rules_path = settings["aggregation-rules"]
RuleManager.read_from(aggregation_rules_path)
router = AggregatedConsistentHashingRouter(RuleManager, settings.REPLICATION_FACTOR)
elif settings.RELAY_METHOD == 'rules':
router = RelayRulesRouter(settings["relay-rules"])
state.client_manager = CarbonClientManager(router)
state.client_manager.setServiceParent(root_service)
for destination in util.parseDestinations(settings.DESTINATIONS):
state.client_manager.startClient(destination)
def createAggregatorService(config):
from carbon.events import metricReceived
from carbon.aggregator import receiver
from carbon.aggregator.rules import RuleManager
from carbon.aggregator import client
from carbon.rewrite import RewriteRuleManager
from carbon.conf import settings
root_service = createBaseService(config)
# Configure application components
metricReceived.installHandler(receiver.process)
RuleManager.read_from(settings["aggregation-rules"])
if exists(settings["rewrite-rules"]):
RewriteRuleManager.read_from(settings["rewrite-rules"])
client.connect(settings["DESTINATION_HOST"],
int(settings["DESTINATION_PORT"]))
return root_service
def __init__(self, config, cluster='main', aggregation_rules=None):
relay_method = config.relay_method(cluster=cluster)
if relay_method == "consistent-hashing":
# Support multiple versions of carbon, the API changed in 0.10.
args = inspect.getargspec(ConsistentHashingRouter.__init__).args
if 'replication_factor' in args:
r = ConsistentHashingRouter(config.replication_factor(cluster))
else:
class Settings(object):
REPLICATION_FACTOR = config.replication_factor(cluster)
DIVERSE_REPLICAS = False
r = ConsistentHashingRouter(Settings())
# 'hash_type' was added only in carbon 1.0.2 or master
args = inspect.getargspec(ConsistentHashRing.__init__).args
if 'hash_type' in args:
r.ring = ConsistentHashRing(
hash_type=config.hashing_type(cluster))
elif relay_method == "aggregated-consistent-hashing":
if aggregation_rules:
RuleManager.read_from(aggregation_rules)
r = AggregatedConsistentHashingRouter(
RuleManager, config.replication_factor(cluster))
self.ring = r
try:
dest_list = config.destinations(cluster)
self.destinations = util.parseDestinations(dest_list)
except ValueError as e:
raise SystemExit("Unable to parse destinations!" + str(e))
for d in self.destinations:
self.ring.addDestination(d)
def tearDown(self):
instrumentation.stats.clear()
BufferManager.clear()
RuleManager.clear()
def tearDown(self): instrumentation.stats.clear() BufferManager.clear() RuleManager.clear()
from carbon.conf import settings
settings.readFrom(options.config, 'aggregator')
# Import application components
from carbon.log import logToStdout, logToDir
from carbon.instrumentation import startRecording
from carbon.listeners import MetricLineReceiver, MetricPickleReceiver, startListener
from carbon.aggregator.rules import RuleManager
from carbon.aggregator import receiver
from carbon.aggregator import client
from carbon.rewrite import RewriteRuleManager
from carbon.events import metricReceived
from carbon.util import daemonize
RuleManager.read_from(options.rules)
rewrite_rules_conf = join(CONF_DIR, 'rewrite-rules.conf')
if exists(rewrite_rules_conf):
RewriteRuleManager.read_from(rewrite_rules_conf)
# --debug
if options.debug:
logToStdout()
else:
if not isdir(options.logdir):
os.makedirs(options.logdir)
daemonize()
# Read config (we want failures to occur before daemonizing)
from carbon.conf import settings
settings.readFrom(options.config, 'aggregator')
# Import application components
from carbon.log import logToStdout, logToDir
from carbon.instrumentation import startRecording
from carbon.listeners import MetricLineReceiver, MetricPickleReceiver, startListener
from carbon.aggregator.rules import RuleManager
from carbon.aggregator import receiver
from carbon.aggregator import client
from carbon.rewrite import RewriteRuleManager
from carbon.events import metricReceived
from carbon.util import daemonize
RuleManager.read_from(options.rules)
rewrite_rules_conf = join(CONF_DIR, 'rewrite-rules.conf')
if exists(rewrite_rules_conf):
RewriteRuleManager.read_from(rewrite_rules_conf)
# --debug
if options.debug:
logToStdout()
else:
if not isdir(options.logdir):
os.makedirs(options.logdir)
daemonize()
