def process(self, metric, datapoint):
increment('datapointsReceived')
for rule in RewriteRuleManager.rules(PRE):
metric = rule.apply(metric)
aggregate_metrics = set()
for rule in RuleManager.rules:
aggregate_metric = rule.get_aggregate_metric(metric)
if aggregate_metric is None:
continue
else:
aggregate_metrics.add(aggregate_metric)
values_buffer = BufferManager.get_buffer(aggregate_metric)
if not values_buffer.configured:
values_buffer.configure_aggregation(rule.frequency, rule.aggregation_func)
values_buffer.input(datapoint)
for rule in RewriteRuleManager.rules(POST):
metric = rule.apply(metric)
if metric not in aggregate_metrics:
yield (metric, datapoint)
def process(self, metric, datapoint):
increment('datapointsReceived')
for rule in RewriteRuleManager.rules(PRE):
metric = rule.apply(metric)
aggregate_metrics = set()
for rule in RuleManager.rules:
aggregate_metric = rule.get_aggregate_metric(metric)
if aggregate_metric is None:
continue
else:
aggregate_metrics.add(aggregate_metric)
values_buffer = BufferManager.get_buffer(aggregate_metric)
if not values_buffer.configured:
values_buffer.configure_aggregation(rule.frequency,
rule.aggregation_func)
values_buffer.input(datapoint)
for rule in RewriteRuleManager.rules(POST):
metric = rule.apply(metric)
if metric not in aggregate_metrics:
if settings.LOG_AGGREGATOR_MISSES and len(aggregate_metrics) == 0:
log.msg(
"Couldn't match metric %s with any aggregation rule. Passing on un-aggregated."
% metric)
yield (metric, datapoint)
def process(self, metric, datapoint):
increment('datapointsReceived')
for rule in RewriteRuleManager.rules(PRE):
metric = rule.apply(metric)
aggregate_metrics = set()
for rule in RuleManager.rules:
aggregate_metric = rule.get_aggregate_metric(metric)
if aggregate_metric is None:
continue
else:
aggregate_metrics.add(aggregate_metric)
values_buffer = BufferManager.get_buffer(aggregate_metric)
if not values_buffer.configured:
values_buffer.configure_aggregation(rule.frequency,
rule.aggregation_func)
values_buffer.input(datapoint)
for rule in RewriteRuleManager.rules(POST):
metric = rule.apply(metric)
if metric not in aggregate_metrics:
yield (metric, datapoint)
def createAggregatorService(config):
from carbon.aggregator import receiver
from carbon.aggregator.rules import RuleManager
from carbon.routers import ConsistentHashingRouter
from carbon.client import CarbonClientManager
from carbon.rewrite import RewriteRuleManager
from carbon.conf import settings
from carbon import events
root_service = createBaseService(config)
# Configure application components
router = ConsistentHashingRouter()
client_manager = CarbonClientManager(router)
client_manager.setServiceParent(root_service)
events.metricReceived.addHandler(receiver.process)
events.metricGenerated.addHandler(client_manager.sendDatapoint)
RuleManager.read_from(settings["aggregation-rules"])
if exists(settings["rewrite-rules"]):
RewriteRuleManager.read_from(settings["rewrite-rules"])
if not settings.DESTINATIONS:
raise CarbonConfigException("Required setting DESTINATIONS is missing from carbon.conf")
for destination in util.parseDestinations(settings.DESTINATIONS):
client_manager.startClient(destination)
return root_service
def process(self, metric, datapoint):
increment('datapointsReceived')
for rule in RewriteRuleManager.rules(PRE):
metric = rule.apply(metric)
aggregate_metrics = set()
for rule in RuleManager.rules:
aggregate_metric = rule.get_aggregate_metric(metric)
if aggregate_metric is None:
continue
else:
aggregate_metrics.add(aggregate_metric)
values_buffer = BufferManager.get_buffer(aggregate_metric)
if not values_buffer.configured:
values_buffer.configure_aggregation(rule.frequency, rule.aggregation_func)
values_buffer.input(datapoint)
for rule in RewriteRuleManager.rules(POST):
metric = rule.apply(metric)
if settings.FORWARD_ALL and metric not in aggregate_metrics:
if settings.LOG_AGGREGATOR_MISSES and len(aggregate_metrics) == 0:
log.msg(
"Couldn't match metric %s with any aggregation rule. Passing on un-aggregated." % metric)
yield (metric, datapoint)
def createAggregatorService(config):
from carbon.aggregator import receiver
from carbon.aggregator.rules import RuleManager
from carbon.routers import ConsistentHashingRouter
from carbon.client import CarbonClientManager
from carbon.rewrite import RewriteRuleManager
from carbon.conf import settings
from carbon import events
root_service = createBaseService(config)
# Configure application components
router = ConsistentHashingRouter(
settings.REPLICATION_FACTOR,
diverse_replicas=settings.DIVERSE_REPLICAS)
client_manager = CarbonClientManager(router)
client_manager.setServiceParent(root_service)
events.metricReceived.addHandler(receiver.process)
events.metricGenerated.addHandler(client_manager.sendDatapoint)
RuleManager.read_from(settings["aggregation-rules"])
if exists(settings["rewrite-rules"]):
RewriteRuleManager.read_from(settings["rewrite-rules"])
if not settings.DESTINATIONS:
raise CarbonConfigException(
"Required setting DESTINATIONS is missing from carbon.conf")
for destination in util.parseDestinations(settings.DESTINATIONS):
client_manager.startClient(destination)
return root_service
def test_read_clears_if_no_file(self):
import carbon.rewrite
RewriteRuleManager.rules_file = 'foo.conf'
with patch.object(carbon.rewrite, 'exists', Mock(return_value=False)):
with patch.object(RewriteRuleManager, 'clear') as clear_mock:
RewriteRuleManager.read_rules()
clear_mock.assert_called_once_with()
def test_read_records_mtime(self):
import carbon.rewrite
RewriteRuleManager.rules_file = 'foo.conf'
with patch.object(carbon.rewrite, 'open', mock_open(), create=True):
with patch.object(carbon.rewrite, 'exists', Mock(return_value=True)):
with patch.object(carbon.rewrite, 'getmtime', Mock(return_value=1234)):
RewriteRuleManager.read_rules()
self.assertEqual(1234, RewriteRuleManager.rules_last_read)
def test_read_opens_file_if_mtime_newer(self):
import carbon.rewrite
RewriteRuleManager.rules_last_read = 1234
RewriteRuleManager.rules_file = 'foo.conf'
with patch.object(carbon.rewrite, 'open', mock_open(), create=True) as open_mock:
with patch.object(carbon.rewrite, 'exists', Mock(return_value=True)):
with patch.object(carbon.rewrite, 'getmtime', Mock(return_value=5678)):
RewriteRuleManager.read_rules()
self.assertTrue(open_mock.called)
def test_read_doesnt_open_file_if_mtime_unchanged(self):
import carbon.rewrite
mtime = 1234
RewriteRuleManager.rules_last_read = mtime
RewriteRuleManager.rules_file = 'foo.conf'
with patch.object(carbon.rewrite, 'open', mock_open(), create=True) as open_mock:
with patch.object(carbon.rewrite, 'exists', Mock(return_value=True)):
with patch.object(carbon.rewrite, 'getmtime', Mock(return_value=1234)):
RewriteRuleManager.read_rules()
self.assertFalse(open_mock.called)
def test_section_parsed_into_ruleset(self):
import carbon.rewrite
open_mock = Mock(return_value=iter(self.sample_config.splitlines()))
RewriteRuleManager.rules_file = 'foo.conf'
with patch.object(carbon.rewrite, 'open', open_mock, create=True):
with patch.object(carbon.rewrite, 'exists', Mock(return_value=True)):
with patch.object(carbon.rewrite, 'getmtime', Mock(return_value=1234)):
RewriteRuleManager.read_rules()
self.assertTrue('pre' in RewriteRuleManager.rulesets)
def test_comments_ignored(self):
import carbon.rewrite
open_mock = Mock(return_value=iter(self.commented_config.splitlines()))
RewriteRuleManager.rules_file = 'foo.conf'
with patch.object(carbon.rewrite, 'open', open_mock, create=True):
with patch.object(carbon.rewrite, 'exists', Mock(return_value=True)):
with patch.object(carbon.rewrite, 'getmtime', Mock(return_value=1234)):
RewriteRuleManager.read_rules()
self.assertEqual(1, len(RewriteRuleManager.rules('pre')))
def test_rules_unchanged_if_mtime_unchanged(self):
import carbon.rewrite
mtime = 1234
rulesets = {'pre': [Mock(RewriteRule)]}
RewriteRuleManager.rules_last_read = mtime
RewriteRuleManager.rulesets.update(rulesets)
RewriteRuleManager.rules_file = 'foo.conf'
with patch.object(carbon.rewrite, 'exists', Mock(return_value=True)):
with patch.object(carbon.rewrite, 'getmtime', Mock(return_value=mtime)):
RewriteRuleManager.read_rules()
self.assertEqual(rulesets, RewriteRuleManager.rulesets)
def createAggregatorService(config):
from carbon.events import metricReceived
from carbon.aggregator import receiver
from carbon.aggregator.rules import RuleManager
from carbon.aggregator import client
from carbon.rewrite import RewriteRuleManager
from carbon.conf import settings
root_service = createBaseService(config)
# Configure application components
metricReceived.installHandler(receiver.process)
RuleManager.read_from(settings["aggregation-rules"])
if exists(settings["rewrite-rules"]):
RewriteRuleManager.read_from(settings["rewrite-rules"])
client.connect(settings["DESTINATION_HOST"],
int(settings["DESTINATION_PORT"]))
return root_service
# Import application components
from carbon.log import logToStdout, logToDir
from carbon.instrumentation import startRecording
from carbon.listeners import MetricLineReceiver, MetricPickleReceiver, startListener
from carbon.aggregator.rules import RuleManager
from carbon.aggregator import receiver
from carbon.aggregator import client
from carbon.rewrite import RewriteRuleManager
from carbon.events import metricReceived
from carbon.util import daemonize
RuleManager.read_from(options.rules)
rewrite_rules_conf = join(CONF_DIR, 'rewrite-rules.conf')
if exists(rewrite_rules_conf):
RewriteRuleManager.read_from(rewrite_rules_conf)
# --debug
if options.debug:
logToStdout()
else:
if not isdir(options.logdir):
os.makedirs(options.logdir)
daemonize()
pidfile = open(options.pidfile, 'w')
pidfile.write( str(os.getpid()) )
pidfile.close()
def test_read_from_starts_task(self):
with patch.object(RewriteRuleManager, 'read_rules'):
with patch.object(RewriteRuleManager.read_task, 'start') as task_start_mock:
RewriteRuleManager.read_from('foo.conf')
self.assertEqual(1, task_start_mock.call_count)
# Import application components
from carbon.log import logToStdout, logToDir
from carbon.instrumentation import startRecording
from carbon.listeners import MetricLineReceiver, MetricPickleReceiver, startListener
from carbon.aggregator.rules import RuleManager
from carbon.aggregator import receiver
from carbon.aggregator import client
from carbon.rewrite import RewriteRuleManager
from carbon.events import metricReceived
from carbon.util import daemonize
RuleManager.read_from(options.rules)
rewrite_rules_conf = join(CONF_DIR, 'rewrite-rules.conf')
if exists(rewrite_rules_conf):
RewriteRuleManager.read_from(rewrite_rules_conf)
# --debug
if options.debug:
logToStdout()
else:
if not isdir(options.logdir):
os.makedirs(options.logdir)
daemonize()
pidfile = open(options.pidfile, 'w')
pidfile.write(str(os.getpid()))
pidfile.close()
def tearDown(self): RewriteRuleManager.clear()
def setupRewriterProcessor(root_service, settings): from carbon.rewrite import RewriteRuleManager rewrite_rules_path = settings["rewrite-rules"] RewriteRuleManager.read_from(rewrite_rules_path)
def tearDown(self):
instrumentation.stats.clear()
BufferManager.clear()
RuleManager.clear()
RewriteRuleManager.clear()
def test_request_for_nonexistent_rules_returns_iterable(self):
try:
iter(RewriteRuleManager.rules('foo'))
except TypeError:
self.fail("RewriteRuleManager.rules() returned a non-iterable type")
def tearDown(self): instrumentation.stats.clear() BufferManager.clear() RuleManager.clear() RewriteRuleManager.clear()
def test_read_from_starts_task(self):
with patch.object(RewriteRuleManager, 'read_rules'):
with patch.object(RewriteRuleManager.read_task, 'start') as task_start_mock:
RewriteRuleManager.read_from('foo.conf')
task_start_mock.assert_called_once()
def tearDown(self): RewriteRuleManager.rules_file = None RewriteRuleManager.rules_last_read = 0.0 RewriteRuleManager.clear()
def test_read_from_starts_task(self):
with patch.object(RewriteRuleManager, 'read_rules'):
with patch.object(RewriteRuleManager.read_task,
'start') as task_start_mock:
RewriteRuleManager.read_from('foo.conf')
self.assertEqual(1, task_start_mock.call_count)
def setupRewriterProcessor(root_service, settings):
from carbon.rewrite import RewriteRuleManager
rewrite_rules_path = settings["rewrite-rules"]
RewriteRuleManager.read_from(rewrite_rules_path)
