def process(self, metric, datapoint): increment('datapointsReceived') for rule in RewriteRuleManager.rules(PRE): metric = rule.apply(metric) aggregate_metrics = set() for rule in RuleManager.rules: aggregate_metric = rule.get_aggregate_metric(metric) if aggregate_metric is None: continue else: aggregate_metrics.add(aggregate_metric) values_buffer = BufferManager.get_buffer(aggregate_metric) if not values_buffer.configured: values_buffer.configure_aggregation(rule.frequency, rule.aggregation_func) values_buffer.input(datapoint) for rule in RewriteRuleManager.rules(POST): metric = rule.apply(metric) if metric not in aggregate_metrics: yield (metric, datapoint)
def process(self, metric, datapoint): increment('datapointsReceived') for rule in RewriteRuleManager.rules(PRE): metric = rule.apply(metric) aggregate_metrics = set() for rule in RuleManager.rules: aggregate_metric = rule.get_aggregate_metric(metric) if aggregate_metric is None: continue else: aggregate_metrics.add(aggregate_metric) values_buffer = BufferManager.get_buffer(aggregate_metric) if not values_buffer.configured: values_buffer.configure_aggregation(rule.frequency, rule.aggregation_func) values_buffer.input(datapoint) for rule in RewriteRuleManager.rules(POST): metric = rule.apply(metric) if metric not in aggregate_metrics: if settings.LOG_AGGREGATOR_MISSES and len(aggregate_metrics) == 0: log.msg( "Couldn't match metric %s with any aggregation rule. Passing on un-aggregated." % metric) yield (metric, datapoint)
def createAggregatorService(config): from carbon.aggregator import receiver from carbon.aggregator.rules import RuleManager from carbon.routers import ConsistentHashingRouter from carbon.client import CarbonClientManager from carbon.rewrite import RewriteRuleManager from carbon.conf import settings from carbon import events root_service = createBaseService(config) # Configure application components router = ConsistentHashingRouter() client_manager = CarbonClientManager(router) client_manager.setServiceParent(root_service) events.metricReceived.addHandler(receiver.process) events.metricGenerated.addHandler(client_manager.sendDatapoint) RuleManager.read_from(settings["aggregation-rules"]) if exists(settings["rewrite-rules"]): RewriteRuleManager.read_from(settings["rewrite-rules"]) if not settings.DESTINATIONS: raise CarbonConfigException("Required setting DESTINATIONS is missing from carbon.conf") for destination in util.parseDestinations(settings.DESTINATIONS): client_manager.startClient(destination) return root_service
def process(self, metric, datapoint): increment('datapointsReceived') for rule in RewriteRuleManager.rules(PRE): metric = rule.apply(metric) aggregate_metrics = set() for rule in RuleManager.rules: aggregate_metric = rule.get_aggregate_metric(metric) if aggregate_metric is None: continue else: aggregate_metrics.add(aggregate_metric) values_buffer = BufferManager.get_buffer(aggregate_metric) if not values_buffer.configured: values_buffer.configure_aggregation(rule.frequency, rule.aggregation_func) values_buffer.input(datapoint) for rule in RewriteRuleManager.rules(POST): metric = rule.apply(metric) if settings.FORWARD_ALL and metric not in aggregate_metrics: if settings.LOG_AGGREGATOR_MISSES and len(aggregate_metrics) == 0: log.msg( "Couldn't match metric %s with any aggregation rule. Passing on un-aggregated." % metric) yield (metric, datapoint)
def createAggregatorService(config): from carbon.aggregator import receiver from carbon.aggregator.rules import RuleManager from carbon.routers import ConsistentHashingRouter from carbon.client import CarbonClientManager from carbon.rewrite import RewriteRuleManager from carbon.conf import settings from carbon import events root_service = createBaseService(config) # Configure application components router = ConsistentHashingRouter( settings.REPLICATION_FACTOR, diverse_replicas=settings.DIVERSE_REPLICAS) client_manager = CarbonClientManager(router) client_manager.setServiceParent(root_service) events.metricReceived.addHandler(receiver.process) events.metricGenerated.addHandler(client_manager.sendDatapoint) RuleManager.read_from(settings["aggregation-rules"]) if exists(settings["rewrite-rules"]): RewriteRuleManager.read_from(settings["rewrite-rules"]) if not settings.DESTINATIONS: raise CarbonConfigException( "Required setting DESTINATIONS is missing from carbon.conf") for destination in util.parseDestinations(settings.DESTINATIONS): client_manager.startClient(destination) return root_service
def test_read_clears_if_no_file(self): import carbon.rewrite RewriteRuleManager.rules_file = 'foo.conf' with patch.object(carbon.rewrite, 'exists', Mock(return_value=False)): with patch.object(RewriteRuleManager, 'clear') as clear_mock: RewriteRuleManager.read_rules() clear_mock.assert_called_once_with()
def test_read_records_mtime(self): import carbon.rewrite RewriteRuleManager.rules_file = 'foo.conf' with patch.object(carbon.rewrite, 'open', mock_open(), create=True): with patch.object(carbon.rewrite, 'exists', Mock(return_value=True)): with patch.object(carbon.rewrite, 'getmtime', Mock(return_value=1234)): RewriteRuleManager.read_rules() self.assertEqual(1234, RewriteRuleManager.rules_last_read)
def test_read_opens_file_if_mtime_newer(self): import carbon.rewrite RewriteRuleManager.rules_last_read = 1234 RewriteRuleManager.rules_file = 'foo.conf' with patch.object(carbon.rewrite, 'open', mock_open(), create=True) as open_mock: with patch.object(carbon.rewrite, 'exists', Mock(return_value=True)): with patch.object(carbon.rewrite, 'getmtime', Mock(return_value=5678)): RewriteRuleManager.read_rules() self.assertTrue(open_mock.called)
def test_read_doesnt_open_file_if_mtime_unchanged(self): import carbon.rewrite mtime = 1234 RewriteRuleManager.rules_last_read = mtime RewriteRuleManager.rules_file = 'foo.conf' with patch.object(carbon.rewrite, 'open', mock_open(), create=True) as open_mock: with patch.object(carbon.rewrite, 'exists', Mock(return_value=True)): with patch.object(carbon.rewrite, 'getmtime', Mock(return_value=1234)): RewriteRuleManager.read_rules() self.assertFalse(open_mock.called)
def test_section_parsed_into_ruleset(self): import carbon.rewrite open_mock = Mock(return_value=iter(self.sample_config.splitlines())) RewriteRuleManager.rules_file = 'foo.conf' with patch.object(carbon.rewrite, 'open', open_mock, create=True): with patch.object(carbon.rewrite, 'exists', Mock(return_value=True)): with patch.object(carbon.rewrite, 'getmtime', Mock(return_value=1234)): RewriteRuleManager.read_rules() self.assertTrue('pre' in RewriteRuleManager.rulesets)
def test_comments_ignored(self): import carbon.rewrite open_mock = Mock(return_value=iter(self.commented_config.splitlines())) RewriteRuleManager.rules_file = 'foo.conf' with patch.object(carbon.rewrite, 'open', open_mock, create=True): with patch.object(carbon.rewrite, 'exists', Mock(return_value=True)): with patch.object(carbon.rewrite, 'getmtime', Mock(return_value=1234)): RewriteRuleManager.read_rules() self.assertEqual(1, len(RewriteRuleManager.rules('pre')))
def test_rules_unchanged_if_mtime_unchanged(self): import carbon.rewrite mtime = 1234 rulesets = {'pre': [Mock(RewriteRule)]} RewriteRuleManager.rules_last_read = mtime RewriteRuleManager.rulesets.update(rulesets) RewriteRuleManager.rules_file = 'foo.conf' with patch.object(carbon.rewrite, 'exists', Mock(return_value=True)): with patch.object(carbon.rewrite, 'getmtime', Mock(return_value=mtime)): RewriteRuleManager.read_rules() self.assertEqual(rulesets, RewriteRuleManager.rulesets)
def createAggregatorService(config): from carbon.events import metricReceived from carbon.aggregator import receiver from carbon.aggregator.rules import RuleManager from carbon.aggregator import client from carbon.rewrite import RewriteRuleManager from carbon.conf import settings root_service = createBaseService(config) # Configure application components metricReceived.installHandler(receiver.process) RuleManager.read_from(settings["aggregation-rules"]) if exists(settings["rewrite-rules"]): RewriteRuleManager.read_from(settings["rewrite-rules"]) client.connect(settings["DESTINATION_HOST"], int(settings["DESTINATION_PORT"])) return root_service
# Import application components from carbon.log import logToStdout, logToDir from carbon.instrumentation import startRecording from carbon.listeners import MetricLineReceiver, MetricPickleReceiver, startListener from carbon.aggregator.rules import RuleManager from carbon.aggregator import receiver from carbon.aggregator import client from carbon.rewrite import RewriteRuleManager from carbon.events import metricReceived from carbon.util import daemonize RuleManager.read_from(options.rules) rewrite_rules_conf = join(CONF_DIR, 'rewrite-rules.conf') if exists(rewrite_rules_conf): RewriteRuleManager.read_from(rewrite_rules_conf) # --debug if options.debug: logToStdout() else: if not isdir(options.logdir): os.makedirs(options.logdir) daemonize() pidfile = open(options.pidfile, 'w') pidfile.write( str(os.getpid()) ) pidfile.close()
def test_read_from_starts_task(self): with patch.object(RewriteRuleManager, 'read_rules'): with patch.object(RewriteRuleManager.read_task, 'start') as task_start_mock: RewriteRuleManager.read_from('foo.conf') self.assertEqual(1, task_start_mock.call_count)
# Import application components from carbon.log import logToStdout, logToDir from carbon.instrumentation import startRecording from carbon.listeners import MetricLineReceiver, MetricPickleReceiver, startListener from carbon.aggregator.rules import RuleManager from carbon.aggregator import receiver from carbon.aggregator import client from carbon.rewrite import RewriteRuleManager from carbon.events import metricReceived from carbon.util import daemonize RuleManager.read_from(options.rules) rewrite_rules_conf = join(CONF_DIR, 'rewrite-rules.conf') if exists(rewrite_rules_conf): RewriteRuleManager.read_from(rewrite_rules_conf) # --debug if options.debug: logToStdout() else: if not isdir(options.logdir): os.makedirs(options.logdir) daemonize() pidfile = open(options.pidfile, 'w') pidfile.write(str(os.getpid())) pidfile.close()
def tearDown(self): RewriteRuleManager.clear()
def setupRewriterProcessor(root_service, settings): from carbon.rewrite import RewriteRuleManager rewrite_rules_path = settings["rewrite-rules"] RewriteRuleManager.read_from(rewrite_rules_path)
def tearDown(self): instrumentation.stats.clear() BufferManager.clear() RuleManager.clear() RewriteRuleManager.clear()
def test_request_for_nonexistent_rules_returns_iterable(self): try: iter(RewriteRuleManager.rules('foo')) except TypeError: self.fail("RewriteRuleManager.rules() returned a non-iterable type")
def test_read_from_starts_task(self): with patch.object(RewriteRuleManager, 'read_rules'): with patch.object(RewriteRuleManager.read_task, 'start') as task_start_mock: RewriteRuleManager.read_from('foo.conf') task_start_mock.assert_called_once()
def tearDown(self): RewriteRuleManager.rules_file = None RewriteRuleManager.rules_last_read = 0.0 RewriteRuleManager.clear()