def test_wrong_dest(self): # Some cases of invalid input, e.g. invalid/missing port. dests = [ "1.2.3.4", # No port "1.2.3.4:huh", # Invalid port (must be int) "[fe80::3285:a9ff:fe91:e287]", # No port "[ffff::1.2.3.4]:notaport" # Invalid port ] for dest in dests: try: parseDestinations([dest]) except ValueError: continue raise AssertionError("Invalid input was accepted.")
def createAggregatorService(config): from carbon.aggregator import receiver from carbon.aggregator.rules import RuleManager from carbon.routers import ConsistentHashingRouter from carbon.client import CarbonClientManager from carbon.rewrite import RewriteRuleManager from carbon.conf import settings from carbon import events root_service = createBaseService(config) # Configure application components router = ConsistentHashingRouter() client_manager = CarbonClientManager(router) client_manager.setServiceParent(root_service) events.metricReceived.addHandler(receiver.process) events.metricGenerated.addHandler(client_manager.sendDatapoint) RuleManager.read_from(settings["aggregation-rules"]) if exists(settings["rewrite-rules"]): RewriteRuleManager.read_from(settings["rewrite-rules"]) if not settings.DESTINATIONS: raise CarbonConfigException("Required setting DESTINATIONS is missing from carbon.conf") for destination in util.parseDestinations(settings.DESTINATIONS): client_manager.startClient(destination) return root_service
def createRelayService(config): from carbon.routers import RelayRulesRouter, ConsistentHashingRouter, AggregatedConsistentHashingRouter from carbon.client import CarbonClientManager from carbon.conf import settings from carbon import events root_service = createBaseService(config) # Configure application components if settings.RELAY_METHOD == 'rules': router = RelayRulesRouter(settings["relay-rules"]) elif settings.RELAY_METHOD == 'consistent-hashing': router = ConsistentHashingRouter(settings.REPLICATION_FACTOR) elif settings.RELAY_METHOD == 'aggregated-consistent-hashing': from carbon.aggregator.rules import RuleManager RuleManager.read_from(settings["aggregation-rules"]) router = AggregatedConsistentHashingRouter(RuleManager, settings.REPLICATION_FACTOR) client_manager = CarbonClientManager(router) client_manager.setServiceParent(root_service) events.metricReceived.addHandler(client_manager.sendDatapoint) events.metricGenerated.addHandler(client_manager.sendDatapoint) events.specialMetricReceived.addHandler(client_manager.sendHighPriorityDatapoint) events.specialMetricGenerated.addHandler(client_manager.sendHighPriorityDatapoint) if not settings.DESTINATIONS: raise CarbonConfigException("Required setting DESTINATIONS is missing from carbon.conf") for destination in util.parseDestinations(settings.DESTINATIONS): client_manager.startClient(destination) return root_service
def createAggregatorService(config): from carbon.aggregator import receiver from carbon.aggregator.rules import RuleManager from carbon.routers import ConsistentHashingRouter from carbon.client import CarbonClientManager from carbon.rewrite import RewriteRuleManager from carbon.conf import settings from carbon import events root_service = createBaseService(config) # Configure application components router = ConsistentHashingRouter( settings.REPLICATION_FACTOR, diverse_replicas=settings.DIVERSE_REPLICAS) client_manager = CarbonClientManager(router) client_manager.setServiceParent(root_service) events.metricReceived.addHandler(receiver.process) events.metricGenerated.addHandler(client_manager.sendDatapoint) RuleManager.read_from(settings["aggregation-rules"]) if exists(settings["rewrite-rules"]): RewriteRuleManager.read_from(settings["rewrite-rules"]) if not settings.DESTINATIONS: raise CarbonConfigException( "Required setting DESTINATIONS is missing from carbon.conf") for destination in util.parseDestinations(settings.DESTINATIONS): client_manager.startClient(destination) return root_service
def createRelayService(config): from carbon.routers import RelayRulesRouter, ConsistentHashingRouter, AggregatedConsistentHashingRouter from carbon.client import CarbonClientManager from carbon.conf import settings from carbon import events root_service = createBaseService(config) # Configure application components if settings.RELAY_METHOD == 'rules': router = RelayRulesRouter(settings["relay-rules"]) elif settings.RELAY_METHOD == 'consistent-hashing': router = ConsistentHashingRouter( settings.REPLICATION_FACTOR, diverse_replicas=settings.DIVERSE_REPLICAS) elif settings.RELAY_METHOD == 'aggregated-consistent-hashing': from carbon.aggregator.rules import RuleManager RuleManager.read_from(settings["aggregation-rules"]) router = AggregatedConsistentHashingRouter(RuleManager, settings.REPLICATION_FACTOR) client_manager = CarbonClientManager(router) client_manager.setServiceParent(root_service) events.metricReceived.addHandler(client_manager.sendDatapoint) events.metricGenerated.addHandler(client_manager.sendDatapoint) if not settings.DESTINATIONS: raise CarbonConfigException( "Required setting DESTINATIONS is missing from carbon.conf") for destination in util.parseDestinations(settings.DESTINATIONS): client_manager.startClient(destination) return root_service
def setupRelayProcessor(root_service, settings): from carbon.routers import AggregatedConsistentHashingRouter, \ ConsistentHashingRouter, RelayRulesRouter from carbon.client import CarbonClientManager if settings.RELAY_METHOD == 'consistent-hashing': router = ConsistentHashingRouter( settings.REPLICATION_FACTOR, diverse_replicas=settings.DIVERSE_REPLICAS) elif settings.RELAY_METHOD == 'aggregated-consistent-hashing': from carbon.aggregator.rules import RuleManager aggregation_rules_path = settings["aggregation-rules"] RuleManager.read_from(aggregation_rules_path) router = AggregatedConsistentHashingRouter( RuleManager, settings.REPLICATION_FACTOR, diverse_replicas=settings.DIVERSE_REPLICAS) elif settings.RELAY_METHOD == 'rules': router = RelayRulesRouter(settings["relay-rules"]) state.client_manager = CarbonClientManager(router) state.client_manager.setServiceParent(root_service) for destination in util.parseDestinations(settings.DESTINATIONS): state.client_manager.startClient(destination)
def __init__(self, config, cluster='main'): # Support multiple versions of carbon, the API changed in 0.10. args = inspect.getargspec(ConsistentHashingRouter.__init__).args if 'replication_factor' in args: r = ConsistentHashingRouter(config.replication_factor(cluster)) else: class Settings(object): REPLICATION_FACTOR = config.replication_factor(cluster) DIVERSE_REPLICAS = False ROUTER_HASH_TYPE = None r = ConsistentHashingRouter(Settings()) # 'hash_type' was added only in carbon 1.0.2 or master args = inspect.getargspec(ConsistentHashRing.__init__).args if 'hash_type' in args: r.ring = ConsistentHashRing(nodes=[], hash_type=config.hashing_type(cluster)) self.ring = r try: dest_list = config.destinations(cluster) self.destinations = util.parseDestinations(dest_list) except ValueError as e: raise SystemExit("Unable to parse destinations!" + str(e)) for d in self.destinations: self.ring.addDestination(d)
def _get_nodes(config, cluster_name, node=None): """Get the list of nodes in another cluster, excludes 'node'.""" res = set() for entry in util.parseDestinations(config.destinations(cluster_name)): res.add(entry[0]) # Only keep the host. if node in res: res.remove(node) return res
def __init__(self, config, cluster='main'): self.ring = ConsistentHashingRouter(config.replication_factor(cluster)) try: dest_list = config.destinations(cluster) self.destinations = util.parseDestinations(dest_list) except ValueError as e: raise SystemExit("Unable to parse destinations!" + str(e)) for d in self.destinations: self.ring.addDestination(d)
def setupRelayProcessor(root_service, settings): from carbon.routers import DatapointRouter from carbon.client import CarbonClientManager router_class = DatapointRouter.plugins[settings.RELAY_METHOD] router = router_class(settings) state.client_manager = CarbonClientManager(router) state.client_manager.setServiceParent(root_service) for destination in util.parseDestinations(settings.DESTINATIONS): state.client_manager.startClient(destination)
def loadRelayRules(path): rules = [] parser = OrderedConfigParser() if not parser.read(path): raise CarbonConfigException("Could not read rules file %s" % path) defaultRule = None for section in parser.sections(): if not parser.has_option(section, 'destinations'): raise CarbonConfigException( "Rules file %s section %s does not define a " "'destinations' list" % (path, section)) destination_strings = parser.get(section, 'destinations').split(',') destinations = parseDestinations(destination_strings) if parser.has_option(section, 'pattern'): if parser.has_option(section, 'default'): raise CarbonConfigException( "Section %s contains both 'pattern' and " "'default'. You must use one or the other." % section) pattern = parser.get(section, 'pattern') regex = re.compile(pattern, re.I) continue_matching = False if parser.has_option(section, 'continue'): continue_matching = parser.getboolean(section, 'continue') rule = RelayRule(condition=regex.search, destinations=destinations, continue_matching=continue_matching) rules.append(rule) continue if parser.has_option(section, 'default'): if not parser.getboolean(section, 'default'): continue # just ignore default = false if defaultRule: raise CarbonConfigException( "Only one default rule can be specified") defaultRule = RelayRule(condition=lambda metric: True, destinations=destinations) if not defaultRule: raise CarbonConfigException( "No default rule defined. You must specify exactly one " "rule with 'default = true' instead of a pattern.") rules.append(defaultRule) return rules
def __init__(self, config, cluster='main', aggregation_rules=None): relay_method = config.relay_method(cluster=cluster) if relay_method == "consistent-hashing": self.ring = ConsistentHashingRouter(config.replication_factor(cluster)) elif relay_method == "aggregated-consistent-hashing": if aggregation_rules: RuleManager.read_from(aggregation_rules) self.ring = AggregatedConsistentHashingRouter(RuleManager, config.replication_factor(cluster)) try: dest_list = config.destinations(cluster) self.destinations = util.parseDestinations(dest_list) except ValueError as e: raise SystemExit("Unable to parse destinations!" + str(e)) for d in self.destinations: self.ring.addDestination(d)
def loadRelayRules(path): rules = [] parser = OrderedConfigParser() if not parser.read(path): raise CarbonConfigException("Could not read rules file %s" % path) defaultRule = None for section in parser.sections(): if not parser.has_option(section, 'destinations'): raise CarbonConfigException("Rules file %s section %s does not define a " "'destinations' list" % (path, section)) destination_strings = parser.get(section, 'destinations').split(',') destinations = parseDestinations(destination_strings) if parser.has_option(section, 'pattern'): if parser.has_option(section, 'default'): raise CarbonConfigException("Section %s contains both 'pattern' and " "'default'. You must use one or the other." % section) pattern = parser.get(section, 'pattern') regex = re.compile(pattern, re.I) continue_matching = False if parser.has_option(section, 'continue'): continue_matching = parser.getboolean(section, 'continue') rule = RelayRule( condition=regex.search, destinations=destinations, continue_matching=continue_matching) rules.append(rule) continue if parser.has_option(section, 'default'): if not parser.getboolean(section, 'default'): continue # just ignore default = false if defaultRule: raise CarbonConfigException("Only one default rule can be specified") defaultRule = RelayRule(condition=lambda metric: True, destinations=destinations) if not defaultRule: raise CarbonConfigException("No default rule defined. You must specify exactly one " "rule with 'default = true' instead of a pattern.") rules.append(defaultRule) return rules
def test_valid_dest_unbracketed(self): # Tests valid destinations in the unbracketed form of <host>. dests = [ "127.0.0.1:1234:alpha", # Full IPv4 address "127.1:1234:beta", # 'Short' IPv4 address "localhost:987:epsilon", # Relative domain name "foo.bar.baz.uk.:890:sigma" # Absolute domain name ] expected = [("127.0.0.1", 1234, "alpha"), ("127.1", 1234, "beta"), ("localhost", 987, "epsilon"), ("foo.bar.baz.uk.", 890, "sigma")] actual = parseDestinations(dests) self.assertEquals(len(expected), len(actual)) for exp, act in zip(expected, actual): self.assertEquals(exp, act)
def test_valid_dest_without_instance(self): # Tests destinations without instance specified. dests = [ "1.2.3.4:5678", "[::1]:2", "stats.example.co.uk:8125", "[127.0.0.1]:78", # Odd use of the bracket feature, but why not? "[why.not.this.com]:89", ] expected = [("1.2.3.4", 5678, None), ("::1", 2, None), ("stats.example.co.uk", 8125, None), ("127.0.0.1", 78, None), ("why.not.this.com", 89, None)] actual = parseDestinations(dests) self.assertEquals(len(expected), len(actual)) for exp, act in zip(expected, actual): self.assertEquals(exp, act)
def setupRelayProcessor(root_service, settings): from carbon.routers import AggregatedConsistentHashingRouter, \ ConsistentHashingRouter, RelayRulesRouter from carbon.client import CarbonClientManager if settings.RELAY_METHOD == 'consistent-hashing': router = ConsistentHashingRouter(settings.REPLICATION_FACTOR) elif settings.RELAY_METHOD == 'aggregated-consistent-hashing': from carbon.aggregator.rules import RuleManager aggregation_rules_path = settings["aggregation-rules"] RuleManager.read_from(aggregation_rules_path) router = AggregatedConsistentHashingRouter(RuleManager, settings.REPLICATION_FACTOR) elif settings.RELAY_METHOD == 'rules': router = RelayRulesRouter(settings["relay-rules"]) state.client_manager = CarbonClientManager(router) state.client_manager.setServiceParent(root_service) for destination in util.parseDestinations(settings.DESTINATIONS): state.client_manager.startClient(destination)
def test_valid_dest_bracketed(self): # Tests valid destinations in the bracketed form of <host>. dests = [ "[fe80:dead:beef:cafe:0007:0007:0007:0001]:123:gamma", # Full IPv6 address "[fe80:1234::7]:456:theta", # Compact IPv6 address "[::]:1:o", # Very compact IPv6 address "[ffff::127.0.0.1]:789:omicron" # IPv6 mapped IPv4 address ] expected = [ ("fe80:dead:beef:cafe:0007:0007:0007:0001", 123, "gamma"), ("fe80:1234::7", 456, "theta"), ("::", 1, "o"), ("ffff::127.0.0.1", 789, "omicron"), ] actual = parseDestinations(dests) self.assertEquals(len(expected), len(actual)) for exp, act in zip(expected, actual): self.assertEquals(exp, act)
def test_valid_dest_unbracketed(self): # Tests valid destinations in the unbracketed form of <host>. dests = [ "127.0.0.1:1234:alpha", # Full IPv4 address "127.1:1234:beta", # 'Short' IPv4 address "localhost:987:epsilon", # Relative domain name "foo.bar.baz.uk.:890:sigma" # Absolute domain name ] expected = [ ("127.0.0.1", 1234, "alpha"), ("127.1", 1234, "beta"), ("localhost", 987, "epsilon"), ("foo.bar.baz.uk.", 890, "sigma") ] actual = parseDestinations(dests) self.assertEquals(len(expected), len(actual)) for exp, act in zip(expected, actual): self.assertEquals(exp, act)
def __init__(self, config, cluster='main'): # Support multiple versions of carbon, the API changed in 0.10. args = inspect.getargspec(ConsistentHashingRouter.__init__).args if 'replication_factor' in args: ring = ConsistentHashingRouter(config.replication_factor(cluster)) else: class Settings(object): REPLICATION_FACTOR = config.replication_factor(cluster) DIVERSE_REPLICAS = False ring = ConsistentHashingRouter(Settings()) self.ring = ring try: dest_list = config.destinations(cluster) self.destinations = util.parseDestinations(dest_list) except ValueError as e: raise SystemExit("Unable to parse destinations!" + str(e)) for d in self.destinations: self.ring.addDestination(d)
def test_valid_dest_without_instance(self): # Tests destinations without instance specified. dests = [ "1.2.3.4:5678", "[::1]:2", "stats.example.co.uk:8125", "[127.0.0.1]:78", # Odd use of the bracket feature, but why not? "[why.not.this.com]:89", ] expected = [ ("1.2.3.4", 5678, None), ("::1", 2, None), ("stats.example.co.uk", 8125, None), ("127.0.0.1", 78, None), ("why.not.this.com", 89, None) ] actual = parseDestinations(dests) self.assertEquals(len(expected), len(actual)) for exp, act in zip(expected, actual): self.assertEquals(exp, act)
def __init__(self, config, cluster='main', aggregation_rules=None): relay_method = config.relay_method(cluster=cluster) if relay_method == "consistent-hashing": # Support multiple versions of carbon, the API changed in 0.10. args = inspect.getargspec(ConsistentHashingRouter.__init__).args if 'replication_factor' in args: r = ConsistentHashingRouter(config.replication_factor(cluster)) else: class Settings(object): REPLICATION_FACTOR = config.replication_factor(cluster) DIVERSE_REPLICAS = False r = ConsistentHashingRouter(Settings()) # 'hash_type' was added only in carbon 1.0.2 or master args = inspect.getargspec(ConsistentHashRing.__init__).args if 'hash_type' in args: r.ring = ConsistentHashRing( hash_type=config.hashing_type(cluster)) elif relay_method == "aggregated-consistent-hashing": if aggregation_rules: RuleManager.read_from(aggregation_rules) r = AggregatedConsistentHashingRouter( RuleManager, config.replication_factor(cluster)) self.ring = r try: dest_list = config.destinations(cluster) self.destinations = util.parseDestinations(dest_list) except ValueError as e: raise SystemExit("Unable to parse destinations!" + str(e)) for d in self.destinations: self.ring.addDestination(d)
def parseDestination(destination): return parseDestinations([destination])[0]
metric_key = arg # Check required key if not metric_key: print('Usage: python graphite-router.py -k <metric key>') sys.exit(2) ## Settings # Absolute path to the Graphite Data Directory DATA_DIR = join(ROOT_DIR, 'storage/whisper') # Parse config settings.readFrom(join(ROOT_DIR, 'conf/carbon.conf'), 'relay') # Read in destinations from config destinations = util.parseDestinations(settings.DESTINATIONS) # Setup Router router = ConsistentHashingRouter(settings.REPLICATION_FACTOR) for destination in destinations: router.addDestination(destination); # Echo routes print('routes for ' + metric_key) routes = router.getDestinations(metric_key) for route in routes: print(route)