def test_wrong_dest(self):
# Some cases of invalid input, e.g. invalid/missing port.
dests = [
"1.2.3.4", # No port
"1.2.3.4:huh", # Invalid port (must be int)
"[fe80::3285:a9ff:fe91:e287]", # No port
"[ffff::1.2.3.4]:notaport" # Invalid port
]
for dest in dests:
try:
parseDestinations([dest])
except ValueError:
continue
raise AssertionError("Invalid input was accepted.")
def createAggregatorService(config):
from carbon.aggregator import receiver
from carbon.aggregator.rules import RuleManager
from carbon.routers import ConsistentHashingRouter
from carbon.client import CarbonClientManager
from carbon.rewrite import RewriteRuleManager
from carbon.conf import settings
from carbon import events
root_service = createBaseService(config)
# Configure application components
router = ConsistentHashingRouter()
client_manager = CarbonClientManager(router)
client_manager.setServiceParent(root_service)
events.metricReceived.addHandler(receiver.process)
events.metricGenerated.addHandler(client_manager.sendDatapoint)
RuleManager.read_from(settings["aggregation-rules"])
if exists(settings["rewrite-rules"]):
RewriteRuleManager.read_from(settings["rewrite-rules"])
if not settings.DESTINATIONS:
raise CarbonConfigException("Required setting DESTINATIONS is missing from carbon.conf")
for destination in util.parseDestinations(settings.DESTINATIONS):
client_manager.startClient(destination)
return root_service
def createRelayService(config):
from carbon.routers import RelayRulesRouter, ConsistentHashingRouter, AggregatedConsistentHashingRouter
from carbon.client import CarbonClientManager
from carbon.conf import settings
from carbon import events
root_service = createBaseService(config)
# Configure application components
if settings.RELAY_METHOD == 'rules':
router = RelayRulesRouter(settings["relay-rules"])
elif settings.RELAY_METHOD == 'consistent-hashing':
router = ConsistentHashingRouter(settings.REPLICATION_FACTOR)
elif settings.RELAY_METHOD == 'aggregated-consistent-hashing':
from carbon.aggregator.rules import RuleManager
RuleManager.read_from(settings["aggregation-rules"])
router = AggregatedConsistentHashingRouter(RuleManager, settings.REPLICATION_FACTOR)
client_manager = CarbonClientManager(router)
client_manager.setServiceParent(root_service)
events.metricReceived.addHandler(client_manager.sendDatapoint)
events.metricGenerated.addHandler(client_manager.sendDatapoint)
events.specialMetricReceived.addHandler(client_manager.sendHighPriorityDatapoint)
events.specialMetricGenerated.addHandler(client_manager.sendHighPriorityDatapoint)
if not settings.DESTINATIONS:
raise CarbonConfigException("Required setting DESTINATIONS is missing from carbon.conf")
for destination in util.parseDestinations(settings.DESTINATIONS):
client_manager.startClient(destination)
return root_service
def createAggregatorService(config):
from carbon.aggregator import receiver
from carbon.aggregator.rules import RuleManager
from carbon.routers import ConsistentHashingRouter
from carbon.client import CarbonClientManager
from carbon.rewrite import RewriteRuleManager
from carbon.conf import settings
from carbon import events
root_service = createBaseService(config)
# Configure application components
router = ConsistentHashingRouter(
settings.REPLICATION_FACTOR,
diverse_replicas=settings.DIVERSE_REPLICAS)
client_manager = CarbonClientManager(router)
client_manager.setServiceParent(root_service)
events.metricReceived.addHandler(receiver.process)
events.metricGenerated.addHandler(client_manager.sendDatapoint)
RuleManager.read_from(settings["aggregation-rules"])
if exists(settings["rewrite-rules"]):
RewriteRuleManager.read_from(settings["rewrite-rules"])
if not settings.DESTINATIONS:
raise CarbonConfigException(
"Required setting DESTINATIONS is missing from carbon.conf")
for destination in util.parseDestinations(settings.DESTINATIONS):
client_manager.startClient(destination)
return root_service
def createRelayService(config):
from carbon.routers import RelayRulesRouter, ConsistentHashingRouter, AggregatedConsistentHashingRouter
from carbon.client import CarbonClientManager
from carbon.conf import settings
from carbon import events
root_service = createBaseService(config)
# Configure application components
if settings.RELAY_METHOD == 'rules':
router = RelayRulesRouter(settings["relay-rules"])
elif settings.RELAY_METHOD == 'consistent-hashing':
router = ConsistentHashingRouter(
settings.REPLICATION_FACTOR,
diverse_replicas=settings.DIVERSE_REPLICAS)
elif settings.RELAY_METHOD == 'aggregated-consistent-hashing':
from carbon.aggregator.rules import RuleManager
RuleManager.read_from(settings["aggregation-rules"])
router = AggregatedConsistentHashingRouter(RuleManager,
settings.REPLICATION_FACTOR)
client_manager = CarbonClientManager(router)
client_manager.setServiceParent(root_service)
events.metricReceived.addHandler(client_manager.sendDatapoint)
events.metricGenerated.addHandler(client_manager.sendDatapoint)
if not settings.DESTINATIONS:
raise CarbonConfigException(
"Required setting DESTINATIONS is missing from carbon.conf")
for destination in util.parseDestinations(settings.DESTINATIONS):
client_manager.startClient(destination)
return root_service
def setupRelayProcessor(root_service, settings):
from carbon.routers import AggregatedConsistentHashingRouter, \
ConsistentHashingRouter, RelayRulesRouter
from carbon.client import CarbonClientManager
if settings.RELAY_METHOD == 'consistent-hashing':
router = ConsistentHashingRouter(
settings.REPLICATION_FACTOR,
diverse_replicas=settings.DIVERSE_REPLICAS)
elif settings.RELAY_METHOD == 'aggregated-consistent-hashing':
from carbon.aggregator.rules import RuleManager
aggregation_rules_path = settings["aggregation-rules"]
RuleManager.read_from(aggregation_rules_path)
router = AggregatedConsistentHashingRouter(
RuleManager,
settings.REPLICATION_FACTOR,
diverse_replicas=settings.DIVERSE_REPLICAS)
elif settings.RELAY_METHOD == 'rules':
router = RelayRulesRouter(settings["relay-rules"])
state.client_manager = CarbonClientManager(router)
state.client_manager.setServiceParent(root_service)
for destination in util.parseDestinations(settings.DESTINATIONS):
state.client_manager.startClient(destination)
def __init__(self, config, cluster='main'):
# Support multiple versions of carbon, the API changed in 0.10.
args = inspect.getargspec(ConsistentHashingRouter.__init__).args
if 'replication_factor' in args:
r = ConsistentHashingRouter(config.replication_factor(cluster))
else:
class Settings(object):
REPLICATION_FACTOR = config.replication_factor(cluster)
DIVERSE_REPLICAS = False
ROUTER_HASH_TYPE = None
r = ConsistentHashingRouter(Settings())
# 'hash_type' was added only in carbon 1.0.2 or master
args = inspect.getargspec(ConsistentHashRing.__init__).args
if 'hash_type' in args:
r.ring = ConsistentHashRing(nodes=[],
hash_type=config.hashing_type(cluster))
self.ring = r
try:
dest_list = config.destinations(cluster)
self.destinations = util.parseDestinations(dest_list)
except ValueError as e:
raise SystemExit("Unable to parse destinations!" + str(e))
for d in self.destinations:
self.ring.addDestination(d)
def _get_nodes(config, cluster_name, node=None):
"""Get the list of nodes in another cluster, excludes 'node'."""
res = set()
for entry in util.parseDestinations(config.destinations(cluster_name)):
res.add(entry[0]) # Only keep the host.
if node in res:
res.remove(node)
return res
def __init__(self, config, cluster='main'):
self.ring = ConsistentHashingRouter(config.replication_factor(cluster))
try:
dest_list = config.destinations(cluster)
self.destinations = util.parseDestinations(dest_list)
except ValueError as e:
raise SystemExit("Unable to parse destinations!" + str(e))
for d in self.destinations:
self.ring.addDestination(d)
def __init__(self, config, cluster='main'):
self.ring = ConsistentHashingRouter(config.replication_factor(cluster))
try:
dest_list = config.destinations(cluster)
self.destinations = util.parseDestinations(dest_list)
except ValueError as e:
raise SystemExit("Unable to parse destinations!" + str(e))
for d in self.destinations:
self.ring.addDestination(d)
def setupRelayProcessor(root_service, settings):
from carbon.routers import DatapointRouter
from carbon.client import CarbonClientManager
router_class = DatapointRouter.plugins[settings.RELAY_METHOD]
router = router_class(settings)
state.client_manager = CarbonClientManager(router)
state.client_manager.setServiceParent(root_service)
for destination in util.parseDestinations(settings.DESTINATIONS):
state.client_manager.startClient(destination)
def setupRelayProcessor(root_service, settings):
from carbon.routers import DatapointRouter
from carbon.client import CarbonClientManager
router_class = DatapointRouter.plugins[settings.RELAY_METHOD]
router = router_class(settings)
state.client_manager = CarbonClientManager(router)
state.client_manager.setServiceParent(root_service)
for destination in util.parseDestinations(settings.DESTINATIONS):
state.client_manager.startClient(destination)
def loadRelayRules(path):
rules = []
parser = OrderedConfigParser()
if not parser.read(path):
raise CarbonConfigException("Could not read rules file %s" % path)
defaultRule = None
for section in parser.sections():
if not parser.has_option(section, 'destinations'):
raise CarbonConfigException(
"Rules file %s section %s does not define a "
"'destinations' list" % (path, section))
destination_strings = parser.get(section, 'destinations').split(',')
destinations = parseDestinations(destination_strings)
if parser.has_option(section, 'pattern'):
if parser.has_option(section, 'default'):
raise CarbonConfigException(
"Section %s contains both 'pattern' and "
"'default'. You must use one or the other." % section)
pattern = parser.get(section, 'pattern')
regex = re.compile(pattern, re.I)
continue_matching = False
if parser.has_option(section, 'continue'):
continue_matching = parser.getboolean(section, 'continue')
rule = RelayRule(condition=regex.search,
destinations=destinations,
continue_matching=continue_matching)
rules.append(rule)
continue
if parser.has_option(section, 'default'):
if not parser.getboolean(section, 'default'):
continue # just ignore default = false
if defaultRule:
raise CarbonConfigException(
"Only one default rule can be specified")
defaultRule = RelayRule(condition=lambda metric: True,
destinations=destinations)
if not defaultRule:
raise CarbonConfigException(
"No default rule defined. You must specify exactly one "
"rule with 'default = true' instead of a pattern.")
rules.append(defaultRule)
return rules
def __init__(self, config, cluster='main', aggregation_rules=None):
relay_method = config.relay_method(cluster=cluster)
if relay_method == "consistent-hashing":
self.ring = ConsistentHashingRouter(config.replication_factor(cluster))
elif relay_method == "aggregated-consistent-hashing":
if aggregation_rules:
RuleManager.read_from(aggregation_rules)
self.ring = AggregatedConsistentHashingRouter(RuleManager, config.replication_factor(cluster))
try:
dest_list = config.destinations(cluster)
self.destinations = util.parseDestinations(dest_list)
except ValueError as e:
raise SystemExit("Unable to parse destinations!" + str(e))
for d in self.destinations:
self.ring.addDestination(d)
def loadRelayRules(path):
rules = []
parser = OrderedConfigParser()
if not parser.read(path):
raise CarbonConfigException("Could not read rules file %s" % path)
defaultRule = None
for section in parser.sections():
if not parser.has_option(section, 'destinations'):
raise CarbonConfigException("Rules file %s section %s does not define a "
"'destinations' list" % (path, section))
destination_strings = parser.get(section, 'destinations').split(',')
destinations = parseDestinations(destination_strings)
if parser.has_option(section, 'pattern'):
if parser.has_option(section, 'default'):
raise CarbonConfigException("Section %s contains both 'pattern' and "
"'default'. You must use one or the other." % section)
pattern = parser.get(section, 'pattern')
regex = re.compile(pattern, re.I)
continue_matching = False
if parser.has_option(section, 'continue'):
continue_matching = parser.getboolean(section, 'continue')
rule = RelayRule(
condition=regex.search, destinations=destinations, continue_matching=continue_matching)
rules.append(rule)
continue
if parser.has_option(section, 'default'):
if not parser.getboolean(section, 'default'):
continue # just ignore default = false
if defaultRule:
raise CarbonConfigException("Only one default rule can be specified")
defaultRule = RelayRule(condition=lambda metric: True,
destinations=destinations)
if not defaultRule:
raise CarbonConfigException("No default rule defined. You must specify exactly one "
"rule with 'default = true' instead of a pattern.")
rules.append(defaultRule)
return rules
def test_valid_dest_unbracketed(self):
# Tests valid destinations in the unbracketed form of <host>.
dests = [
"127.0.0.1:1234:alpha", # Full IPv4 address
"127.1:1234:beta", # 'Short' IPv4 address
"localhost:987:epsilon", # Relative domain name
"foo.bar.baz.uk.:890:sigma" # Absolute domain name
]
expected = [("127.0.0.1", 1234, "alpha"), ("127.1", 1234, "beta"),
("localhost", 987, "epsilon"),
("foo.bar.baz.uk.", 890, "sigma")]
actual = parseDestinations(dests)
self.assertEquals(len(expected), len(actual))
for exp, act in zip(expected, actual):
self.assertEquals(exp, act)
def test_valid_dest_without_instance(self):
# Tests destinations without instance specified.
dests = [
"1.2.3.4:5678",
"[::1]:2",
"stats.example.co.uk:8125",
"[127.0.0.1]:78", # Odd use of the bracket feature, but why not?
"[why.not.this.com]:89",
]
expected = [("1.2.3.4", 5678, None), ("::1", 2, None),
("stats.example.co.uk", 8125, None),
("127.0.0.1", 78, None), ("why.not.this.com", 89, None)]
actual = parseDestinations(dests)
self.assertEquals(len(expected), len(actual))
for exp, act in zip(expected, actual):
self.assertEquals(exp, act)
def setupRelayProcessor(root_service, settings):
from carbon.routers import AggregatedConsistentHashingRouter, \
ConsistentHashingRouter, RelayRulesRouter
from carbon.client import CarbonClientManager
if settings.RELAY_METHOD == 'consistent-hashing':
router = ConsistentHashingRouter(settings.REPLICATION_FACTOR)
elif settings.RELAY_METHOD == 'aggregated-consistent-hashing':
from carbon.aggregator.rules import RuleManager
aggregation_rules_path = settings["aggregation-rules"]
RuleManager.read_from(aggregation_rules_path)
router = AggregatedConsistentHashingRouter(RuleManager, settings.REPLICATION_FACTOR)
elif settings.RELAY_METHOD == 'rules':
router = RelayRulesRouter(settings["relay-rules"])
state.client_manager = CarbonClientManager(router)
state.client_manager.setServiceParent(root_service)
for destination in util.parseDestinations(settings.DESTINATIONS):
state.client_manager.startClient(destination)
def test_valid_dest_bracketed(self):
# Tests valid destinations in the bracketed form of <host>.
dests = [
"[fe80:dead:beef:cafe:0007:0007:0007:0001]:123:gamma", # Full IPv6 address
"[fe80:1234::7]:456:theta", # Compact IPv6 address
"[::]:1:o", # Very compact IPv6 address
"[ffff::127.0.0.1]:789:omicron" # IPv6 mapped IPv4 address
]
expected = [
("fe80:dead:beef:cafe:0007:0007:0007:0001", 123, "gamma"),
("fe80:1234::7", 456, "theta"),
("::", 1, "o"),
("ffff::127.0.0.1", 789, "omicron"),
]
actual = parseDestinations(dests)
self.assertEquals(len(expected), len(actual))
for exp, act in zip(expected, actual):
self.assertEquals(exp, act)
def test_valid_dest_unbracketed(self):
# Tests valid destinations in the unbracketed form of <host>.
dests = [
"127.0.0.1:1234:alpha", # Full IPv4 address
"127.1:1234:beta", # 'Short' IPv4 address
"localhost:987:epsilon", # Relative domain name
"foo.bar.baz.uk.:890:sigma" # Absolute domain name
]
expected = [
("127.0.0.1", 1234, "alpha"),
("127.1", 1234, "beta"),
("localhost", 987, "epsilon"),
("foo.bar.baz.uk.", 890, "sigma")
]
actual = parseDestinations(dests)
self.assertEquals(len(expected), len(actual))
for exp, act in zip(expected, actual):
self.assertEquals(exp, act)
def __init__(self, config, cluster='main'):
# Support multiple versions of carbon, the API changed in 0.10.
args = inspect.getargspec(ConsistentHashingRouter.__init__).args
if 'replication_factor' in args:
ring = ConsistentHashingRouter(config.replication_factor(cluster))
else:
class Settings(object):
REPLICATION_FACTOR = config.replication_factor(cluster)
DIVERSE_REPLICAS = False
ring = ConsistentHashingRouter(Settings())
self.ring = ring
try:
dest_list = config.destinations(cluster)
self.destinations = util.parseDestinations(dest_list)
except ValueError as e:
raise SystemExit("Unable to parse destinations!" + str(e))
for d in self.destinations:
self.ring.addDestination(d)
def test_valid_dest_without_instance(self):
# Tests destinations without instance specified.
dests = [
"1.2.3.4:5678",
"[::1]:2",
"stats.example.co.uk:8125",
"[127.0.0.1]:78", # Odd use of the bracket feature, but why not?
"[why.not.this.com]:89",
]
expected = [
("1.2.3.4", 5678, None),
("::1", 2, None),
("stats.example.co.uk", 8125, None),
("127.0.0.1", 78, None),
("why.not.this.com", 89, None)
]
actual = parseDestinations(dests)
self.assertEquals(len(expected), len(actual))
for exp, act in zip(expected, actual):
self.assertEquals(exp, act)
def __init__(self, config, cluster='main', aggregation_rules=None):
relay_method = config.relay_method(cluster=cluster)
if relay_method == "consistent-hashing":
# Support multiple versions of carbon, the API changed in 0.10.
args = inspect.getargspec(ConsistentHashingRouter.__init__).args
if 'replication_factor' in args:
r = ConsistentHashingRouter(config.replication_factor(cluster))
else:
class Settings(object):
REPLICATION_FACTOR = config.replication_factor(cluster)
DIVERSE_REPLICAS = False
r = ConsistentHashingRouter(Settings())
# 'hash_type' was added only in carbon 1.0.2 or master
args = inspect.getargspec(ConsistentHashRing.__init__).args
if 'hash_type' in args:
r.ring = ConsistentHashRing(
hash_type=config.hashing_type(cluster))
elif relay_method == "aggregated-consistent-hashing":
if aggregation_rules:
RuleManager.read_from(aggregation_rules)
r = AggregatedConsistentHashingRouter(
RuleManager, config.replication_factor(cluster))
self.ring = r
try:
dest_list = config.destinations(cluster)
self.destinations = util.parseDestinations(dest_list)
except ValueError as e:
raise SystemExit("Unable to parse destinations!" + str(e))
for d in self.destinations:
self.ring.addDestination(d)
def parseDestination(destination):
return parseDestinations([destination])[0]
metric_key = arg
# Check required key
if not metric_key:
print('Usage: python graphite-router.py -k <metric key>')
sys.exit(2)
## Settings
# Absolute path to the Graphite Data Directory
DATA_DIR = join(ROOT_DIR, 'storage/whisper')
# Parse config
settings.readFrom(join(ROOT_DIR, 'conf/carbon.conf'), 'relay')
# Read in destinations from config
destinations = util.parseDestinations(settings.DESTINATIONS)
# Setup Router
router = ConsistentHashingRouter(settings.REPLICATION_FACTOR)
for destination in destinations:
router.addDestination(destination);
# Echo routes
print('routes for ' + metric_key)
routes = router.getDestinations(metric_key)
for route in routes:
print(route)
def parseDestination(destination):
return parseDestinations([destination])[0]
