def loadRelayRules(path):
rules = []
parser = OrderedConfigParser()
if not parser.read(path):
raise CarbonConfigException("Could not read rules file %s" % path)
defaultRule = None
for section in parser.sections():
if not parser.has_option(section, 'destinations'):
raise CarbonConfigException(
"Rules file %s section %s does not define a "
"'destinations' list" % (path, section))
destination_strings = parser.get(section, 'destinations').split(',')
destinations = parseDestinations(destination_strings)
if parser.has_option(section, 'pattern'):
if parser.has_option(section, 'default'):
raise CarbonConfigException(
"Section %s contains both 'pattern' and "
"'default'. You must use one or the other." % section)
pattern = parser.get(section, 'pattern')
regex = re.compile(pattern, re.I)
continue_matching = False
if parser.has_option(section, 'continue'):
continue_matching = parser.getboolean(section, 'continue')
rule = RelayRule(condition=regex.search,
destinations=destinations,
continue_matching=continue_matching)
rules.append(rule)
continue
if parser.has_option(section, 'default'):
if not parser.getboolean(section, 'default'):
continue # just ignore default = false
if defaultRule:
raise CarbonConfigException(
"Only one default rule can be specified")
defaultRule = RelayRule(condition=lambda metric: True,
destinations=destinations)
if not defaultRule:
raise CarbonConfigException(
"No default rule defined. You must specify exactly one "
"rule with 'default = true' instead of a pattern.")
rules.append(defaultRule)
return rules
def readFrom(self, path, section):
parser = ConfigParser()
if not parser.read(path):
raise CarbonConfigException("Failed to read config file %s" % path)
if not parser.has_section(section):
return
for key, value in parser.items(section):
key = key.upper()
# Detect type from defaults dict
if key in defaults:
valueType = type(defaults[key])
else:
valueType = str
if valueType is list:
value = [v.strip() for v in value.split(',')]
elif valueType is bool:
value = parser.getboolean(section, key)
else:
# Attempt to figure out numeric types automatically
try:
value = int(value)
except ValueError:
try:
value = float(value)
except ValueError:
pass
self[key] = value
def createRelayService(config):
from carbon.routers import RelayRulesRouter, ConsistentHashingRouter, AggregatedConsistentHashingRouter
from carbon.client import CarbonClientManager
from carbon.conf import settings
from carbon import events
root_service = createBaseService(config)
# Configure application components
if settings.RELAY_METHOD == 'rules':
router = RelayRulesRouter(settings["relay-rules"])
elif settings.RELAY_METHOD == 'consistent-hashing':
router = ConsistentHashingRouter(
settings.REPLICATION_FACTOR,
diverse_replicas=settings.DIVERSE_REPLICAS)
elif settings.RELAY_METHOD == 'aggregated-consistent-hashing':
from carbon.aggregator.rules import RuleManager
RuleManager.read_from(settings["aggregation-rules"])
router = AggregatedConsistentHashingRouter(RuleManager,
settings.REPLICATION_FACTOR)
client_manager = CarbonClientManager(router)
client_manager.setServiceParent(root_service)
events.metricReceived.addHandler(client_manager.sendDatapoint)
events.metricGenerated.addHandler(client_manager.sendDatapoint)
if not settings.DESTINATIONS:
raise CarbonConfigException(
"Required setting DESTINATIONS is missing from carbon.conf")
for destination in util.parseDestinations(settings.DESTINATIONS):
client_manager.startClient(destination)
return root_service
def createAggregatorService(config):
from carbon.aggregator import receiver
from carbon.aggregator.rules import RuleManager
from carbon.routers import ConsistentHashingRouter
from carbon.client import CarbonClientManager
from carbon.rewrite import RewriteRuleManager
from carbon.conf import settings
from carbon import events
root_service = createBaseService(config)
# Configure application components
router = ConsistentHashingRouter(
settings.REPLICATION_FACTOR,
diverse_replicas=settings.DIVERSE_REPLICAS)
client_manager = CarbonClientManager(router)
client_manager.setServiceParent(root_service)
events.metricReceived.addHandler(receiver.process)
events.metricGenerated.addHandler(client_manager.sendDatapoint)
RuleManager.read_from(settings["aggregation-rules"])
if exists(settings["rewrite-rules"]):
RewriteRuleManager.read_from(settings["rewrite-rules"])
if not settings.DESTINATIONS:
raise CarbonConfigException(
"Required setting DESTINATIONS is missing from carbon.conf")
for destination in util.parseDestinations(settings.DESTINATIONS):
client_manager.startClient(destination)
return root_service
def createManholeListener():
sshRealm = TerminalRealm()
sshRealm.chainedProtocolFactory.protocolFactory = lambda _: Manhole(
namespace)
if settings.MANHOLE_PUBLIC_KEY == 'None':
credChecker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
credChecker.addUser(settings.MANHOLE_USER.encode('utf-8'),
''.encode('utf-8'))
else:
userKeys = {
settings.MANHOLE_USER.encode('utf-8'):
settings.MANHOLE_PUBLIC_KEY.encode('utf-8'),
}
credChecker = PublicKeyChecker(userKeys)
sshPortal = portal.Portal(sshRealm)
sshPortal.registerChecker(credChecker)
sessionFactory = ConchFactory(sshPortal)
# set ssh host keys
if settings.MANHOLE_HOST_KEY_DIR == "":
raise CarbonConfigException("MANHOLE_HOST_KEY_DIR not defined")
openSSHFactory = OpenSSHFactory()
openSSHFactory.dataRoot = settings.MANHOLE_HOST_KEY_DIR
sessionFactory.publicKeys = openSSHFactory.getPublicKeys()
sessionFactory.privateKeys = openSSHFactory.getPrivateKeys()
return sessionFactory
def setupAggregatorProcessor(root_service, settings):
from carbon.aggregator.processor import AggregationProcessor # Register the plugin class
from carbon.aggregator.rules import RuleManager
aggregation_rules_path = settings["aggregation-rules"]
if not exists(aggregation_rules_path):
raise CarbonConfigException("aggregation processor: file does not exist {0}".format(aggregation_rules_path))
RuleManager.read_from(aggregation_rules_path)
def read(self, path):
# Verifies a file exists *and* is readable
if not os.access(path, os.R_OK):
raise CarbonConfigException("Error: Missing config file or wrong perms on %s" % path)
result = ConfigParser.read(self, path)
sections = []
for line in open(path):
line = line.strip()
if line.startswith('[') and line.endswith(']'):
sections.append(line[1:-1])
self._ordered_sections = sections
return result
def read_config(program, options, **kwargs):
"""
Read settings for 'program' from configuration file specified by
'options["config"]', with missing values provided by 'defaults'.
"""
settings = Settings()
settings.update(defaults)
# Initialize default values if not set yet.
for name, value in kwargs.items():
settings.setdefault(name, value)
graphite_root = kwargs.get("ROOT_DIR")
if graphite_root is None:
graphite_root = os.environ.get('GRAPHITE_ROOT')
if graphite_root is None:
raise CarbonConfigException("Either ROOT_DIR or GRAPHITE_ROOT "
"needs to be provided.")
# Default config directory to root-relative, unless overridden by the
# 'GRAPHITE_CONF_DIR' environment variable.
settings.setdefault(
"CONF_DIR",
os.environ.get("GRAPHITE_CONF_DIR", join(graphite_root, "conf")))
if options["config"] is None:
options["config"] = join(settings["CONF_DIR"], "carbon.conf")
else:
# Set 'CONF_DIR' to the parent directory of the 'carbon.conf' config
# file.
settings["CONF_DIR"] = dirname(normpath(options["config"]))
# Storage directory can be overridden by the 'GRAPHITE_STORAGE_DIR'
# environment variable. It defaults to a path relative to GRAPHITE_ROOT
# for backwards compatibility though.
settings.setdefault(
"STORAGE_DIR",
os.environ.get("GRAPHITE_STORAGE_DIR", join(graphite_root, "storage")))
def update_STORAGE_DIR_deps():
# By default, everything is written to subdirectories of the storage dir.
settings.setdefault("PID_DIR", settings["STORAGE_DIR"])
settings.setdefault("LOG_DIR",
join(settings["STORAGE_DIR"], "log", program))
settings.setdefault("LOCAL_DATA_DIR",
join(settings["STORAGE_DIR"], "whisper"))
settings.setdefault("WHITELISTS_DIR",
join(settings["STORAGE_DIR"], "lists"))
# Read configuration options from program-specific section.
section = program[len("carbon-"):]
config = options["config"]
if not exists(config):
raise CarbonConfigException("Error: missing required config %r" %
config)
settings.readFrom(config, section)
settings.setdefault("instance", options["instance"])
update_STORAGE_DIR_deps()
# If a specific instance of the program is specified, augment the settings
# with the instance-specific settings and provide sane defaults for
# optional settings.
if options["instance"]:
settings.readFrom(config, "%s:%s" % (section, options["instance"]))
settings["pidfile"] = (options["pidfile"] or join(
settings["PID_DIR"], "%s-%s.pid" % (program, options["instance"])))
settings["LOG_DIR"] = (options["logdir"] or join(
settings["LOG_DIR"], "%s-%s" % (program, options["instance"])))
else:
settings["pidfile"] = (options["pidfile"] or join(
settings["PID_DIR"], '%s.pid' % program))
settings["LOG_DIR"] = (options["logdir"] or settings["LOG_DIR"])
update_STORAGE_DIR_deps()
return settings