def loadStorageSchemas():
schemaList = []
config = OrderedConfigParser()
config.read(STORAGE_SCHEMAS_CONFIG)
for section in config.sections():
options = dict(config.items(section))
matchAll = options.get('match-all')
pattern = options.get('pattern')
listName = options.get('list')
retentions = options['retentions'].split(',')
archives = [Archive.fromString(s) for s in retentions]
if matchAll:
mySchema = DefaultSchema(section, archives)
elif pattern:
mySchema = PatternSchema(section, pattern, archives)
elif listName:
mySchema = ListSchema(section, listName, archives)
archiveList = [a.getTuple() for a in archives]
try:
whisper.validateArchiveList(archiveList)
schemaList.append(mySchema)
except whisper.InvalidConfiguration, e:
log.msg("Invalid schemas found in %s: %s" % (section, e))
def loadStorageSchemas():
schemaList = []
config = OrderedConfigParser()
config.read(STORAGE_SCHEMAS_CONFIG)
for section in config.sections():
options = dict( config.items(section) )
pattern = options.get('pattern')
retentions = options['retentions'].split(',')
archives = [ Archive.fromString(s) for s in retentions ]
if pattern:
mySchema = PatternSchema(section, pattern, archives)
else:
log.err("Section missing 'pattern': %s" % section)
continue
archiveList = [a.getTuple() for a in archives]
try:
whisper.validateArchiveList(archiveList)
schemaList.append(mySchema)
except whisper.InvalidConfiguration, e:
log.msg("Invalid schemas found in %s: %s" % (section, e) )
def loadStorageSchemas():
schemaList = []
config = OrderedConfigParser()
config.read(STORAGE_SCHEMAS_CONFIG)
for section in config.sections():
options = dict(config.items(section))
pattern = options.get('pattern')
try:
retentions = options['retentions'].split(',')
archives = [Archive.fromString(s) for s in retentions]
except KeyError:
log.err("Schema %s missing 'retentions', skipping" % section)
continue
if pattern:
mySchema = PatternSchema(section, pattern, archives)
else:
log.err("Schema %s missing 'pattern', skipping" % section)
continue
archiveList = [a.getTuple() for a in archives]
try:
if state.database is not None:
state.database.validateArchiveList(archiveList)
schemaList.append(mySchema)
except ValueError as e:
log.msg("Invalid schemas found in %s: %s" % (section, e))
schemaList.append(defaultSchema)
return schemaList
def loadRules(path):
global defaultRule
assert not rules, "rules already loaded"
parser = OrderedConfigParser()
if not parser.read(path):
raise ValueError("Could not read rules file %s" % path)
for section in parser.sections():
if not parser.has_option(section, 'servers'):
raise ValueError("Rules file %s section %s does not define a 'servers' list" % (path, section))
servers = [s.strip() for s in parser.get(section, 'servers').split(',')]
if parser.has_option(section, 'pattern'):
assert not parser.has_option(section, 'default'), "Section %s contains both 'pattern' and 'default'. You must use one or the other." % section
pattern = parser.get(section, 'pattern')
regex = re.compile(pattern, re.I)
rules.append( Rule(condition=regex.search, destinations=servers) )
continue
if parser.has_option(section, 'default'):
if not parser.getboolean(section, 'default'): continue # just ignore default = false
assert not defaultRule, "Two default rules? Seriously?"
defaultRule = Rule(condition=lambda metric: True, destinations=servers)
assert defaultRule, "No default rule defined. You must specify exactly one rule with 'default = true' instead of a pattern."
def loadStorageSchemas():
schemaList = []
config = OrderedConfigParser()
config.read(STORAGE_SCHEMAS_CONFIG)
for section in config.sections():
options = dict(config.items(section))
matchAll = options.get("match-all")
pattern = options.get("pattern")
listName = options.get("list")
retentions = options["retentions"].split(",")
archives = [Archive.fromString(s) for s in retentions]
if matchAll:
mySchema = DefaultSchema(section, archives)
elif pattern:
mySchema = PatternSchema(section, pattern, archives)
elif listName:
mySchema = ListSchema(section, listName, archives)
else:
raise ValueError('schema "%s" has no pattern or list parameter configured' % section)
schemaList.append(mySchema)
schemaList.append(defaultSchema)
return schemaList
def loadAggregationSchemas():
# NOTE: This abuses the Schema classes above, and should probably be refactored.
schemaList = []
config = OrderedConfigParser()
try:
config.read(STORAGE_AGGREGATION_CONFIG)
except (IOError, CarbonConfigException):
log.msg("%s not found, ignoring." % STORAGE_AGGREGATION_CONFIG)
for section in config.sections():
options = dict(config.items(section))
matchAll = options.get('match-all')
pattern = options.get('pattern')
listName = options.get('list')
xFilesFactor = options.get('xfilesfactor')
aggregationMethod = options.get('aggregationmethod')
try:
if xFilesFactor is not None:
xFilesFactor = float(xFilesFactor)
assert 0 <= xFilesFactor <= 1
if aggregationMethod is not None:
assert aggregationMethod in whisper.aggregationMethods
except:
log.msg("Invalid schemas found in %s." % section)
continue
archives = (xFilesFactor, aggregationMethod)
if matchAll:
mySchema = DefaultSchema(section, archives)
elif pattern:
mySchema = PatternSchema(section, pattern, archives)
elif listName:
mySchema = ListSchema(section, listName, archives)
schemaList.append(mySchema)
schemaList.append(defaultAggregation)
return schemaList
def loadAggregationSchemas():
# NOTE: This abuses the Schema classes above, and should probably be refactored.
schemaList = []
config = OrderedConfigParser()
try:
config.read(STORAGE_AGGREGATION_CONFIG)
except (IOError, CarbonConfigException):
log.msg("%s not found or wrong perms, ignoring." % STORAGE_AGGREGATION_CONFIG)
for section in config.sections():
options = dict(config.items(section))
pattern = options.get('pattern')
xFilesFactor = options.get('xfilesfactor')
aggregationMethod = options.get('aggregationmethod')
try:
if xFilesFactor is not None:
xFilesFactor = float(xFilesFactor)
assert 0 <= xFilesFactor <= 1
if aggregationMethod is not None:
if state.database is not None:
assert aggregationMethod in state.database.aggregationMethods
except ValueError:
log.msg("Invalid schemas found in %s." % section)
continue
archives = (xFilesFactor, aggregationMethod)
if pattern:
mySchema = PatternSchema(section, pattern, archives)
else:
log.err("Section missing 'pattern': %s" % section)
continue
schemaList.append(mySchema)
schemaList.append(defaultAggregation)
return schemaList
def loadStorageSchemas():
schemaList = []
config = OrderedConfigParser()
config.read(STORAGE_SCHEMAS_CONFIG)
for section in config.sections():
options = dict( config.items(section) )
matchAll = options.get('match-all')
pattern = options.get('pattern')
listName = options.get('list')
retentions = options['retentions'].split(',')
archives = [ Archive.fromString(s) for s in retentions ]
if matchAll:
mySchema = DefaultSchema(section, archives)
elif pattern:
mySchema = PatternSchema(section, pattern, archives)
elif listName:
mySchema = ListSchema(section, listName, archives)
else:
raise ValueError('schema "%s" has no pattern or list parameter configured' % section)
schemaList.append( mySchema )
schemaList.append( defaultSchema )
return schemaList
def loadStorageSchemas():
schemaList = []
config = OrderedConfigParser()
config.read(STORAGE_SCHEMAS_CONFIG)
for section in config.sections():
options = dict(config.items(section))
matchAll = options.get('match-all')
pattern = options.get('pattern')
listName = options.get('list')
retentions = options['retentions'].split(',')
archives = [Archive.fromString(s) for s in retentions]
if matchAll:
mySchema = DefaultSchema(section, archives)
elif pattern:
mySchema = PatternSchema(section, pattern, archives)
elif listName:
mySchema = ListSchema(section, listName, archives)
archiveList = [a.getTuple() for a in archives]
try:
whisper.validateArchiveList(archiveList)
schemaList.append(mySchema)
except whisper.InvalidConfiguration, e:
log.msg("Invalid schemas found in %s: %s" % (section, e))
def loadRelayRules(path):
rules = []
parser = OrderedConfigParser()
if not parser.read(path):
raise Exception("Could not read rules file %s" % path)
defaultRule = None
for section in parser.sections():
if not parser.has_option(section, 'destinations'):
raise ValueError("Rules file %s section %s does not define a "
"'destinations' list" % (path, section))
destination_strings = parser.get(section, 'destinations').split(',')
destinations = parseDestinations(destination_strings)
if parser.has_option(section, 'pattern'):
if parser.has_option(section, 'default'):
raise Exception("Section %s contains both 'pattern' and "
"'default'. You must use one or the other." % section)
pattern = parser.get(section, 'pattern')
regex = re.compile(pattern, re.I)
rule = RelayRule(condition=regex.search, destinations=destinations)
rules.append(rule)
continue
if parser.has_option(section, 'default'):
if not parser.getboolean(section, 'default'):
continue # just ignore default = false
if defaultRule:
raise Exception("Two default rules? Seriously?")
defaultRule = RelayRule(condition=lambda metric: True,
destinations=destinations)
if not defaultRule:
raise Exception("No default rule defined. You must specify exactly one "
"rule with 'default = true' instead of a pattern.")
rules.append(defaultRule)
return rules
def loadStorageSchemas():
schemaList = []
config = OrderedConfigParser()
config.read(STORAGE_SCHEMAS_CONFIG)
for section in config.sections():
options = dict(config.items(section))
pattern = options.get('pattern')
retentions = options['retentions'].split(',')
archives = [Archive.fromString(s) for s in retentions]
if pattern:
mySchema = PatternSchema(section, pattern, archives)
else:
log.err("Section missing 'pattern': %s" % section)
continue
archiveList = [a.getTuple() for a in archives]
try:
whisper.validateArchiveList(archiveList)
schemaList.append(mySchema)
except whisper.InvalidConfiguration, e:
log.msg("Invalid schemas found in %s: %s" % (section, e))
def loadStorageSchemas():
schemaList = []
config = OrderedConfigParser()
config.read(STORAGE_SCHEMAS_CONFIG)
for section in config.sections():
options = dict(config.items(section))
pattern = options.get('pattern')
try:
retentions = options.get('retentions').split(',')
archives = [Archive.fromString(s) for s in retentions]
except KeyError:
log.err("Schema %s missing 'retentions', skipping" % section)
continue
if pattern:
mySchema = PatternSchema(section, pattern, archives)
else:
log.err("Schema %s missing 'pattern', skipping" % section)
continue
archiveList = [a.getTuple() for a in archives]
try:
if state.database is not None:
state.database.validateArchiveList(archiveList)
schemaList.append(mySchema)
except ValueError as e:
log.msg("Invalid schemas found in %s: %s" % (section, e))
schemaList.append(defaultSchema)
return schemaList
def loadAggregationSchemas():
# NOTE: This abuses the Schema classes above, and should probably be refactored.
schemaList = []
config = OrderedConfigParser()
try:
config.read(STORAGE_AGGREGATION_CONFIG)
except (IOError, CarbonConfigException):
log.msg("%s not found or wrong perms, ignoring." %
STORAGE_AGGREGATION_CONFIG)
for section in config.sections():
options = dict(config.items(section))
matchAll = options.get('match-all')
pattern = options.get('pattern')
listName = options.get('list')
xFilesFactor = options.get('xfilesfactor')
aggregationMethod = options.get('aggregationmethod')
try:
if xFilesFactor is not None:
xFilesFactor = float(xFilesFactor)
assert 0 <= xFilesFactor <= 1
if aggregationMethod is not None:
assert aggregationMethod in whisper.aggregationMethods
except:
log.msg("Invalid schemas found in %s." % section)
continue
archives = (xFilesFactor, aggregationMethod)
if matchAll:
mySchema = DefaultSchema(section, archives)
elif pattern:
mySchema = PatternSchema(section, pattern, archives)
elif listName:
mySchema = ListSchema(section, listName, archives)
schemaList.append(mySchema)
schemaList.append(defaultAggregation)
return schemaList
def loadAggregationSchemas():
# NOTE: This abuses the Schema classes above, and should probably be refactored.
schemaList = []
config = OrderedConfigParser()
try:
config.read(STORAGE_AGGREGATION_CONFIG)
except (IOError, CarbonConfigException):
log.msg("%s not found or wrong perms, ignoring." %
STORAGE_AGGREGATION_CONFIG)
for section in config.sections():
options = dict(config.items(section))
pattern = options.get('pattern')
xFilesFactor = options.get('xfilesfactor')
aggregationMethod = options.get('aggregationmethod')
try:
if xFilesFactor is not None:
xFilesFactor = float(xFilesFactor)
assert 0 <= xFilesFactor <= 1
if aggregationMethod is not None:
if state.database is not None:
assert aggregationMethod in state.database.aggregationMethods
except ValueError:
log.msg("Invalid schemas found in %s." % section)
continue
archives = (xFilesFactor, aggregationMethod)
if pattern:
mySchema = PatternSchema(section, pattern, archives)
else:
log.err("Section missing 'pattern': %s" % section)
continue
schemaList.append(mySchema)
schemaList.append(defaultAggregation)
return schemaList
def loadRelayRules(path):
rules = []
parser = OrderedConfigParser()
if not parser.read(path):
raise CarbonConfigException("Could not read rules file %s" % path)
defaultRule = None
for section in parser.sections():
if not parser.has_option(section, 'destinations'):
raise CarbonConfigException("Rules file %s section %s does not define a "
"'destinations' list" % (path, section))
destination_strings = parser.get(section, 'destinations').split(',')
destinations = parseDestinations(destination_strings)
if parser.has_option(section, 'pattern'):
if parser.has_option(section, 'default'):
raise CarbonConfigException("Section %s contains both 'pattern' and "
"'default'. You must use one or the other." % section)
pattern = parser.get(section, 'pattern')
regex = re.compile(pattern, re.I)
continue_matching = False
if parser.has_option(section, 'continue'):
continue_matching = parser.getboolean(section, 'continue')
rule = RelayRule(condition=regex.search, destinations=destinations, continue_matching=continue_matching)
rules.append(rule)
continue
if parser.has_option(section, 'default'):
if not parser.getboolean(section, 'default'):
continue # just ignore default = false
if defaultRule:
raise CarbonConfigException("Only one default rule can be specified")
defaultRule = RelayRule(condition=lambda metric: True,
destinations=destinations)
if not defaultRule:
raise CarbonConfigException("No default rule defined. You must specify exactly one "
"rule with 'default = true' instead of a pattern.")
rules.append(defaultRule)
return rules
def loadRules(path):
global defaultRule
assert not rules, "rules already loaded"
parser = OrderedConfigParser()
if not parser.read(path):
raise ValueError("Could not read rules file %s" % path)
for section in parser.sections():
if not parser.has_option(section, 'servers'):
raise ValueError(
"Rules file %s section %s does not define a 'servers' list" %
(path, section))
hosts = parseHostList(parser.get(section, 'servers').split(','))
if parser.has_option(section, 'pattern'):
assert not parser.has_option(
section, 'default'
), "Section %s contains both 'pattern' and 'default'. You must use one or the other." % section
pattern = parser.get(section, 'pattern')
regex = re.compile(pattern, re.I)
rules.append(Rule(condition=regex.search, destinations=hosts))
continue
if parser.has_option(section, 'default'):
if not parser.getboolean(section, 'default'):
continue # just ignore default = false
assert not defaultRule, "Two default rules? Seriously?"
defaultRule = Rule(condition=lambda metric: True,
destinations=hosts)
assert defaultRule, "No default rule defined. You must specify exactly one rule with 'default = true' instead of a pattern."
rules.append(defaultRule)
