def test_parse(self):
conf = Configuration(
name='invalid_path',
path='/dev/null',
parse=False
)
expected = {}
actual = conf._parse_section(path='./data/empty.ini')
self.assertEqual(expected, actual)
expected = {'section': {'key': 'value'}}
actual = conf._parse_section(path='./data/single_section.ini')
self.assertEqual(expected, actual)
expected = {'section': {'key': 'value', 'key_two': 'other_value'}}
actual = conf._parse_section(path='./data/single_section.ini', defaults={'key_two': 'other_value'})
self.assertEqual(expected, actual)
expected = {'section': {'key': 'value', 'key_two': 'other_value'}}
actual = conf._parse_section(path='./data/single_section.ini', defaults={'key_two': 'other_value'}, only_section='section')
self.assertEqual(expected, actual)
expected = {}
actual = conf._parse_section(path='./data/single_section.ini', defaults={'key_two': 'other_value'}, remove_section='section')
self.assertEqual(expected, actual)
expected = {
'derp': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'sleep', 'til': 'brooklyn'},
'multiple_sections': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'one'},
'section': {'cats': '1', 'expected': 'the spanish inquisition', 'key': 'value', 'no': 'one'}
}
actual = conf._parse_section(path='./data/multiple_sections.ini', defaults={
'no': 'one',
'expected': 'the spanish inquisition',
'cats': '1',
})
self.assertEqual(expected, actual)
expected = {
'multiple_sections': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'one'},
}
actual = conf._parse_section(path='./data/multiple_sections.ini', only_section='multiple_sections', defaults={
'no': 'one',
'expected': 'the spanish inquisition',
'cats': '1',
})
self.assertEqual(expected, actual)
expected = {
'multiple_sections': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'one'},
'section': {'cats': '1', 'expected': 'the spanish inquisition', 'key': 'value', 'no': 'one'}
}
actual = conf._parse_section(path='./data/multiple_sections.ini', remove_section='derp', defaults={
'no': 'one',
'expected': 'the spanish inquisition',
'cats': '1',
})
self.assertEqual(expected, actual)
def test_has_section(self):
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
)
self.assertEqual(True, conf.has('multiple_sections'))
self.assertEqual(True, conf.has('derp'))
self.assertEqual(True, conf.has('derp', 'no'))
def test_has_section(self):
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
)
self.assertEqual(True, conf.has('multiple_sections'))
self.assertEqual(True, conf.has('derp'))
self.assertEqual(True, conf.has('derp', 'no'))
def test_custom_config_parser(self):
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
config_parser=TestConfigParser,
parse=False
)
self.assertRaises(NotImplementedError, lambda: conf._parse_section(path='./data/multiple_sections.ini'))
def test_custom_config_parser(self):
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
config_parser=TestConfigParser,
parse=False
)
self.assertRaises(NotImplementedError, lambda: conf._parse_section(path='./data/multiple_sections.ini'))
def test_invalid_path(self):
conf = Configuration(
name='invalid_path',
path='/dev/null',
parse=False
)
self.assertRaises(IOError, lambda: conf._parse_section(path=''))
self.assertRaises(IOError, lambda: conf._parse_section(path=None))
self.assertRaises(IOError, lambda: conf._parse_section(path='.'))
self.assertRaises(IOError, lambda: conf._parse_section(path='/non-existent/path'))
def test_parse(self):
conf = Configuration(
name='invalid_path',
path='/dev/null',
parse=False
)
expected = {}
actual = conf._parse_section(path='./data/empty.ini')
self.assertDictEqual(expected, actual)
expected = {'section': {'key': 'value'}}
actual = conf._parse_section(path='./data/single_section.ini')
self.assertDictEqual(expected, actual)
expected = {'section': {'key': 'value', 'key_two': 'other_value'}}
actual = conf._parse_section(path='./data/single_section.ini', defaults={'key_two': 'other_value'})
self.assertDictEqual(expected, actual)
expected = {'section': {'key': 'value', 'key_two': 'other_value'}}
actual = conf._parse_section(path='./data/single_section.ini', defaults={'key_two': 'other_value'}, only_section='section')
self.assertDictEqual(expected, actual)
expected = {}
actual = conf._parse_section(path='./data/single_section.ini', defaults={'key_two': 'other_value'}, remove_section='section')
self.assertDictEqual(expected, actual)
expected = {
'derp': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'sleep', 'til': 'brooklyn'},
'multiple_sections': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'one'},
'section': {'cats': '1', 'expected': 'the spanish inquisition', 'key': 'value', 'no': 'one'}
}
actual = conf._parse_section(path='./data/multiple_sections.ini', defaults={
'no': 'one',
'expected': 'the spanish inquisition',
'cats': '1',
})
self.assertDictEqual(expected, actual)
expected = {
'multiple_sections': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'one'},
}
actual = conf._parse_section(path='./data/multiple_sections.ini', only_section='multiple_sections', defaults={
'no': 'one',
'expected': 'the spanish inquisition',
'cats': '1',
})
self.assertDictEqual(expected, actual)
expected = {
'multiple_sections': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'one'},
'section': {'cats': '1', 'expected': 'the spanish inquisition', 'key': 'value', 'no': 'one'}
}
actual = conf._parse_section(path='./data/multiple_sections.ini', remove_section='derp', defaults={
'no': 'one',
'expected': 'the spanish inquisition',
'cats': '1',
})
self.assertDictEqual(expected, actual)
def test_raw(self):
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
main_defaults={'main_key': 'main_value'},
section_defaults={'section_key': 'section_value'}
)
self.assertEqual({'main_key': 'main_value'}, conf.raw('multiple_sections'))
self.assertEqual({'key': 'value', 'section_key': 'section_value'}, conf.raw('section'))
self.assertEqual({'no': 'sleep', 'section_key': 'section_value', 'til': 'brooklyn'}, conf.raw('derp'))
expected = {
'multiple_sections': {'main_key': 'main_value'},
'sections': {
'derp': {'no': 'sleep', 'section_key': 'section_value', 'til': 'brooklyn'},
'section': {'key': 'value', 'section_key': 'section_value'}
}
}
self.assertEqual(expected, conf.raw())
def test_raw(self):
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
main_defaults={'main_key': 'main_value'},
section_defaults={'section_key': 'section_value'}
)
self.assertEqual({'main_key': 'main_value'}, conf.raw('multiple_sections'))
self.assertEqual({'key': 'value', 'section_key': 'section_value'}, conf.raw('section'))
self.assertEqual({'no': 'sleep', 'section_key': 'section_value', 'til': 'brooklyn'}, conf.raw('derp'))
expected = {
'multiple_sections': {'main_key': 'main_value'},
'sections': {
'derp': {'no': 'sleep', 'section_key': 'section_value', 'til': 'brooklyn'},
'section': {'key': 'value', 'section_key': 'section_value'}
}
}
self.assertEqual(expected, conf.raw())
def test_get(self):
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
)
self.assertEqual({}, conf.get('multiple_sections'))
self.assertEqual({'til': 'brooklyn', 'no': 'sleep'}, conf.get('derp'))
self.assertEqual('brooklyn', conf.get('derp', 'til'))
self.assertEqual(None, conf.get('derp', 'missing_key'))
self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))
def test_invalid_path(self):
conf = Configuration(
name='invalid_path',
path='/dev/null',
parse=False
)
self.assertRaises(IOError, lambda: conf._parse_section(path=''))
self.assertRaises(IOError, lambda: conf._parse_section(path=None))
self.assertRaises(IOError, lambda: conf._parse_section(path='.'))
self.assertRaises(IOError, lambda: conf._parse_section(path='/non-existent/path'))
def test_get(self):
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
)
self.assertEqual({}, conf.get('multiple_sections'))
self.assertEqual({'til': 'brooklyn', 'no': 'sleep'}, conf.get('derp'))
self.assertEqual('brooklyn', conf.get('derp', 'til'))
self.assertEqual(None, conf.get('derp', 'missing_key'))
self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))
def test_readme(self):
def digitize(config):
for key in config:
if not config[key].isdigit():
try:
config[key] = float(config[key])
except ValueError:
pass
else:
try:
config[key] = int(config[key])
except ValueError:
pass
return config
conf = Configuration(
name='derp',
path='./data/conf',
main_defaults={
'no': 'one',
'expected': 'the spanish inquisition',
'cats': '1',
},
section_parser=digitize
)
expected = {
'sections': {'herp': {'sleep': 1, 'wait': 5.0, 'timeout': 'seventy'}},
'derp': {'expected': 'the spanish inquisition', 'til': 'brooklyn', 'cats': '1', 'no': 'sleep'}
}
self.assertEqual(expected, conf.raw())
actual = conf.get(section='derp', key='no', default="jumping")
self.assertEqual('sleep', actual)
actual = conf.get(section='derp', key='til')
self.assertEqual('brooklyn', actual)
actual = conf.get(section='derp', key='cats')
self.assertEqual('1', actual)
actual = conf.get(section='derp', key='dogs')
self.assertEqual(None, actual)
actual = conf.get(section='herp', key='sleep')
self.assertEqual(1, actual)
actual = conf.get(section='herp', key='wait')
self.assertEqual(5.0, actual)
actual = conf.get(section='herp', key='timeout')
self.assertEqual('seventy', actual)
actual = conf.has(section='derp')
self.assertEqual(True, actual)
actual = conf.has(section='derp', key='no')
self.assertEqual(True, actual)
expected = {
'sections': {
'herp': {
'sleep': 1,
'wait': 5.0,
'timeout': 'seventy'
}
},
'derp': {
'expected': 'the spanish inquisition',
'til': 'brooklyn',
'cats': '1',
'no': 'sleep'
}
}
self.assertEqual(expected, conf.raw())
def test_parser(self):
def safe_cast(val):
try:
return int(val)
except ValueError:
return val
def all_as_bool(config):
for key in config:
config[key] = bool(config[key])
return config
def all_as_int(config):
for key in config:
try:
config[key] = int(config[key])
except ValueError:
pass
return config
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
confd_path='./data/conf.d',
main_defaults={'main_key': 'main_value'},
section_defaults={'sleep': '2', 'wait': '30'},
main_parser=all_as_bool
)
self.assertEqual({'main_key': True}, conf.get('multiple_sections'))
self.assertEqual(True, conf.get('multiple_sections', 'main_key'))
self.assertEqual('1', conf.get('another/conf', 'sleep'))
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
confd_path='./data/conf.d',
main_defaults={'main_key': 'main_value'},
section_defaults={'sleep': '2', 'wait': '30'},
section_parser=all_as_bool
)
self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
self.assertEqual(1, conf.get('another/conf', 'sleep'))
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
confd_path='./data/conf.d',
main_defaults={'main_key': 'main_value'},
section_defaults={'sleep': '2', 'wait': '30'},
main_parser=all_as_bool,
section_parser=all_as_int
)
self.assertEqual({'main_key': True}, conf.get('multiple_sections'))
self.assertEqual(True, conf.get('multiple_sections', 'main_key'))
self.assertEqual(None, conf.get('derp', 'missing_key'))
self.assertEqual(1, conf.get('another/conf', 'sleep'))
def test_confd(self):
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
confd_path='./data/conf.d',
main_defaults={'main_key': 'main_value'}
)
self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
self.assertEqual({'til': 'brooklyn', 'no': 'sleep'}, conf.get('derp'))
self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
self.assertEqual('brooklyn', conf.get('derp', 'til'))
self.assertEqual(None, conf.get('derp', 'missing_key'))
self.assertEqual(None, conf.get('derp', 'main_key'))
self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))
expected = {
'multiple_sections': {'main_key': 'main_value'},
'sections': {
'another/conf': {'sleep': '1', 'wait': '15'},
'derp': {'no': 'sleep', 'til': 'brooklyn'},
'section': {'key': 'value'},
'test': {'conf': 'path/to/another/conf', 'sleep': '15'}
}
}
self.assertEqual(expected, conf.raw())
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
confd_path='./data/conf.d',
section_defaults={'sleep': '2', 'wait': '30'}
)
self.assertEqual({}, conf.get('multiple_sections'))
self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp'))
self.assertEqual(None, conf.get('multiple_sections', 'main_key'))
self.assertEqual('brooklyn', conf.get('derp', 'til'))
self.assertEqual(None, conf.get('derp', 'missing_key'))
self.assertEqual(None, conf.get('derp', 'main_key'))
self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))
expected = {
'multiple_sections': {},
'sections': {
'another/conf': {'sleep': '1', 'wait': '15'},
'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'},
'section': {'key': 'value', 'sleep': '2', 'wait': '30'},
'test': {'conf': 'path/to/another/conf', 'sleep': '15', 'wait': '30'}
}
}
self.assertEqual(expected, conf.raw())
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
confd_path='./data/conf.d',
main_defaults={'main_key': 'main_value'},
section_defaults={'sleep': '2', 'wait': '30'}
)
self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp'))
self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
self.assertEqual('brooklyn', conf.get('derp', 'til'))
self.assertEqual(None, conf.get('derp', 'missing_key'))
self.assertEqual(None, conf.get('derp', 'main_key'))
self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))
expected = {
'multiple_sections': {'main_key': 'main_value'},
'sections': {
'another/conf': {'sleep': '1', 'wait': '15'},
'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'},
'section': {'key': 'value', 'sleep': '2', 'wait': '30'},
'test': {'conf': 'path/to/another/conf', 'sleep': '15', 'wait': '30'}
}
}
self.assertEqual(expected, conf.raw())
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
confd_path='./data/conf.d',
conf_ext='conf',
main_defaults={'main_key': 'main_value'},
section_defaults={'sleep': '2', 'wait': '30'}
)
self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp'))
self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
self.assertEqual('brooklyn', conf.get('derp', 'til'))
self.assertEqual(None, conf.get('derp', 'missing_key'))
self.assertEqual(None, conf.get('derp', 'main_key'))
self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))
expected = {
'multiple_sections': {'main_key': 'main_value'},
'sections': {
'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'},
'section': {'key': 'value', 'sleep': '2', 'wait': '30'},
}
}
self.assertEqual(expected, conf.raw())
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
confd_path='./data/conf.d',
conf_ext='.conf',
main_defaults={'main_key': 'main_value'},
section_defaults={'sleep': '2', 'wait': '30'}
)
self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp'))
self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
self.assertEqual('brooklyn', conf.get('derp', 'til'))
self.assertEqual(None, conf.get('derp', 'missing_key'))
self.assertEqual(None, conf.get('derp', 'main_key'))
self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))
expected = {
'multiple_sections': {'main_key': 'main_value'},
'sections': {
'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'},
'section': {'key': 'value', 'sleep': '2', 'wait': '30'},
}
}
self.assertEqual(expected, conf.raw())
def _parse(self, args):
def _main_parser(config):
transpose = ['config', 'confd_path', 'debug', 'daemonize', 'files', 'format', 'fqdn', 'hostname', 'path', 'pid', 'transport']
namspace_dict = vars(args)
for key in transpose:
if key not in namspace_dict or namspace_dict[key] is None or namspace_dict[key] == '':
continue
config[key] = namspace_dict[key]
if args.mode:
config['zeromq_bind'] = args.mode
# HACK: Python 2.6 ConfigParser does not properly
# handle non-string values
for key in config:
if config[key] == '':
config[key] = None
require_bool = ['debug', 'daemonize', 'fqdn', 'rabbitmq_exchange_durable', 'rabbitmq_queue_durable',
'rabbitmq_ha_queue', 'rabbitmq_ssl', 'tcp_ssl_enabled', 'tcp_ssl_verify']
for key in require_bool:
config[key] = bool(int(config[key]))
require_int = [
'max_failure',
'max_queue_size',
'queue_timeout',
'rabbitmq_port',
'rabbitmq_timeout',
'rabbitmq_delivery_mode',
'respawn_delay',
'subprocess_poll_sleep',
'refresh_worker_process',
'tcp_port',
'udp_port',
'wait_timeout',
'zeromq_hwm',
'logstash_version',
'kafka_batch_n',
'kafka_batch_t',
'kafka_ack_timeout',
'number_of_consumer_processes',
'ignore_old_files'
]
for key in require_int:
if config[key] is not None:
config[key] = int(config[key])
require_float = [
'update_file_mapping_time',
'discover_interval',
]
for key in require_float:
if config[key] is not None:
config[key] = float(config[key])
if config.get('format') == 'null':
config['format'] = 'raw'
if config['files'] is not None and type(config['files']) == str:
config['files'] = config['files'].split(',')
if config['path'] is not None:
config['path'] = os.path.realpath(config['path'])
if not os.path.isdir(config['path']):
raise LookupError('{0} does not exist'.format(config['path']))
if config.get('hostname') is None:
if config.get('fqdn') is True:
config['hostname'] = socket.getfqdn()
else:
config['hostname'] = socket.gethostname()
if config.get('sincedb_path'):
config['sincedb_path'] = os.path.realpath(config.get('sincedb_path'))
if config['zeromq_address'] and type(config['zeromq_address']) == str:
config['zeromq_address'] = [x.strip() for x in config.get('zeromq_address').split(',')]
if config.get('ssh_options') is not None:
csv = config.get('ssh_options')
config['ssh_options'] = []
if type(csv) == str:
for opt in csv.split(','):
config['ssh_options'].append('-o %s' % opt.strip())
else:
config['ssh_options'] = []
config['globs'] = {}
return config
def _section_parser(config, raise_exceptions=True):
'''Parse a given INI-style config file using ConfigParser module.
Stanza's names match file names, and properties are defaulted as in
http://logstash.net/docs/1.1.1/inputs/file
Config file example:
[/var/log/syslog]
type: syslog
tags: sys,main
[/var/log/auth]
type: syslog
;tags: auth,main
'''
fields = config.get('add_field', '')
if type(fields) != dict:
try:
if type(fields) == str:
fields = filter(None, fields.split(','))
if len(fields) == 0:
config['fields'] = {}
elif (len(fields) % 2) == 1:
if raise_exceptions:
raise Exception('Wrong number of values for add_field')
else:
fieldkeys = fields[0::2]
fieldvalues = [[x] for x in fields[1::2]]
config['fields'] = dict(zip(fieldkeys, fieldvalues))
except TypeError:
config['fields'] = {}
if 'add_field' in config:
del config['add_field']
envFields = config.get('add_field_env', '')
if type(envFields) != dict:
try:
if type(envFields) == str:
envFields = envFields.replace(" ","")
envFields = filter(None, envFields.split(','))
if len(envFields) == 0:
config['envFields'] = {}
elif (len(envFields) % 2) == 1:
if raise_exceptions:
raise Exception('Wrong number of values for add_field_env')
else:
envFieldkeys = envFields[0::2]
envFieldvalues = []
for x in envFields[1::2]:
envFieldvalues.append(os.environ.get(x))
config['fields'].update(dict(zip(envFieldkeys, envFieldvalues)))
except TypeError:
config['envFields'] = {}
if 'add_field_env' in config:
del config['add_field_env']
try:
tags = config.get('tags', '')
if type(tags) == str:
tags = filter(None, tags.split(','))
if len(tags) == 0:
tags = []
config['tags'] = tags
except TypeError:
config['tags'] = []
if config.get('format') == 'null':
config['format'] = 'raw'
file_type = config.get('type', None)
if not file_type:
config['type'] = 'file'
require_bool = ['debug', 'ignore_empty', 'ignore_truncate']
for k in require_bool:
config[k] = bool(int(config[k]))
config['delimiter'] = config['delimiter'].decode('string-escape')
if config['multiline_regex_after']:
config['multiline_regex_after'] = re.compile(config['multiline_regex_after'])
if config['multiline_regex_before']:
config['multiline_regex_before'] = re.compile(config['multiline_regex_before'])
require_int = ['sincedb_write_interval', 'stat_interval', 'tail_lines']
for k in require_int:
config[k] = int(config[k])
return config
conf = Configuration(
name='beaver',
path=self._configfile,
main_defaults=self._main_defaults,
section_defaults=self._section_defaults,
main_parser=_main_parser,
section_parser=_section_parser,
path_from_main='confd_path',
config_parser=self._config_parser
)
config = conf.raw()
self._beaver_config = config['beaver']
self._file_config = config['sections']
self._main_parser = _main_parser(self._main_defaults)
self._section_defaults = _section_parser(self._section_defaults, raise_exceptions=False)
self._files = {}
for section in config['sections']:
globs = eglob(section, config['sections'][section].get('exclude', ''))
if not globs:
self._logger.debug('Skipping glob due to no files found: %s' % section)
continue
for globbed_file in globs:
self._files[os.path.realpath(globbed_file)] = config['sections'][section]
def _parse(self, args):
def _main_parser(config):
transpose = ['config', 'debug', 'daemonize', 'files', 'format', 'fqdn', 'hostname', 'path', 'pid', 'transport']
namspace_dict = vars(args)
for key in transpose:
if key not in namspace_dict or namspace_dict[key] is None or namspace_dict[key] == '':
continue
config[key] = namspace_dict[key]
if args.mode:
config['zeromq_bind'] = args.mode
# HACK: Python 2.6 ConfigParser does not properly
# handle non-string values
for key in config:
if config[key] == '':
config[key] = None
require_bool = ['debug', 'daemonize', 'fqdn', 'rabbitmq_exchange_durable', 'rabbitmq_queue_durable', 'rabbitmq_ha_queue']
for key in require_bool:
config[key] = bool(int(config[key]))
require_int = [
'max_failure',
'max_queue_size',
'queue_timeout',
'rabbitmq_port',
'respawn_delay',
'subprocess_poll_sleep',
'udp_port',
'wait_timeout',
'zeromq_hwm',
]
for key in require_int:
if config[key] is not None:
config[key] = int(config[key])
require_float = [
'update_file_mapping_time',
'discover_interval',
]
for key in require_float:
if config[key] is not None:
config[key] = float(config[key])
if config['files'] is not None and type(config['files']) == str:
config['files'] = config['files'].split(',')
config['path'] = os.path.realpath(config['path'])
if not os.path.isdir(config['path']):
raise LookupError('{0} does not exist'.format(config['path']))
if config.get('hostname') is None:
if config.get('fqdn') is True:
config['hostname'] = socket.getfqdn()
else:
config['hostname'] = socket.gethostname()
if config.get('sincedb_path'):
config['sincedb_path'] = os.path.realpath(config.get('sincedb_path'))
config['globs'] = {}
return config
def _section_parser(config, raise_exceptions=True):
'''Parse a given INI-style config file using ConfigParser module.
Stanza's names match file names, and properties are defaulted as in
http://logstash.net/docs/1.1.1/inputs/file
Config file example:
[/var/log/syslog]
type: syslog
tags: sys,main
[/var/log/auth]
type: syslog
;tags: auth,main
'''
fields = config.get('add_field', '')
if type(fields) != dict:
try:
if type(fields) == str:
fields = filter(None, fields.split(','))
if len(fields) == 0:
config['fields'] = {}
elif (len(fields) % 2) == 1:
if raise_exceptions:
raise Exception('Wrong number of values for add_field')
else:
fieldkeys = fields[0::2]
fieldvalues = [[x] for x in fields[1::2]]
config['fields'] = dict(zip(fieldkeys, fieldvalues))
except TypeError:
config['fields'] = {}
if 'add_field' in config:
del config['add_field']
try:
tags = config.get('tags', '')
if type(tags) == str:
tags = filter(None, tags.split(','))
if len(tags) == 0:
tags = []
config['tags'] = tags
except TypeError:
config['tags'] = []
try:
file_type = config.get('type', 'file')
if not file_type:
file_type = 'file'
config['type'] = file_type
except:
config['type'] = 'file'
if config['type']:
if raise_exceptions:
raise Exception('Missing mandatory config "type"')
require_bool = ['debug', 'ignore_empty', 'ignore_truncate']
for k in require_bool:
config[k] = bool(int(config[k]))
require_int = ['sincedb_write_interval', 'stat_interval', 'tail_lines']
for k in require_int:
config[k] = int(config[k])
return config
conf = Configuration(
name='beaver',
path=self._configfile,
main_defaults=self._main_defaults,
section_defaults=self._section_defaults,
main_parser=_main_parser,
section_parser=_section_parser,
)
config = conf.raw()
self._beaver_config = config['beaver']
self._file_config = config['sections']
self._main_parser = _main_parser(self._main_defaults)
self._section_defaults = _section_parser(self._section_defaults, raise_exceptions=False)
self._files = {}
for section in config['sections']:
globs = eglob(section, config['sections'][section].get('exclude', ''))
if not globs:
self._logger.debug('Skipping glob due to no files found: %s' % section)
continue
for globbed_file in globs:
self._files[os.path.realpath(globbed_file)] = config['sections'][section]
def test_confd(self):
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
confd_path='./data/conf.d',
main_defaults={'main_key': 'main_value'}
)
self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
self.assertEqual({'til': 'brooklyn', 'no': 'sleep'}, conf.get('derp'))
self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
self.assertEqual('brooklyn', conf.get('derp', 'til'))
self.assertEqual(None, conf.get('derp', 'missing_key'))
self.assertEqual(None, conf.get('derp', 'main_key'))
self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))
expected = {
'multiple_sections': {'main_key': 'main_value'},
'sections': {
'another/conf': {'sleep': '1', 'wait': '15'},
'derp': {'no': 'sleep', 'til': 'brooklyn'},
'section': {'key': 'value'},
'test': {'conf': 'path/to/another/conf', 'sleep': '15'}
}
}
self.assertEqual(expected, conf.raw())
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
confd_path='./data/conf.d',
section_defaults={'sleep': '2', 'wait': '30'}
)
self.assertEqual({}, conf.get('multiple_sections'))
self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp'))
self.assertEqual(None, conf.get('multiple_sections', 'main_key'))
self.assertEqual('brooklyn', conf.get('derp', 'til'))
self.assertEqual(None, conf.get('derp', 'missing_key'))
self.assertEqual(None, conf.get('derp', 'main_key'))
self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))
expected = {
'multiple_sections': {},
'sections': {
'another/conf': {'sleep': '1', 'wait': '15'},
'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'},
'section': {'key': 'value', 'sleep': '2', 'wait': '30'},
'test': {'conf': 'path/to/another/conf', 'sleep': '15', 'wait': '30'}
}
}
self.assertEqual(expected, conf.raw())
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
confd_path='./data/conf.d',
main_defaults={'main_key': 'main_value'},
section_defaults={'sleep': '2', 'wait': '30'}
)
self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp'))
self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
self.assertEqual('brooklyn', conf.get('derp', 'til'))
self.assertEqual(None, conf.get('derp', 'missing_key'))
self.assertEqual(None, conf.get('derp', 'main_key'))
self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))
expected = {
'multiple_sections': {'main_key': 'main_value'},
'sections': {
'another/conf': {'sleep': '1', 'wait': '15'},
'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'},
'section': {'key': 'value', 'sleep': '2', 'wait': '30'},
'test': {'conf': 'path/to/another/conf', 'sleep': '15', 'wait': '30'}
}
}
self.assertEqual(expected, conf.raw())
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
confd_path='./data/conf.d',
conf_ext='conf',
main_defaults={'main_key': 'main_value'},
section_defaults={'sleep': '2', 'wait': '30'}
)
self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp'))
self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
self.assertEqual('brooklyn', conf.get('derp', 'til'))
self.assertEqual(None, conf.get('derp', 'missing_key'))
self.assertEqual(None, conf.get('derp', 'main_key'))
self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))
expected = {
'multiple_sections': {'main_key': 'main_value'},
'sections': {
'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'},
'section': {'key': 'value', 'sleep': '2', 'wait': '30'},
}
}
self.assertEqual(expected, conf.raw())
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
confd_path='./data/conf.d',
conf_ext='.conf',
main_defaults={'main_key': 'main_value'},
section_defaults={'sleep': '2', 'wait': '30'}
)
self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp'))
self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
self.assertEqual('brooklyn', conf.get('derp', 'til'))
self.assertEqual(None, conf.get('derp', 'missing_key'))
self.assertEqual(None, conf.get('derp', 'main_key'))
self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))
expected = {
'multiple_sections': {'main_key': 'main_value'},
'sections': {
'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'},
'section': {'key': 'value', 'sleep': '2', 'wait': '30'},
}
}
self.assertEqual(expected, conf.raw())
def test_readme(self):
def digitize(config):
for key in config:
if not config[key].isdigit():
try:
config[key] = float(config[key])
except ValueError:
pass
else:
try:
config[key] = int(config[key])
except ValueError:
pass
return config
conf = Configuration(
name='derp',
path='./data/conf',
main_defaults={
'no': 'one',
'expected': 'the spanish inquisition',
'cats': '1',
},
section_parser=digitize
)
expected = {
'sections': {'herp': {'sleep': 1, 'wait': 5.0, 'timeout': 'seventy'}},
'derp': {'expected': 'the spanish inquisition', 'til': 'brooklyn', 'cats': '1', 'no': 'sleep'}
}
self.assertEqual(expected, conf.raw())
actual = conf.get(section='derp', key='no', default="jumping")
self.assertEqual('sleep', actual)
actual = conf.get(section='derp', key='til')
self.assertEqual('brooklyn', actual)
actual = conf.get(section='derp', key='cats')
self.assertEqual('1', actual)
actual = conf.get(section='derp', key='dogs')
self.assertEqual(None, actual)
actual = conf.get(section='herp', key='sleep')
self.assertEqual(1, actual)
actual = conf.get(section='herp', key='wait')
self.assertEqual(5.0, actual)
actual = conf.get(section='herp', key='timeout')
self.assertEqual('seventy', actual)
actual = conf.has(section='derp')
self.assertEqual(True, actual)
actual = conf.has(section='derp', key='no')
self.assertEqual(True, actual)
expected = {
'sections': {
'herp': {
'sleep': 1,
'wait': 5.0,
'timeout': 'seventy'
}
},
'derp': {
'expected': 'the spanish inquisition',
'til': 'brooklyn',
'cats': '1',
'no': 'sleep'
}
}
self.assertEqual(expected, conf.raw())
def test_parser(self):
def safe_cast(val):
try:
return int(val)
except ValueError:
return val
def all_as_bool(config):
for key in config:
config[key] = bool(config[key])
return config
def all_as_int(config):
for key in config:
try:
config[key] = int(config[key])
except ValueError:
pass
return config
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
confd_path='./data/conf.d',
main_defaults={'main_key': 'main_value'},
section_defaults={'sleep': '2', 'wait': '30'},
main_parser=all_as_bool
)
self.assertEqual({'main_key': True}, conf.get('multiple_sections'))
self.assertEqual(True, conf.get('multiple_sections', 'main_key'))
self.assertEqual('1', conf.get('another/conf', 'sleep'))
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
confd_path='./data/conf.d',
main_defaults={'main_key': 'main_value'},
section_defaults={'sleep': '2', 'wait': '30'},
section_parser=all_as_bool
)
self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
self.assertEqual(1, conf.get('another/conf', 'sleep'))
conf = Configuration(
name='multiple_sections',
path='./data/multiple_sections.ini',
confd_path='./data/conf.d',
main_defaults={'main_key': 'main_value'},
section_defaults={'sleep': '2', 'wait': '30'},
main_parser=all_as_bool,
section_parser=all_as_int
)
self.assertEqual({'main_key': True}, conf.get('multiple_sections'))
self.assertEqual(True, conf.get('multiple_sections', 'main_key'))
self.assertEqual(None, conf.get('derp', 'missing_key'))
self.assertEqual(1, conf.get('another/conf', 'sleep'))
def _parse(self, args):
def _main_parser(config):
transpose = [
'config', 'confd_path', 'debug', 'daemonize', 'files',
'format', 'fqdn', 'hostname', 'path', 'pid', 'transport'
]
namspace_dict = vars(args)
for key in transpose:
if key not in namspace_dict or namspace_dict[
key] is None or namspace_dict[key] == '':
continue
config[key] = namspace_dict[key]
if args.mode:
config['zeromq_bind'] = args.mode
# HACK: Python 2.6 ConfigParser does not properly
# handle non-string values
for key in config:
if config[key] == '':
config[key] = None
require_bool = [
'debug', 'daemonize', 'fqdn', 'rabbitmq_exchange_durable',
'rabbitmq_queue_durable', 'rabbitmq_ha_queue'
]
for key in require_bool:
config[key] = bool(int(config[key]))
require_int = [
'max_failure',
'max_queue_size',
'queue_timeout',
'rabbitmq_port',
'respawn_delay',
'subprocess_poll_sleep',
'tcp_port',
'udp_port',
'wait_timeout',
'zeromq_hwm',
]
for key in require_int:
if config[key] is not None:
config[key] = int(config[key])
require_float = [
'update_file_mapping_time',
'discover_interval',
]
for key in require_float:
if config[key] is not None:
config[key] = float(config[key])
if config.get('format') == 'null':
config['format'] = 'raw'
if config['files'] is not None and type(config['files']) == str:
config['files'] = config['files'].split(',')
if config['path'] is not None:
config['path'] = os.path.realpath(config['path'])
if not os.path.isdir(config['path']):
raise LookupError('{0} does not exist'.format(
config['path']))
if config.get('hostname') is None:
if config.get('fqdn') is True:
config['hostname'] = socket.getfqdn()
else:
config['hostname'] = socket.gethostname()
if config.get('sincedb_path'):
config['sincedb_path'] = os.path.realpath(
config.get('sincedb_path'))
config['globs'] = {}
return config
def _section_parser(config, raise_exceptions=True):
'''Parse a given INI-style config file using ConfigParser module.
Stanza's names match file names, and properties are defaulted as in
http://logstash.net/docs/1.1.1/inputs/file
Config file example:
[/var/log/syslog]
type: syslog
tags: sys,main
[/var/log/auth]
type: syslog
;tags: auth,main
'''
fields = config.get('add_field', '')
if type(fields) != dict:
try:
if type(fields) == str:
fields = filter(None, fields.split(','))
if len(fields) == 0:
config['fields'] = {}
elif (len(fields) % 2) == 1:
if raise_exceptions:
raise Exception(
'Wrong number of values for add_field')
else:
fieldkeys = fields[0::2]
fieldvalues = [[x] for x in fields[1::2]]
config['fields'] = dict(zip(fieldkeys, fieldvalues))
except TypeError:
config['fields'] = {}
if 'add_field' in config:
del config['add_field']
try:
tags = config.get('tags', '')
if type(tags) == str:
tags = filter(None, tags.split(','))
if len(tags) == 0:
tags = []
config['tags'] = tags
except TypeError:
config['tags'] = []
if config.get('format') == 'null':
config['format'] = 'raw'
file_type = config.get('type', None)
if not file_type:
config['type'] = 'file'
require_bool = ['debug', 'ignore_empty', 'ignore_truncate']
for k in require_bool:
config[k] = bool(int(config[k]))
require_int = [
'sincedb_write_interval', 'stat_interval', 'tail_lines'
]
for k in require_int:
config[k] = int(config[k])
return config
conf = Configuration(name='beaver',
path=self._configfile,
main_defaults=self._main_defaults,
section_defaults=self._section_defaults,
main_parser=_main_parser,
section_parser=_section_parser,
path_from_main='confd_path')
config = conf.raw()
self._beaver_config = config['beaver']
self._file_config = config['sections']
self._main_parser = _main_parser(self._main_defaults)
self._section_defaults = _section_parser(self._section_defaults,
raise_exceptions=False)
self._files = {}
for section in config['sections']:
globs = eglob(section,
config['sections'][section].get('exclude', ''))
if not globs:
self._logger.debug('Skipping glob due to no files found: %s' %
section)
continue
for globbed_file in globs:
self._files[os.path.realpath(
globbed_file)] = config['sections'][section]
def _parse(self, args):
def _main_parser(config):
transpose = [
"config",
"confd_path",
"debug",
"daemonize",
"files",
"format",
"fqdn",
"hostname",
"path",
"pid",
"transport",
]
namspace_dict = vars(args)
for key in transpose:
if key not in namspace_dict or namspace_dict[key] is None or namspace_dict[key] == "":
continue
config[key] = namspace_dict[key]
if args.mode:
config["zeromq_bind"] = args.mode
# HACK: Python 2.6 ConfigParser does not properly
# handle non-string values
for key in config:
if config[key] == "":
config[key] = None
require_bool = [
"debug",
"daemonize",
"fqdn",
"rabbitmq_exchange_durable",
"rabbitmq_queue_durable",
"rabbitmq_ha_queue",
"rabbitmq_ssl",
"tcp_ssl_enabled",
"tcp_ssl_verify",
]
for key in require_bool:
config[key] = bool(int(config[key]))
require_int = [
"max_failure",
"max_queue_size",
"queue_timeout",
"rabbitmq_port",
"rabbitmq_timeout",
"rabbitmq_delivery_mode",
"respawn_delay",
"subprocess_poll_sleep",
"refresh_worker_process",
"tcp_port",
"udp_port",
"wait_timeout",
"zeromq_hwm",
"logstash_version",
"kafka_batch_n",
"kafka_batch_t",
"kafka_ack_timeout",
"number_of_consumer_processes",
]
for key in require_int:
if config[key] is not None:
config[key] = int(config[key])
require_float = ["update_file_mapping_time", "discover_interval"]
for key in require_float:
if config[key] is not None:
config[key] = float(config[key])
if config.get("format") == "null":
config["format"] = "raw"
if config["files"] is not None and type(config["files"]) == str:
config["files"] = config["files"].split(",")
if config["path"] is not None:
config["path"] = os.path.realpath(config["path"])
if not os.path.isdir(config["path"]):
raise LookupError("{0} does not exist".format(config["path"]))
if config.get("hostname") is None:
if config.get("fqdn") is True:
config["hostname"] = socket.getfqdn()
else:
config["hostname"] = socket.gethostname()
if config.get("sincedb_path"):
config["sincedb_path"] = os.path.realpath(config.get("sincedb_path"))
if config["zeromq_address"] and type(config["zeromq_address"]) == str:
config["zeromq_address"] = [x.strip() for x in config.get("zeromq_address").split(",")]
if config.get("ssh_options") is not None:
csv = config.get("ssh_options")
config["ssh_options"] = []
if type(csv) == str:
for opt in csv.split(","):
config["ssh_options"].append("-o %s" % opt.strip())
else:
config["ssh_options"] = []
config["globs"] = {}
return config
def _section_parser(config, raise_exceptions=True):
"""Parse a given INI-style config file using ConfigParser module.
Stanza's names match file names, and properties are defaulted as in
http://logstash.net/docs/1.1.1/inputs/file
Config file example:
[/var/log/syslog]
type: syslog
tags: sys,main
[/var/log/auth]
type: syslog
;tags: auth,main
"""
fields = config.get("add_field", "")
if type(fields) != dict:
try:
if type(fields) == str:
fields = filter(None, fields.split(","))
if len(fields) == 0:
config["fields"] = {}
elif (len(fields) % 2) == 1:
if raise_exceptions:
raise Exception("Wrong number of values for add_field")
else:
fieldkeys = fields[0::2]
fieldvalues = [[x] for x in fields[1::2]]
config["fields"] = dict(zip(fieldkeys, fieldvalues))
except TypeError:
config["fields"] = {}
if "add_field" in config:
del config["add_field"]
envFields = config.get("add_field_env", "")
if type(envFields) != dict:
try:
if type(envFields) == str:
envFields = envFields.replace(" ", "")
envFields = filter(None, envFields.split(","))
if len(envFields) == 0:
config["envFields"] = {}
elif (len(envFields) % 2) == 1:
if raise_exceptions:
raise Exception("Wrong number of values for add_field_env")
else:
envFieldkeys = envFields[0::2]
envFieldvalues = []
for x in envFields[1::2]:
envFieldvalues.append(os.environ.get(x))
config["fields"].update(dict(zip(envFieldkeys, envFieldvalues)))
except TypeError:
config["envFields"] = {}
if "add_field_env" in config:
del config["add_field_env"]
try:
tags = config.get("tags", "")
if type(tags) == str:
tags = filter(None, tags.split(","))
if len(tags) == 0:
tags = []
config["tags"] = tags
except TypeError:
config["tags"] = []
if config.get("format") == "null":
config["format"] = "raw"
file_type = config.get("type", None)
if not file_type:
config["type"] = "file"
require_bool = ["debug", "ignore_empty", "ignore_truncate"]
for k in require_bool:
config[k] = bool(int(config[k]))
config["delimiter"] = config["delimiter"].decode("string-escape")
if config["multiline_regex_after"]:
config["multiline_regex_after"] = re.compile(config["multiline_regex_after"])
if config["multiline_regex_before"]:
config["multiline_regex_before"] = re.compile(config["multiline_regex_before"])
require_int = ["sincedb_write_interval", "stat_interval", "tail_lines"]
for k in require_int:
config[k] = int(config[k])
return config
conf = Configuration(
name="beaver",
path=self._configfile,
main_defaults=self._main_defaults,
section_defaults=self._section_defaults,
main_parser=_main_parser,
section_parser=_section_parser,
path_from_main="confd_path",
config_parser=self._config_parser,
)
config = conf.raw()
self._beaver_config = config["beaver"]
self._file_config = config["sections"]
self._main_parser = _main_parser(self._main_defaults)
self._section_defaults = _section_parser(self._section_defaults, raise_exceptions=False)
self._files = {}
for section in config["sections"]:
globs = eglob(section, config["sections"][section].get("exclude", ""))
if not globs:
self._logger.debug("Skipping glob due to no files found: %s" % section)
continue
for globbed_file in globs:
self._files[os.path.realpath(globbed_file)] = config["sections"][section]