Exemplo n.º 1
0
    def test_parse(self):
        conf = Configuration(
            name='invalid_path',
            path='/dev/null',
            parse=False
        )

        expected = {}
        actual = conf._parse_section(path='./data/empty.ini')
        self.assertEqual(expected, actual)

        expected = {'section': {'key': 'value'}}
        actual = conf._parse_section(path='./data/single_section.ini')
        self.assertEqual(expected, actual)

        expected = {'section': {'key': 'value', 'key_two': 'other_value'}}
        actual = conf._parse_section(path='./data/single_section.ini', defaults={'key_two': 'other_value'})
        self.assertEqual(expected, actual)

        expected = {'section': {'key': 'value', 'key_two': 'other_value'}}
        actual = conf._parse_section(path='./data/single_section.ini', defaults={'key_two': 'other_value'}, only_section='section')
        self.assertEqual(expected, actual)

        expected = {}
        actual = conf._parse_section(path='./data/single_section.ini', defaults={'key_two': 'other_value'}, remove_section='section')
        self.assertEqual(expected, actual)

        expected = {
            'derp': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'sleep', 'til': 'brooklyn'},
            'multiple_sections': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'one'},
            'section': {'cats': '1', 'expected': 'the spanish inquisition', 'key': 'value', 'no': 'one'}
        }
        actual = conf._parse_section(path='./data/multiple_sections.ini', defaults={
            'no': 'one',
            'expected': 'the spanish inquisition',
            'cats': '1',
        })
        self.assertEqual(expected, actual)

        expected = {
            'multiple_sections': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'one'},
        }
        actual = conf._parse_section(path='./data/multiple_sections.ini', only_section='multiple_sections', defaults={
            'no': 'one',
            'expected': 'the spanish inquisition',
            'cats': '1',
        })
        self.assertEqual(expected, actual)

        expected = {
            'multiple_sections': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'one'},
            'section': {'cats': '1', 'expected': 'the spanish inquisition', 'key': 'value', 'no': 'one'}
        }
        actual = conf._parse_section(path='./data/multiple_sections.ini', remove_section='derp', defaults={
            'no': 'one',
            'expected': 'the spanish inquisition',
            'cats': '1',
        })
        self.assertEqual(expected, actual)
Exemplo n.º 2
0
 def test_has_section(self):
     conf = Configuration(
         name='multiple_sections',
         path='./data/multiple_sections.ini',
     )
     self.assertEqual(True, conf.has('multiple_sections'))
     self.assertEqual(True, conf.has('derp'))
     self.assertEqual(True, conf.has('derp', 'no'))
Exemplo n.º 3
0
 def test_has_section(self):
     conf = Configuration(
         name='multiple_sections',
         path='./data/multiple_sections.ini',
     )
     self.assertEqual(True, conf.has('multiple_sections'))
     self.assertEqual(True, conf.has('derp'))
     self.assertEqual(True, conf.has('derp', 'no'))
    def test_custom_config_parser(self):
        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            config_parser=TestConfigParser,
            parse=False
        )

        self.assertRaises(NotImplementedError, lambda: conf._parse_section(path='./data/multiple_sections.ini'))
Exemplo n.º 5
0
    def test_custom_config_parser(self):
        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            config_parser=TestConfigParser,
            parse=False
        )

        self.assertRaises(NotImplementedError, lambda: conf._parse_section(path='./data/multiple_sections.ini'))
Exemplo n.º 6
0
 def test_invalid_path(self):
     conf = Configuration(
         name='invalid_path',
         path='/dev/null',
         parse=False
     )
     self.assertRaises(IOError, lambda: conf._parse_section(path=''))
     self.assertRaises(IOError, lambda: conf._parse_section(path=None))
     self.assertRaises(IOError, lambda: conf._parse_section(path='.'))
     self.assertRaises(IOError, lambda: conf._parse_section(path='/non-existent/path'))
Exemplo n.º 7
0
    def test_parse(self):
        conf = Configuration(
            name='invalid_path',
            path='/dev/null',
            parse=False
        )

        expected = {}
        actual = conf._parse_section(path='./data/empty.ini')
        self.assertDictEqual(expected, actual)

        expected = {'section': {'key': 'value'}}
        actual = conf._parse_section(path='./data/single_section.ini')
        self.assertDictEqual(expected, actual)

        expected = {'section': {'key': 'value', 'key_two': 'other_value'}}
        actual = conf._parse_section(path='./data/single_section.ini', defaults={'key_two': 'other_value'})
        self.assertDictEqual(expected, actual)

        expected = {'section': {'key': 'value', 'key_two': 'other_value'}}
        actual = conf._parse_section(path='./data/single_section.ini', defaults={'key_two': 'other_value'}, only_section='section')
        self.assertDictEqual(expected, actual)

        expected = {}
        actual = conf._parse_section(path='./data/single_section.ini', defaults={'key_two': 'other_value'}, remove_section='section')
        self.assertDictEqual(expected, actual)

        expected = {
            'derp': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'sleep', 'til': 'brooklyn'},
            'multiple_sections': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'one'},
            'section': {'cats': '1', 'expected': 'the spanish inquisition', 'key': 'value', 'no': 'one'}
        }
        actual = conf._parse_section(path='./data/multiple_sections.ini', defaults={
            'no': 'one',
            'expected': 'the spanish inquisition',
            'cats': '1',
        })
        self.assertDictEqual(expected, actual)

        expected = {
            'multiple_sections': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'one'},
        }
        actual = conf._parse_section(path='./data/multiple_sections.ini', only_section='multiple_sections', defaults={
            'no': 'one',
            'expected': 'the spanish inquisition',
            'cats': '1',
        })
        self.assertDictEqual(expected, actual)

        expected = {
            'multiple_sections': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'one'},
            'section': {'cats': '1', 'expected': 'the spanish inquisition', 'key': 'value', 'no': 'one'}
        }
        actual = conf._parse_section(path='./data/multiple_sections.ini', remove_section='derp', defaults={
            'no': 'one',
            'expected': 'the spanish inquisition',
            'cats': '1',
        })
        self.assertDictEqual(expected, actual)
Exemplo n.º 8
0
    def test_raw(self):
        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            main_defaults={'main_key': 'main_value'},
            section_defaults={'section_key': 'section_value'}
        )
        self.assertEqual({'main_key': 'main_value'}, conf.raw('multiple_sections'))
        self.assertEqual({'key': 'value', 'section_key': 'section_value'}, conf.raw('section'))
        self.assertEqual({'no': 'sleep', 'section_key': 'section_value', 'til': 'brooklyn'}, conf.raw('derp'))

        expected = {
            'multiple_sections': {'main_key': 'main_value'},
            'sections': {
                'derp': {'no': 'sleep', 'section_key': 'section_value', 'til': 'brooklyn'},
                'section': {'key': 'value', 'section_key': 'section_value'}
            }
        }
        self.assertEqual(expected, conf.raw())
Exemplo n.º 9
0
    def test_raw(self):
        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            main_defaults={'main_key': 'main_value'},
            section_defaults={'section_key': 'section_value'}
        )
        self.assertEqual({'main_key': 'main_value'}, conf.raw('multiple_sections'))
        self.assertEqual({'key': 'value', 'section_key': 'section_value'}, conf.raw('section'))
        self.assertEqual({'no': 'sleep', 'section_key': 'section_value', 'til': 'brooklyn'}, conf.raw('derp'))

        expected = {
            'multiple_sections': {'main_key': 'main_value'},
            'sections': {
                'derp': {'no': 'sleep', 'section_key': 'section_value', 'til': 'brooklyn'},
                'section': {'key': 'value', 'section_key': 'section_value'}
            }
        }
        self.assertEqual(expected, conf.raw())
Exemplo n.º 10
0
    def test_get(self):
        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
        )

        self.assertEqual({}, conf.get('multiple_sections'))
        self.assertEqual({'til': 'brooklyn', 'no': 'sleep'}, conf.get('derp'))
        self.assertEqual('brooklyn', conf.get('derp', 'til'))
        self.assertEqual(None, conf.get('derp', 'missing_key'))
        self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))
Exemplo n.º 11
0
 def test_invalid_path(self):
     conf = Configuration(
         name='invalid_path',
         path='/dev/null',
         parse=False
     )
     self.assertRaises(IOError, lambda: conf._parse_section(path=''))
     self.assertRaises(IOError, lambda: conf._parse_section(path=None))
     self.assertRaises(IOError, lambda: conf._parse_section(path='.'))
     self.assertRaises(IOError, lambda: conf._parse_section(path='/non-existent/path'))
Exemplo n.º 12
0
    def test_get(self):
        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
        )

        self.assertEqual({}, conf.get('multiple_sections'))
        self.assertEqual({'til': 'brooklyn', 'no': 'sleep'}, conf.get('derp'))
        self.assertEqual('brooklyn', conf.get('derp', 'til'))
        self.assertEqual(None, conf.get('derp', 'missing_key'))
        self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))
Exemplo n.º 13
0
    def test_readme(self):
        def digitize(config):
            for key in config:
                if not config[key].isdigit():
                    try:
                        config[key] = float(config[key])
                    except ValueError:
                        pass
                else:
                    try:
                        config[key] = int(config[key])
                    except ValueError:
                        pass

            return config

        conf = Configuration(
            name='derp',
            path='./data/conf',
            main_defaults={
                'no': 'one',
                'expected': 'the spanish inquisition',
                'cats': '1',
            },
            section_parser=digitize
        )
        expected = {
            'sections': {'herp': {'sleep': 1, 'wait': 5.0, 'timeout': 'seventy'}},
            'derp': {'expected': 'the spanish inquisition', 'til': 'brooklyn', 'cats': '1', 'no': 'sleep'}
        }
        self.assertEqual(expected, conf.raw())

        actual = conf.get(section='derp', key='no', default="jumping")
        self.assertEqual('sleep', actual)

        actual = conf.get(section='derp', key='til')
        self.assertEqual('brooklyn', actual)

        actual = conf.get(section='derp', key='cats')
        self.assertEqual('1', actual)

        actual = conf.get(section='derp', key='dogs')
        self.assertEqual(None, actual)

        actual = conf.get(section='herp', key='sleep')
        self.assertEqual(1, actual)

        actual = conf.get(section='herp', key='wait')
        self.assertEqual(5.0, actual)

        actual = conf.get(section='herp', key='timeout')
        self.assertEqual('seventy', actual)

        actual = conf.has(section='derp')
        self.assertEqual(True, actual)

        actual = conf.has(section='derp', key='no')
        self.assertEqual(True, actual)

        expected = {
            'sections': {
                'herp': {
                    'sleep': 1,
                    'wait': 5.0,
                    'timeout': 'seventy'
                }
            },
            'derp': {
                'expected': 'the spanish inquisition',
                'til': 'brooklyn',
                'cats': '1',
                'no': 'sleep'
            }
        }
        self.assertEqual(expected, conf.raw())
Exemplo n.º 14
0
    def test_parser(self):
        def safe_cast(val):
            try:
                return int(val)
            except ValueError:
                return val

        def all_as_bool(config):
            for key in config:
                config[key] = bool(config[key])
            return config

        def all_as_int(config):
            for key in config:
                try:
                    config[key] = int(config[key])
                except ValueError:
                    pass

            return config

        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            confd_path='./data/conf.d',
            main_defaults={'main_key': 'main_value'},
            section_defaults={'sleep': '2', 'wait': '30'},
            main_parser=all_as_bool
        )

        self.assertEqual({'main_key': True}, conf.get('multiple_sections'))
        self.assertEqual(True, conf.get('multiple_sections', 'main_key'))
        self.assertEqual('1', conf.get('another/conf', 'sleep'))

        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            confd_path='./data/conf.d',
            main_defaults={'main_key': 'main_value'},
            section_defaults={'sleep': '2', 'wait': '30'},
            section_parser=all_as_bool
        )

        self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
        self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
        self.assertEqual(1, conf.get('another/conf', 'sleep'))

        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            confd_path='./data/conf.d',
            main_defaults={'main_key': 'main_value'},
            section_defaults={'sleep': '2', 'wait': '30'},
            main_parser=all_as_bool,
            section_parser=all_as_int
        )

        self.assertEqual({'main_key': True}, conf.get('multiple_sections'))
        self.assertEqual(True, conf.get('multiple_sections', 'main_key'))
        self.assertEqual(None, conf.get('derp', 'missing_key'))
        self.assertEqual(1, conf.get('another/conf', 'sleep'))
Exemplo n.º 15
0
    def test_confd(self):
        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            confd_path='./data/conf.d',
            main_defaults={'main_key': 'main_value'}
        )

        self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
        self.assertEqual({'til': 'brooklyn', 'no': 'sleep'}, conf.get('derp'))
        self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
        self.assertEqual('brooklyn', conf.get('derp', 'til'))
        self.assertEqual(None, conf.get('derp', 'missing_key'))
        self.assertEqual(None, conf.get('derp', 'main_key'))
        self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))

        expected = {
            'multiple_sections': {'main_key': 'main_value'},
            'sections': {
                'another/conf': {'sleep': '1', 'wait': '15'},
                'derp': {'no': 'sleep', 'til': 'brooklyn'},
                'section': {'key': 'value'},
                'test': {'conf': 'path/to/another/conf', 'sleep': '15'}
            }
        }
        self.assertEqual(expected, conf.raw())

        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            confd_path='./data/conf.d',
            section_defaults={'sleep': '2', 'wait': '30'}
        )

        self.assertEqual({}, conf.get('multiple_sections'))
        self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp'))
        self.assertEqual(None, conf.get('multiple_sections', 'main_key'))
        self.assertEqual('brooklyn', conf.get('derp', 'til'))
        self.assertEqual(None, conf.get('derp', 'missing_key'))
        self.assertEqual(None, conf.get('derp', 'main_key'))
        self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))

        expected = {
            'multiple_sections': {},
            'sections': {
                'another/conf': {'sleep': '1', 'wait': '15'},
                'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'},
                'section': {'key': 'value', 'sleep': '2', 'wait': '30'},
                'test': {'conf': 'path/to/another/conf', 'sleep': '15', 'wait': '30'}
            }
        }
        self.assertEqual(expected, conf.raw())

        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            confd_path='./data/conf.d',
            main_defaults={'main_key': 'main_value'},
            section_defaults={'sleep': '2', 'wait': '30'}
        )

        self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
        self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp'))
        self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
        self.assertEqual('brooklyn', conf.get('derp', 'til'))
        self.assertEqual(None, conf.get('derp', 'missing_key'))
        self.assertEqual(None, conf.get('derp', 'main_key'))
        self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))

        expected = {
            'multiple_sections': {'main_key': 'main_value'},
            'sections': {
                'another/conf': {'sleep': '1', 'wait': '15'},
                'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'},
                'section': {'key': 'value', 'sleep': '2', 'wait': '30'},
                'test': {'conf': 'path/to/another/conf', 'sleep': '15', 'wait': '30'}
            }
        }
        self.assertEqual(expected, conf.raw())

        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            confd_path='./data/conf.d',
            conf_ext='conf',
            main_defaults={'main_key': 'main_value'},
            section_defaults={'sleep': '2', 'wait': '30'}
        )

        self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
        self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp'))
        self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
        self.assertEqual('brooklyn', conf.get('derp', 'til'))
        self.assertEqual(None, conf.get('derp', 'missing_key'))
        self.assertEqual(None, conf.get('derp', 'main_key'))
        self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))

        expected = {
            'multiple_sections': {'main_key': 'main_value'},
            'sections': {
                'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'},
                'section': {'key': 'value', 'sleep': '2', 'wait': '30'},
            }
        }
        self.assertEqual(expected, conf.raw())

        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            confd_path='./data/conf.d',
            conf_ext='.conf',
            main_defaults={'main_key': 'main_value'},
            section_defaults={'sleep': '2', 'wait': '30'}
        )

        self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
        self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp'))
        self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
        self.assertEqual('brooklyn', conf.get('derp', 'til'))
        self.assertEqual(None, conf.get('derp', 'missing_key'))
        self.assertEqual(None, conf.get('derp', 'main_key'))
        self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))

        expected = {
            'multiple_sections': {'main_key': 'main_value'},
            'sections': {
                'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'},
                'section': {'key': 'value', 'sleep': '2', 'wait': '30'},
            }
        }
        self.assertEqual(expected, conf.raw())
Exemplo n.º 16
0
    def _parse(self, args):
        def _main_parser(config):
            transpose = ['config', 'confd_path', 'debug', 'daemonize', 'files', 'format', 'fqdn', 'hostname', 'path', 'pid', 'transport']
            namspace_dict = vars(args)
            for key in transpose:
                if key not in namspace_dict or namspace_dict[key] is None or namspace_dict[key] == '':
                    continue

                config[key] = namspace_dict[key]

            if args.mode:
                config['zeromq_bind'] = args.mode

            # HACK: Python 2.6 ConfigParser does not properly
            #       handle non-string values
            for key in config:
                if config[key] == '':
                    config[key] = None

            require_bool = ['debug', 'daemonize', 'fqdn', 'rabbitmq_exchange_durable', 'rabbitmq_queue_durable',
                            'rabbitmq_ha_queue', 'rabbitmq_ssl', 'tcp_ssl_enabled', 'tcp_ssl_verify']

            for key in require_bool:
                config[key] = bool(int(config[key]))

            require_int = [
                'max_failure',
                'max_queue_size',
                'queue_timeout',
                'rabbitmq_port',
                'rabbitmq_timeout',
                'rabbitmq_delivery_mode',
                'respawn_delay',
                'subprocess_poll_sleep',
                'refresh_worker_process',
                'tcp_port',
                'udp_port',
                'wait_timeout',
                'zeromq_hwm',
                'logstash_version',
                'kafka_batch_n',
                'kafka_batch_t',
                'kafka_ack_timeout',
                'number_of_consumer_processes',
                'ignore_old_files'
            ]
            for key in require_int:
                if config[key] is not None:
                    config[key] = int(config[key])

            require_float = [
                'update_file_mapping_time',
                'discover_interval',
            ]

            for key in require_float:
                if config[key] is not None:
                    config[key] = float(config[key])

            if config.get('format') == 'null':
                config['format'] = 'raw'

            if config['files'] is not None and type(config['files']) == str:
                config['files'] = config['files'].split(',')

            if config['path'] is not None:
                config['path'] = os.path.realpath(config['path'])
                if not os.path.isdir(config['path']):
                    raise LookupError('{0} does not exist'.format(config['path']))

            if config.get('hostname') is None:
                if config.get('fqdn') is True:
                    config['hostname'] = socket.getfqdn()
                else:
                    config['hostname'] = socket.gethostname()

            if config.get('sincedb_path'):
                config['sincedb_path'] = os.path.realpath(config.get('sincedb_path'))

            if config['zeromq_address'] and type(config['zeromq_address']) == str:
                config['zeromq_address'] = [x.strip() for x in config.get('zeromq_address').split(',')]

            if config.get('ssh_options') is not None:
                csv = config.get('ssh_options')
                config['ssh_options'] = []
                if type(csv) == str:
                    for opt in csv.split(','):
                        config['ssh_options'].append('-o %s' % opt.strip())
            else:
                config['ssh_options'] = []

            config['globs'] = {}

            return config

        def _section_parser(config, raise_exceptions=True):
            '''Parse a given INI-style config file using ConfigParser module.
            Stanza's names match file names, and properties are defaulted as in
            http://logstash.net/docs/1.1.1/inputs/file

            Config file example:

            [/var/log/syslog]
            type: syslog
            tags: sys,main

            [/var/log/auth]
            type: syslog
            ;tags: auth,main
            '''

            fields = config.get('add_field', '')
            if type(fields) != dict:
                try:
                    if type(fields) == str:
                        fields = filter(None, fields.split(','))
                    if len(fields) == 0:
                        config['fields'] = {}
                    elif (len(fields) % 2) == 1:
                        if raise_exceptions:
                            raise Exception('Wrong number of values for add_field')
                    else:
                        fieldkeys = fields[0::2]
                        fieldvalues = [[x] for x in fields[1::2]]
                        config['fields'] = dict(zip(fieldkeys, fieldvalues))
                except TypeError:
                    config['fields'] = {}

            if 'add_field' in config:
                del config['add_field']

            envFields = config.get('add_field_env', '')
            if type(envFields) != dict:
                try:
                    if type(envFields) == str:
                        envFields = envFields.replace(" ","")
                        envFields = filter(None, envFields.split(','))
                    if len(envFields) == 0:
                        config['envFields'] = {}
                    elif (len(envFields) % 2) == 1:
                        if raise_exceptions:
                            raise Exception('Wrong number of values for add_field_env')
                    else:
                        envFieldkeys = envFields[0::2]
                        envFieldvalues = []
                        for x in envFields[1::2]:
                            envFieldvalues.append(os.environ.get(x))
                        config['fields'].update(dict(zip(envFieldkeys, envFieldvalues)))
                except TypeError:
                    config['envFields'] = {}

            if 'add_field_env' in config:
                del config['add_field_env']

            try:
                tags = config.get('tags', '')
                if type(tags) == str:
                    tags = filter(None, tags.split(','))
                if len(tags) == 0:
                    tags = []
                config['tags'] = tags
            except TypeError:
                config['tags'] = []

            if config.get('format') == 'null':
                config['format'] = 'raw'

            file_type = config.get('type', None)
            if not file_type:
                config['type'] = 'file'

            require_bool = ['debug', 'ignore_empty', 'ignore_truncate']
            for k in require_bool:
                config[k] = bool(int(config[k]))

            config['delimiter'] = config['delimiter'].decode('string-escape')

            if config['multiline_regex_after']:
                config['multiline_regex_after'] = re.compile(config['multiline_regex_after'])
            if config['multiline_regex_before']:
                config['multiline_regex_before'] = re.compile(config['multiline_regex_before'])

            require_int = ['sincedb_write_interval', 'stat_interval', 'tail_lines']
            for k in require_int:
                config[k] = int(config[k])

            return config

        conf = Configuration(
            name='beaver',
            path=self._configfile,
            main_defaults=self._main_defaults,
            section_defaults=self._section_defaults,
            main_parser=_main_parser,
            section_parser=_section_parser,
            path_from_main='confd_path',
            config_parser=self._config_parser
        )

        config = conf.raw()
        self._beaver_config = config['beaver']
        self._file_config = config['sections']

        self._main_parser = _main_parser(self._main_defaults)
        self._section_defaults = _section_parser(self._section_defaults, raise_exceptions=False)

        self._files = {}
        for section in config['sections']:
            globs = eglob(section, config['sections'][section].get('exclude', ''))
            if not globs:
                self._logger.debug('Skipping glob due to no files found: %s' % section)
                continue

            for globbed_file in globs:
                self._files[os.path.realpath(globbed_file)] = config['sections'][section]
Exemplo n.º 17
0
    def _parse(self, args):
        def _main_parser(config):
            transpose = ['config', 'debug', 'daemonize', 'files', 'format', 'fqdn', 'hostname', 'path', 'pid', 'transport']
            namspace_dict = vars(args)
            for key in transpose:
                if key not in namspace_dict or namspace_dict[key] is None or namspace_dict[key] == '':
                    continue

                config[key] = namspace_dict[key]

            if args.mode:
                config['zeromq_bind'] = args.mode

            # HACK: Python 2.6 ConfigParser does not properly
            #       handle non-string values
            for key in config:
                if config[key] == '':
                    config[key] = None

            require_bool = ['debug', 'daemonize', 'fqdn', 'rabbitmq_exchange_durable', 'rabbitmq_queue_durable', 'rabbitmq_ha_queue']

            for key in require_bool:
                config[key] = bool(int(config[key]))

            require_int = [
                'max_failure',
                'max_queue_size',
                'queue_timeout',
                'rabbitmq_port',
                'respawn_delay',
                'subprocess_poll_sleep',
                'udp_port',
                'wait_timeout',
                'zeromq_hwm',
            ]
            for key in require_int:
                if config[key] is not None:
                    config[key] = int(config[key])

            require_float = [
                'update_file_mapping_time',
                'discover_interval',
            ]

            for key in require_float:
                if config[key] is not None:
                    config[key] = float(config[key])

            if config['files'] is not None and type(config['files']) == str:
                config['files'] = config['files'].split(',')

            config['path'] = os.path.realpath(config['path'])
            if not os.path.isdir(config['path']):
                raise LookupError('{0} does not exist'.format(config['path']))

            if config.get('hostname') is None:
                if config.get('fqdn') is True:
                    config['hostname'] = socket.getfqdn()
                else:
                    config['hostname'] = socket.gethostname()

            if config.get('sincedb_path'):
                config['sincedb_path'] = os.path.realpath(config.get('sincedb_path'))

            config['globs'] = {}

            return config

        def _section_parser(config, raise_exceptions=True):
            '''Parse a given INI-style config file using ConfigParser module.
            Stanza's names match file names, and properties are defaulted as in
            http://logstash.net/docs/1.1.1/inputs/file

            Config file example:

            [/var/log/syslog]
            type: syslog
            tags: sys,main

            [/var/log/auth]
            type: syslog
            ;tags: auth,main
            '''

            fields = config.get('add_field', '')
            if type(fields) != dict:
                try:
                    if type(fields) == str:
                        fields = filter(None, fields.split(','))
                    if len(fields) == 0:
                        config['fields'] = {}
                    elif (len(fields) % 2) == 1:
                        if raise_exceptions:
                            raise Exception('Wrong number of values for add_field')
                    else:
                        fieldkeys = fields[0::2]
                        fieldvalues = [[x] for x in fields[1::2]]
                        config['fields'] = dict(zip(fieldkeys, fieldvalues))
                except TypeError:
                    config['fields'] = {}

            if 'add_field' in config:
                del config['add_field']

            try:
                tags = config.get('tags', '')
                if type(tags) == str:
                    tags = filter(None, tags.split(','))
                if len(tags) == 0:
                    tags = []
                config['tags'] = tags
            except TypeError:
                config['tags'] = []

            try:
                file_type = config.get('type', 'file')
                if not file_type:
                    file_type = 'file'
                config['type'] = file_type
            except:
                config['type'] = 'file'

            if config['type']:
                if raise_exceptions:
                    raise Exception('Missing mandatory config "type"')

            require_bool = ['debug', 'ignore_empty', 'ignore_truncate']
            for k in require_bool:
                config[k] = bool(int(config[k]))

            require_int = ['sincedb_write_interval', 'stat_interval', 'tail_lines']
            for k in require_int:
                config[k] = int(config[k])

            return config

        conf = Configuration(
            name='beaver',
            path=self._configfile,
            main_defaults=self._main_defaults,
            section_defaults=self._section_defaults,
            main_parser=_main_parser,
            section_parser=_section_parser,
        )

        config = conf.raw()
        self._beaver_config = config['beaver']
        self._file_config = config['sections']

        self._main_parser = _main_parser(self._main_defaults)
        self._section_defaults = _section_parser(self._section_defaults, raise_exceptions=False)

        self._files = {}
        for section in config['sections']:
            globs = eglob(section, config['sections'][section].get('exclude', ''))
            if not globs:
                self._logger.debug('Skipping glob due to no files found: %s' % section)
                continue

            for globbed_file in globs:
                self._files[os.path.realpath(globbed_file)] = config['sections'][section]
Exemplo n.º 18
0
    def test_confd(self):
        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            confd_path='./data/conf.d',
            main_defaults={'main_key': 'main_value'}
        )

        self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
        self.assertEqual({'til': 'brooklyn', 'no': 'sleep'}, conf.get('derp'))
        self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
        self.assertEqual('brooklyn', conf.get('derp', 'til'))
        self.assertEqual(None, conf.get('derp', 'missing_key'))
        self.assertEqual(None, conf.get('derp', 'main_key'))
        self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))

        expected = {
            'multiple_sections': {'main_key': 'main_value'},
            'sections': {
                'another/conf': {'sleep': '1', 'wait': '15'},
                'derp': {'no': 'sleep', 'til': 'brooklyn'},
                'section': {'key': 'value'},
                'test': {'conf': 'path/to/another/conf', 'sleep': '15'}
            }
        }
        self.assertEqual(expected, conf.raw())

        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            confd_path='./data/conf.d',
            section_defaults={'sleep': '2', 'wait': '30'}
        )

        self.assertEqual({}, conf.get('multiple_sections'))
        self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp'))
        self.assertEqual(None, conf.get('multiple_sections', 'main_key'))
        self.assertEqual('brooklyn', conf.get('derp', 'til'))
        self.assertEqual(None, conf.get('derp', 'missing_key'))
        self.assertEqual(None, conf.get('derp', 'main_key'))
        self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))

        expected = {
            'multiple_sections': {},
            'sections': {
                'another/conf': {'sleep': '1', 'wait': '15'},
                'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'},
                'section': {'key': 'value', 'sleep': '2', 'wait': '30'},
                'test': {'conf': 'path/to/another/conf', 'sleep': '15', 'wait': '30'}
            }
        }
        self.assertEqual(expected, conf.raw())

        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            confd_path='./data/conf.d',
            main_defaults={'main_key': 'main_value'},
            section_defaults={'sleep': '2', 'wait': '30'}
        )

        self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
        self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp'))
        self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
        self.assertEqual('brooklyn', conf.get('derp', 'til'))
        self.assertEqual(None, conf.get('derp', 'missing_key'))
        self.assertEqual(None, conf.get('derp', 'main_key'))
        self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))

        expected = {
            'multiple_sections': {'main_key': 'main_value'},
            'sections': {
                'another/conf': {'sleep': '1', 'wait': '15'},
                'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'},
                'section': {'key': 'value', 'sleep': '2', 'wait': '30'},
                'test': {'conf': 'path/to/another/conf', 'sleep': '15', 'wait': '30'}
            }
        }
        self.assertEqual(expected, conf.raw())

        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            confd_path='./data/conf.d',
            conf_ext='conf',
            main_defaults={'main_key': 'main_value'},
            section_defaults={'sleep': '2', 'wait': '30'}
        )

        self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
        self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp'))
        self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
        self.assertEqual('brooklyn', conf.get('derp', 'til'))
        self.assertEqual(None, conf.get('derp', 'missing_key'))
        self.assertEqual(None, conf.get('derp', 'main_key'))
        self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))

        expected = {
            'multiple_sections': {'main_key': 'main_value'},
            'sections': {
                'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'},
                'section': {'key': 'value', 'sleep': '2', 'wait': '30'},
            }
        }
        self.assertEqual(expected, conf.raw())

        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            confd_path='./data/conf.d',
            conf_ext='.conf',
            main_defaults={'main_key': 'main_value'},
            section_defaults={'sleep': '2', 'wait': '30'}
        )

        self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
        self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp'))
        self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
        self.assertEqual('brooklyn', conf.get('derp', 'til'))
        self.assertEqual(None, conf.get('derp', 'missing_key'))
        self.assertEqual(None, conf.get('derp', 'main_key'))
        self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))

        expected = {
            'multiple_sections': {'main_key': 'main_value'},
            'sections': {
                'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'},
                'section': {'key': 'value', 'sleep': '2', 'wait': '30'},
            }
        }
        self.assertEqual(expected, conf.raw())
Exemplo n.º 19
0
    def test_readme(self):
        def digitize(config):
            for key in config:
                if not config[key].isdigit():
                    try:
                        config[key] = float(config[key])
                    except ValueError:
                        pass
                else:
                    try:
                        config[key] = int(config[key])
                    except ValueError:
                        pass

            return config

        conf = Configuration(
            name='derp',
            path='./data/conf',
            main_defaults={
                'no': 'one',
                'expected': 'the spanish inquisition',
                'cats': '1',
            },
            section_parser=digitize
        )
        expected = {
            'sections': {'herp': {'sleep': 1, 'wait': 5.0, 'timeout': 'seventy'}},
            'derp': {'expected': 'the spanish inquisition', 'til': 'brooklyn', 'cats': '1', 'no': 'sleep'}
        }
        self.assertEqual(expected, conf.raw())

        actual = conf.get(section='derp', key='no', default="jumping")
        self.assertEqual('sleep', actual)

        actual = conf.get(section='derp', key='til')
        self.assertEqual('brooklyn', actual)

        actual = conf.get(section='derp', key='cats')
        self.assertEqual('1', actual)

        actual = conf.get(section='derp', key='dogs')
        self.assertEqual(None, actual)

        actual = conf.get(section='herp', key='sleep')
        self.assertEqual(1, actual)

        actual = conf.get(section='herp', key='wait')
        self.assertEqual(5.0, actual)

        actual = conf.get(section='herp', key='timeout')
        self.assertEqual('seventy', actual)

        actual = conf.has(section='derp')
        self.assertEqual(True, actual)

        actual = conf.has(section='derp', key='no')
        self.assertEqual(True, actual)

        expected = {
            'sections': {
                'herp': {
                    'sleep': 1,
                    'wait': 5.0,
                    'timeout': 'seventy'
                }
            },
            'derp': {
                'expected': 'the spanish inquisition',
                'til': 'brooklyn',
                'cats': '1',
                'no': 'sleep'
            }
        }
        self.assertEqual(expected, conf.raw())
Exemplo n.º 20
0
    def test_parser(self):
        def safe_cast(val):
            try:
                return int(val)
            except ValueError:
                return val

        def all_as_bool(config):
            for key in config:
                config[key] = bool(config[key])
            return config

        def all_as_int(config):
            for key in config:
                try:
                    config[key] = int(config[key])
                except ValueError:
                    pass

            return config

        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            confd_path='./data/conf.d',
            main_defaults={'main_key': 'main_value'},
            section_defaults={'sleep': '2', 'wait': '30'},
            main_parser=all_as_bool
        )

        self.assertEqual({'main_key': True}, conf.get('multiple_sections'))
        self.assertEqual(True, conf.get('multiple_sections', 'main_key'))
        self.assertEqual('1', conf.get('another/conf', 'sleep'))

        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            confd_path='./data/conf.d',
            main_defaults={'main_key': 'main_value'},
            section_defaults={'sleep': '2', 'wait': '30'},
            section_parser=all_as_bool
        )

        self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections'))
        self.assertEqual('main_value', conf.get('multiple_sections', 'main_key'))
        self.assertEqual(1, conf.get('another/conf', 'sleep'))

        conf = Configuration(
            name='multiple_sections',
            path='./data/multiple_sections.ini',
            confd_path='./data/conf.d',
            main_defaults={'main_key': 'main_value'},
            section_defaults={'sleep': '2', 'wait': '30'},
            main_parser=all_as_bool,
            section_parser=all_as_int
        )

        self.assertEqual({'main_key': True}, conf.get('multiple_sections'))
        self.assertEqual(True, conf.get('multiple_sections', 'main_key'))
        self.assertEqual(None, conf.get('derp', 'missing_key'))
        self.assertEqual(1, conf.get('another/conf', 'sleep'))
Exemplo n.º 21
0
    def _parse(self, args):
        def _main_parser(config):
            transpose = [
                'config', 'confd_path', 'debug', 'daemonize', 'files',
                'format', 'fqdn', 'hostname', 'path', 'pid', 'transport'
            ]
            namspace_dict = vars(args)
            for key in transpose:
                if key not in namspace_dict or namspace_dict[
                        key] is None or namspace_dict[key] == '':
                    continue

                config[key] = namspace_dict[key]

            if args.mode:
                config['zeromq_bind'] = args.mode

            # HACK: Python 2.6 ConfigParser does not properly
            #       handle non-string values
            for key in config:
                if config[key] == '':
                    config[key] = None

            require_bool = [
                'debug', 'daemonize', 'fqdn', 'rabbitmq_exchange_durable',
                'rabbitmq_queue_durable', 'rabbitmq_ha_queue'
            ]

            for key in require_bool:
                config[key] = bool(int(config[key]))

            require_int = [
                'max_failure',
                'max_queue_size',
                'queue_timeout',
                'rabbitmq_port',
                'respawn_delay',
                'subprocess_poll_sleep',
                'tcp_port',
                'udp_port',
                'wait_timeout',
                'zeromq_hwm',
            ]
            for key in require_int:
                if config[key] is not None:
                    config[key] = int(config[key])

            require_float = [
                'update_file_mapping_time',
                'discover_interval',
            ]

            for key in require_float:
                if config[key] is not None:
                    config[key] = float(config[key])

            if config.get('format') == 'null':
                config['format'] = 'raw'

            if config['files'] is not None and type(config['files']) == str:
                config['files'] = config['files'].split(',')

            if config['path'] is not None:
                config['path'] = os.path.realpath(config['path'])
                if not os.path.isdir(config['path']):
                    raise LookupError('{0} does not exist'.format(
                        config['path']))

            if config.get('hostname') is None:
                if config.get('fqdn') is True:
                    config['hostname'] = socket.getfqdn()
                else:
                    config['hostname'] = socket.gethostname()

            if config.get('sincedb_path'):
                config['sincedb_path'] = os.path.realpath(
                    config.get('sincedb_path'))

            config['globs'] = {}

            return config

        def _section_parser(config, raise_exceptions=True):
            '''Parse a given INI-style config file using ConfigParser module.
            Stanza's names match file names, and properties are defaulted as in
            http://logstash.net/docs/1.1.1/inputs/file

            Config file example:

            [/var/log/syslog]
            type: syslog
            tags: sys,main

            [/var/log/auth]
            type: syslog
            ;tags: auth,main
            '''

            fields = config.get('add_field', '')
            if type(fields) != dict:
                try:
                    if type(fields) == str:
                        fields = filter(None, fields.split(','))
                    if len(fields) == 0:
                        config['fields'] = {}
                    elif (len(fields) % 2) == 1:
                        if raise_exceptions:
                            raise Exception(
                                'Wrong number of values for add_field')
                    else:
                        fieldkeys = fields[0::2]
                        fieldvalues = [[x] for x in fields[1::2]]
                        config['fields'] = dict(zip(fieldkeys, fieldvalues))
                except TypeError:
                    config['fields'] = {}

            if 'add_field' in config:
                del config['add_field']

            try:
                tags = config.get('tags', '')
                if type(tags) == str:
                    tags = filter(None, tags.split(','))
                if len(tags) == 0:
                    tags = []
                config['tags'] = tags
            except TypeError:
                config['tags'] = []

            if config.get('format') == 'null':
                config['format'] = 'raw'

            file_type = config.get('type', None)
            if not file_type:
                config['type'] = 'file'

            require_bool = ['debug', 'ignore_empty', 'ignore_truncate']
            for k in require_bool:
                config[k] = bool(int(config[k]))

            require_int = [
                'sincedb_write_interval', 'stat_interval', 'tail_lines'
            ]
            for k in require_int:
                config[k] = int(config[k])

            return config

        conf = Configuration(name='beaver',
                             path=self._configfile,
                             main_defaults=self._main_defaults,
                             section_defaults=self._section_defaults,
                             main_parser=_main_parser,
                             section_parser=_section_parser,
                             path_from_main='confd_path')

        config = conf.raw()
        self._beaver_config = config['beaver']
        self._file_config = config['sections']

        self._main_parser = _main_parser(self._main_defaults)
        self._section_defaults = _section_parser(self._section_defaults,
                                                 raise_exceptions=False)

        self._files = {}
        for section in config['sections']:
            globs = eglob(section,
                          config['sections'][section].get('exclude', ''))
            if not globs:
                self._logger.debug('Skipping glob due to no files found: %s' %
                                   section)
                continue

            for globbed_file in globs:
                self._files[os.path.realpath(
                    globbed_file)] = config['sections'][section]
Exemplo n.º 22
0
    def _parse(self, args):
        def _main_parser(config):
            transpose = [
                "config",
                "confd_path",
                "debug",
                "daemonize",
                "files",
                "format",
                "fqdn",
                "hostname",
                "path",
                "pid",
                "transport",
            ]
            namspace_dict = vars(args)
            for key in transpose:
                if key not in namspace_dict or namspace_dict[key] is None or namspace_dict[key] == "":
                    continue

                config[key] = namspace_dict[key]

            if args.mode:
                config["zeromq_bind"] = args.mode

            # HACK: Python 2.6 ConfigParser does not properly
            #       handle non-string values
            for key in config:
                if config[key] == "":
                    config[key] = None

            require_bool = [
                "debug",
                "daemonize",
                "fqdn",
                "rabbitmq_exchange_durable",
                "rabbitmq_queue_durable",
                "rabbitmq_ha_queue",
                "rabbitmq_ssl",
                "tcp_ssl_enabled",
                "tcp_ssl_verify",
            ]

            for key in require_bool:
                config[key] = bool(int(config[key]))

            require_int = [
                "max_failure",
                "max_queue_size",
                "queue_timeout",
                "rabbitmq_port",
                "rabbitmq_timeout",
                "rabbitmq_delivery_mode",
                "respawn_delay",
                "subprocess_poll_sleep",
                "refresh_worker_process",
                "tcp_port",
                "udp_port",
                "wait_timeout",
                "zeromq_hwm",
                "logstash_version",
                "kafka_batch_n",
                "kafka_batch_t",
                "kafka_ack_timeout",
                "number_of_consumer_processes",
            ]
            for key in require_int:
                if config[key] is not None:
                    config[key] = int(config[key])

            require_float = ["update_file_mapping_time", "discover_interval"]

            for key in require_float:
                if config[key] is not None:
                    config[key] = float(config[key])

            if config.get("format") == "null":
                config["format"] = "raw"

            if config["files"] is not None and type(config["files"]) == str:
                config["files"] = config["files"].split(",")

            if config["path"] is not None:
                config["path"] = os.path.realpath(config["path"])
                if not os.path.isdir(config["path"]):
                    raise LookupError("{0} does not exist".format(config["path"]))

            if config.get("hostname") is None:
                if config.get("fqdn") is True:
                    config["hostname"] = socket.getfqdn()
                else:
                    config["hostname"] = socket.gethostname()

            if config.get("sincedb_path"):
                config["sincedb_path"] = os.path.realpath(config.get("sincedb_path"))

            if config["zeromq_address"] and type(config["zeromq_address"]) == str:
                config["zeromq_address"] = [x.strip() for x in config.get("zeromq_address").split(",")]

            if config.get("ssh_options") is not None:
                csv = config.get("ssh_options")
                config["ssh_options"] = []
                if type(csv) == str:
                    for opt in csv.split(","):
                        config["ssh_options"].append("-o %s" % opt.strip())
            else:
                config["ssh_options"] = []

            config["globs"] = {}

            return config

        def _section_parser(config, raise_exceptions=True):
            """Parse a given INI-style config file using ConfigParser module.
            Stanza's names match file names, and properties are defaulted as in
            http://logstash.net/docs/1.1.1/inputs/file

            Config file example:

            [/var/log/syslog]
            type: syslog
            tags: sys,main

            [/var/log/auth]
            type: syslog
            ;tags: auth,main
            """

            fields = config.get("add_field", "")
            if type(fields) != dict:
                try:
                    if type(fields) == str:
                        fields = filter(None, fields.split(","))
                    if len(fields) == 0:
                        config["fields"] = {}
                    elif (len(fields) % 2) == 1:
                        if raise_exceptions:
                            raise Exception("Wrong number of values for add_field")
                    else:
                        fieldkeys = fields[0::2]
                        fieldvalues = [[x] for x in fields[1::2]]
                        config["fields"] = dict(zip(fieldkeys, fieldvalues))
                except TypeError:
                    config["fields"] = {}

            if "add_field" in config:
                del config["add_field"]

            envFields = config.get("add_field_env", "")
            if type(envFields) != dict:
                try:
                    if type(envFields) == str:
                        envFields = envFields.replace(" ", "")
                        envFields = filter(None, envFields.split(","))
                    if len(envFields) == 0:
                        config["envFields"] = {}
                    elif (len(envFields) % 2) == 1:
                        if raise_exceptions:
                            raise Exception("Wrong number of values for add_field_env")
                    else:
                        envFieldkeys = envFields[0::2]
                        envFieldvalues = []
                        for x in envFields[1::2]:
                            envFieldvalues.append(os.environ.get(x))
                        config["fields"].update(dict(zip(envFieldkeys, envFieldvalues)))
                except TypeError:
                    config["envFields"] = {}

            if "add_field_env" in config:
                del config["add_field_env"]

            try:
                tags = config.get("tags", "")
                if type(tags) == str:
                    tags = filter(None, tags.split(","))
                if len(tags) == 0:
                    tags = []
                config["tags"] = tags
            except TypeError:
                config["tags"] = []

            if config.get("format") == "null":
                config["format"] = "raw"

            file_type = config.get("type", None)
            if not file_type:
                config["type"] = "file"

            require_bool = ["debug", "ignore_empty", "ignore_truncate"]
            for k in require_bool:
                config[k] = bool(int(config[k]))

            config["delimiter"] = config["delimiter"].decode("string-escape")

            if config["multiline_regex_after"]:
                config["multiline_regex_after"] = re.compile(config["multiline_regex_after"])
            if config["multiline_regex_before"]:
                config["multiline_regex_before"] = re.compile(config["multiline_regex_before"])

            require_int = ["sincedb_write_interval", "stat_interval", "tail_lines"]
            for k in require_int:
                config[k] = int(config[k])

            return config

        conf = Configuration(
            name="beaver",
            path=self._configfile,
            main_defaults=self._main_defaults,
            section_defaults=self._section_defaults,
            main_parser=_main_parser,
            section_parser=_section_parser,
            path_from_main="confd_path",
            config_parser=self._config_parser,
        )

        config = conf.raw()
        self._beaver_config = config["beaver"]
        self._file_config = config["sections"]

        self._main_parser = _main_parser(self._main_defaults)
        self._section_defaults = _section_parser(self._section_defaults, raise_exceptions=False)

        self._files = {}
        for section in config["sections"]:
            globs = eglob(section, config["sections"][section].get("exclude", ""))
            if not globs:
                self._logger.debug("Skipping glob due to no files found: %s" % section)
                continue

            for globbed_file in globs:
                self._files[os.path.realpath(globbed_file)] = config["sections"][section]