def test_parse(self): conf = Configuration( name='invalid_path', path='/dev/null', parse=False ) expected = {} actual = conf._parse_section(path='./data/empty.ini') self.assertEqual(expected, actual) expected = {'section': {'key': 'value'}} actual = conf._parse_section(path='./data/single_section.ini') self.assertEqual(expected, actual) expected = {'section': {'key': 'value', 'key_two': 'other_value'}} actual = conf._parse_section(path='./data/single_section.ini', defaults={'key_two': 'other_value'}) self.assertEqual(expected, actual) expected = {'section': {'key': 'value', 'key_two': 'other_value'}} actual = conf._parse_section(path='./data/single_section.ini', defaults={'key_two': 'other_value'}, only_section='section') self.assertEqual(expected, actual) expected = {} actual = conf._parse_section(path='./data/single_section.ini', defaults={'key_two': 'other_value'}, remove_section='section') self.assertEqual(expected, actual) expected = { 'derp': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'sleep', 'til': 'brooklyn'}, 'multiple_sections': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'one'}, 'section': {'cats': '1', 'expected': 'the spanish inquisition', 'key': 'value', 'no': 'one'} } actual = conf._parse_section(path='./data/multiple_sections.ini', defaults={ 'no': 'one', 'expected': 'the spanish inquisition', 'cats': '1', }) self.assertEqual(expected, actual) expected = { 'multiple_sections': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'one'}, } actual = conf._parse_section(path='./data/multiple_sections.ini', only_section='multiple_sections', defaults={ 'no': 'one', 'expected': 'the spanish inquisition', 'cats': '1', }) self.assertEqual(expected, actual) expected = { 'multiple_sections': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'one'}, 'section': {'cats': '1', 'expected': 'the spanish inquisition', 'key': 'value', 'no': 'one'} } actual = conf._parse_section(path='./data/multiple_sections.ini', remove_section='derp', defaults={ 'no': 'one', 'expected': 'the spanish inquisition', 'cats': '1', }) self.assertEqual(expected, actual)
def test_has_section(self): conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', ) self.assertEqual(True, conf.has('multiple_sections')) self.assertEqual(True, conf.has('derp')) self.assertEqual(True, conf.has('derp', 'no'))
def test_custom_config_parser(self): conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', config_parser=TestConfigParser, parse=False ) self.assertRaises(NotImplementedError, lambda: conf._parse_section(path='./data/multiple_sections.ini'))
def test_invalid_path(self): conf = Configuration( name='invalid_path', path='/dev/null', parse=False ) self.assertRaises(IOError, lambda: conf._parse_section(path='')) self.assertRaises(IOError, lambda: conf._parse_section(path=None)) self.assertRaises(IOError, lambda: conf._parse_section(path='.')) self.assertRaises(IOError, lambda: conf._parse_section(path='/non-existent/path'))
def test_parse(self): conf = Configuration( name='invalid_path', path='/dev/null', parse=False ) expected = {} actual = conf._parse_section(path='./data/empty.ini') self.assertDictEqual(expected, actual) expected = {'section': {'key': 'value'}} actual = conf._parse_section(path='./data/single_section.ini') self.assertDictEqual(expected, actual) expected = {'section': {'key': 'value', 'key_two': 'other_value'}} actual = conf._parse_section(path='./data/single_section.ini', defaults={'key_two': 'other_value'}) self.assertDictEqual(expected, actual) expected = {'section': {'key': 'value', 'key_two': 'other_value'}} actual = conf._parse_section(path='./data/single_section.ini', defaults={'key_two': 'other_value'}, only_section='section') self.assertDictEqual(expected, actual) expected = {} actual = conf._parse_section(path='./data/single_section.ini', defaults={'key_two': 'other_value'}, remove_section='section') self.assertDictEqual(expected, actual) expected = { 'derp': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'sleep', 'til': 'brooklyn'}, 'multiple_sections': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'one'}, 'section': {'cats': '1', 'expected': 'the spanish inquisition', 'key': 'value', 'no': 'one'} } actual = conf._parse_section(path='./data/multiple_sections.ini', defaults={ 'no': 'one', 'expected': 'the spanish inquisition', 'cats': '1', }) self.assertDictEqual(expected, actual) expected = { 'multiple_sections': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'one'}, } actual = conf._parse_section(path='./data/multiple_sections.ini', only_section='multiple_sections', defaults={ 'no': 'one', 'expected': 'the spanish inquisition', 'cats': '1', }) self.assertDictEqual(expected, actual) expected = { 'multiple_sections': {'cats': '1', 'expected': 'the spanish inquisition', 'no': 'one'}, 'section': {'cats': '1', 'expected': 'the spanish inquisition', 'key': 'value', 'no': 'one'} } actual = conf._parse_section(path='./data/multiple_sections.ini', remove_section='derp', defaults={ 'no': 'one', 'expected': 'the spanish inquisition', 'cats': '1', }) self.assertDictEqual(expected, actual)
def test_raw(self): conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', main_defaults={'main_key': 'main_value'}, section_defaults={'section_key': 'section_value'} ) self.assertEqual({'main_key': 'main_value'}, conf.raw('multiple_sections')) self.assertEqual({'key': 'value', 'section_key': 'section_value'}, conf.raw('section')) self.assertEqual({'no': 'sleep', 'section_key': 'section_value', 'til': 'brooklyn'}, conf.raw('derp')) expected = { 'multiple_sections': {'main_key': 'main_value'}, 'sections': { 'derp': {'no': 'sleep', 'section_key': 'section_value', 'til': 'brooklyn'}, 'section': {'key': 'value', 'section_key': 'section_value'} } } self.assertEqual(expected, conf.raw())
def test_get(self): conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', ) self.assertEqual({}, conf.get('multiple_sections')) self.assertEqual({'til': 'brooklyn', 'no': 'sleep'}, conf.get('derp')) self.assertEqual('brooklyn', conf.get('derp', 'til')) self.assertEqual(None, conf.get('derp', 'missing_key')) self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1'))
def test_readme(self): def digitize(config): for key in config: if not config[key].isdigit(): try: config[key] = float(config[key]) except ValueError: pass else: try: config[key] = int(config[key]) except ValueError: pass return config conf = Configuration( name='derp', path='./data/conf', main_defaults={ 'no': 'one', 'expected': 'the spanish inquisition', 'cats': '1', }, section_parser=digitize ) expected = { 'sections': {'herp': {'sleep': 1, 'wait': 5.0, 'timeout': 'seventy'}}, 'derp': {'expected': 'the spanish inquisition', 'til': 'brooklyn', 'cats': '1', 'no': 'sleep'} } self.assertEqual(expected, conf.raw()) actual = conf.get(section='derp', key='no', default="jumping") self.assertEqual('sleep', actual) actual = conf.get(section='derp', key='til') self.assertEqual('brooklyn', actual) actual = conf.get(section='derp', key='cats') self.assertEqual('1', actual) actual = conf.get(section='derp', key='dogs') self.assertEqual(None, actual) actual = conf.get(section='herp', key='sleep') self.assertEqual(1, actual) actual = conf.get(section='herp', key='wait') self.assertEqual(5.0, actual) actual = conf.get(section='herp', key='timeout') self.assertEqual('seventy', actual) actual = conf.has(section='derp') self.assertEqual(True, actual) actual = conf.has(section='derp', key='no') self.assertEqual(True, actual) expected = { 'sections': { 'herp': { 'sleep': 1, 'wait': 5.0, 'timeout': 'seventy' } }, 'derp': { 'expected': 'the spanish inquisition', 'til': 'brooklyn', 'cats': '1', 'no': 'sleep' } } self.assertEqual(expected, conf.raw())
def test_parser(self): def safe_cast(val): try: return int(val) except ValueError: return val def all_as_bool(config): for key in config: config[key] = bool(config[key]) return config def all_as_int(config): for key in config: try: config[key] = int(config[key]) except ValueError: pass return config conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', confd_path='./data/conf.d', main_defaults={'main_key': 'main_value'}, section_defaults={'sleep': '2', 'wait': '30'}, main_parser=all_as_bool ) self.assertEqual({'main_key': True}, conf.get('multiple_sections')) self.assertEqual(True, conf.get('multiple_sections', 'main_key')) self.assertEqual('1', conf.get('another/conf', 'sleep')) conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', confd_path='./data/conf.d', main_defaults={'main_key': 'main_value'}, section_defaults={'sleep': '2', 'wait': '30'}, section_parser=all_as_bool ) self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections')) self.assertEqual('main_value', conf.get('multiple_sections', 'main_key')) self.assertEqual(1, conf.get('another/conf', 'sleep')) conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', confd_path='./data/conf.d', main_defaults={'main_key': 'main_value'}, section_defaults={'sleep': '2', 'wait': '30'}, main_parser=all_as_bool, section_parser=all_as_int ) self.assertEqual({'main_key': True}, conf.get('multiple_sections')) self.assertEqual(True, conf.get('multiple_sections', 'main_key')) self.assertEqual(None, conf.get('derp', 'missing_key')) self.assertEqual(1, conf.get('another/conf', 'sleep'))
def test_confd(self): conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', confd_path='./data/conf.d', main_defaults={'main_key': 'main_value'} ) self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections')) self.assertEqual({'til': 'brooklyn', 'no': 'sleep'}, conf.get('derp')) self.assertEqual('main_value', conf.get('multiple_sections', 'main_key')) self.assertEqual('brooklyn', conf.get('derp', 'til')) self.assertEqual(None, conf.get('derp', 'missing_key')) self.assertEqual(None, conf.get('derp', 'main_key')) self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1')) expected = { 'multiple_sections': {'main_key': 'main_value'}, 'sections': { 'another/conf': {'sleep': '1', 'wait': '15'}, 'derp': {'no': 'sleep', 'til': 'brooklyn'}, 'section': {'key': 'value'}, 'test': {'conf': 'path/to/another/conf', 'sleep': '15'} } } self.assertEqual(expected, conf.raw()) conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', confd_path='./data/conf.d', section_defaults={'sleep': '2', 'wait': '30'} ) self.assertEqual({}, conf.get('multiple_sections')) self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp')) self.assertEqual(None, conf.get('multiple_sections', 'main_key')) self.assertEqual('brooklyn', conf.get('derp', 'til')) self.assertEqual(None, conf.get('derp', 'missing_key')) self.assertEqual(None, conf.get('derp', 'main_key')) self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1')) expected = { 'multiple_sections': {}, 'sections': { 'another/conf': {'sleep': '1', 'wait': '15'}, 'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, 'section': {'key': 'value', 'sleep': '2', 'wait': '30'}, 'test': {'conf': 'path/to/another/conf', 'sleep': '15', 'wait': '30'} } } self.assertEqual(expected, conf.raw()) conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', confd_path='./data/conf.d', main_defaults={'main_key': 'main_value'}, section_defaults={'sleep': '2', 'wait': '30'} ) self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections')) self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp')) self.assertEqual('main_value', conf.get('multiple_sections', 'main_key')) self.assertEqual('brooklyn', conf.get('derp', 'til')) self.assertEqual(None, conf.get('derp', 'missing_key')) self.assertEqual(None, conf.get('derp', 'main_key')) self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1')) expected = { 'multiple_sections': {'main_key': 'main_value'}, 'sections': { 'another/conf': {'sleep': '1', 'wait': '15'}, 'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, 'section': {'key': 'value', 'sleep': '2', 'wait': '30'}, 'test': {'conf': 'path/to/another/conf', 'sleep': '15', 'wait': '30'} } } self.assertEqual(expected, conf.raw()) conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', confd_path='./data/conf.d', conf_ext='conf', main_defaults={'main_key': 'main_value'}, section_defaults={'sleep': '2', 'wait': '30'} ) self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections')) self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp')) self.assertEqual('main_value', conf.get('multiple_sections', 'main_key')) self.assertEqual('brooklyn', conf.get('derp', 'til')) self.assertEqual(None, conf.get('derp', 'missing_key')) self.assertEqual(None, conf.get('derp', 'main_key')) self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1')) expected = { 'multiple_sections': {'main_key': 'main_value'}, 'sections': { 'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, 'section': {'key': 'value', 'sleep': '2', 'wait': '30'}, } } self.assertEqual(expected, conf.raw()) conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', confd_path='./data/conf.d', conf_ext='.conf', main_defaults={'main_key': 'main_value'}, section_defaults={'sleep': '2', 'wait': '30'} ) self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections')) self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp')) self.assertEqual('main_value', conf.get('multiple_sections', 'main_key')) self.assertEqual('brooklyn', conf.get('derp', 'til')) self.assertEqual(None, conf.get('derp', 'missing_key')) self.assertEqual(None, conf.get('derp', 'main_key')) self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1')) expected = { 'multiple_sections': {'main_key': 'main_value'}, 'sections': { 'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, 'section': {'key': 'value', 'sleep': '2', 'wait': '30'}, } } self.assertEqual(expected, conf.raw())
def _parse(self, args): def _main_parser(config): transpose = ['config', 'confd_path', 'debug', 'daemonize', 'files', 'format', 'fqdn', 'hostname', 'path', 'pid', 'transport'] namspace_dict = vars(args) for key in transpose: if key not in namspace_dict or namspace_dict[key] is None or namspace_dict[key] == '': continue config[key] = namspace_dict[key] if args.mode: config['zeromq_bind'] = args.mode # HACK: Python 2.6 ConfigParser does not properly # handle non-string values for key in config: if config[key] == '': config[key] = None require_bool = ['debug', 'daemonize', 'fqdn', 'rabbitmq_exchange_durable', 'rabbitmq_queue_durable', 'rabbitmq_ha_queue', 'rabbitmq_ssl', 'tcp_ssl_enabled', 'tcp_ssl_verify'] for key in require_bool: config[key] = bool(int(config[key])) require_int = [ 'max_failure', 'max_queue_size', 'queue_timeout', 'rabbitmq_port', 'rabbitmq_timeout', 'rabbitmq_delivery_mode', 'respawn_delay', 'subprocess_poll_sleep', 'refresh_worker_process', 'tcp_port', 'udp_port', 'wait_timeout', 'zeromq_hwm', 'logstash_version', 'kafka_batch_n', 'kafka_batch_t', 'kafka_ack_timeout', 'number_of_consumer_processes', 'ignore_old_files' ] for key in require_int: if config[key] is not None: config[key] = int(config[key]) require_float = [ 'update_file_mapping_time', 'discover_interval', ] for key in require_float: if config[key] is not None: config[key] = float(config[key]) if config.get('format') == 'null': config['format'] = 'raw' if config['files'] is not None and type(config['files']) == str: config['files'] = config['files'].split(',') if config['path'] is not None: config['path'] = os.path.realpath(config['path']) if not os.path.isdir(config['path']): raise LookupError('{0} does not exist'.format(config['path'])) if config.get('hostname') is None: if config.get('fqdn') is True: config['hostname'] = socket.getfqdn() else: config['hostname'] = socket.gethostname() if config.get('sincedb_path'): config['sincedb_path'] = os.path.realpath(config.get('sincedb_path')) if config['zeromq_address'] and type(config['zeromq_address']) == str: config['zeromq_address'] = [x.strip() for x in config.get('zeromq_address').split(',')] if config.get('ssh_options') is not None: csv = config.get('ssh_options') config['ssh_options'] = [] if type(csv) == str: for opt in csv.split(','): config['ssh_options'].append('-o %s' % opt.strip()) else: config['ssh_options'] = [] config['globs'] = {} return config def _section_parser(config, raise_exceptions=True): '''Parse a given INI-style config file using ConfigParser module. Stanza's names match file names, and properties are defaulted as in http://logstash.net/docs/1.1.1/inputs/file Config file example: [/var/log/syslog] type: syslog tags: sys,main [/var/log/auth] type: syslog ;tags: auth,main ''' fields = config.get('add_field', '') if type(fields) != dict: try: if type(fields) == str: fields = filter(None, fields.split(',')) if len(fields) == 0: config['fields'] = {} elif (len(fields) % 2) == 1: if raise_exceptions: raise Exception('Wrong number of values for add_field') else: fieldkeys = fields[0::2] fieldvalues = [[x] for x in fields[1::2]] config['fields'] = dict(zip(fieldkeys, fieldvalues)) except TypeError: config['fields'] = {} if 'add_field' in config: del config['add_field'] envFields = config.get('add_field_env', '') if type(envFields) != dict: try: if type(envFields) == str: envFields = envFields.replace(" ","") envFields = filter(None, envFields.split(',')) if len(envFields) == 0: config['envFields'] = {} elif (len(envFields) % 2) == 1: if raise_exceptions: raise Exception('Wrong number of values for add_field_env') else: envFieldkeys = envFields[0::2] envFieldvalues = [] for x in envFields[1::2]: envFieldvalues.append(os.environ.get(x)) config['fields'].update(dict(zip(envFieldkeys, envFieldvalues))) except TypeError: config['envFields'] = {} if 'add_field_env' in config: del config['add_field_env'] try: tags = config.get('tags', '') if type(tags) == str: tags = filter(None, tags.split(',')) if len(tags) == 0: tags = [] config['tags'] = tags except TypeError: config['tags'] = [] if config.get('format') == 'null': config['format'] = 'raw' file_type = config.get('type', None) if not file_type: config['type'] = 'file' require_bool = ['debug', 'ignore_empty', 'ignore_truncate'] for k in require_bool: config[k] = bool(int(config[k])) config['delimiter'] = config['delimiter'].decode('string-escape') if config['multiline_regex_after']: config['multiline_regex_after'] = re.compile(config['multiline_regex_after']) if config['multiline_regex_before']: config['multiline_regex_before'] = re.compile(config['multiline_regex_before']) require_int = ['sincedb_write_interval', 'stat_interval', 'tail_lines'] for k in require_int: config[k] = int(config[k]) return config conf = Configuration( name='beaver', path=self._configfile, main_defaults=self._main_defaults, section_defaults=self._section_defaults, main_parser=_main_parser, section_parser=_section_parser, path_from_main='confd_path', config_parser=self._config_parser ) config = conf.raw() self._beaver_config = config['beaver'] self._file_config = config['sections'] self._main_parser = _main_parser(self._main_defaults) self._section_defaults = _section_parser(self._section_defaults, raise_exceptions=False) self._files = {} for section in config['sections']: globs = eglob(section, config['sections'][section].get('exclude', '')) if not globs: self._logger.debug('Skipping glob due to no files found: %s' % section) continue for globbed_file in globs: self._files[os.path.realpath(globbed_file)] = config['sections'][section]
def _parse(self, args): def _main_parser(config): transpose = ['config', 'debug', 'daemonize', 'files', 'format', 'fqdn', 'hostname', 'path', 'pid', 'transport'] namspace_dict = vars(args) for key in transpose: if key not in namspace_dict or namspace_dict[key] is None or namspace_dict[key] == '': continue config[key] = namspace_dict[key] if args.mode: config['zeromq_bind'] = args.mode # HACK: Python 2.6 ConfigParser does not properly # handle non-string values for key in config: if config[key] == '': config[key] = None require_bool = ['debug', 'daemonize', 'fqdn', 'rabbitmq_exchange_durable', 'rabbitmq_queue_durable', 'rabbitmq_ha_queue'] for key in require_bool: config[key] = bool(int(config[key])) require_int = [ 'max_failure', 'max_queue_size', 'queue_timeout', 'rabbitmq_port', 'respawn_delay', 'subprocess_poll_sleep', 'udp_port', 'wait_timeout', 'zeromq_hwm', ] for key in require_int: if config[key] is not None: config[key] = int(config[key]) require_float = [ 'update_file_mapping_time', 'discover_interval', ] for key in require_float: if config[key] is not None: config[key] = float(config[key]) if config['files'] is not None and type(config['files']) == str: config['files'] = config['files'].split(',') config['path'] = os.path.realpath(config['path']) if not os.path.isdir(config['path']): raise LookupError('{0} does not exist'.format(config['path'])) if config.get('hostname') is None: if config.get('fqdn') is True: config['hostname'] = socket.getfqdn() else: config['hostname'] = socket.gethostname() if config.get('sincedb_path'): config['sincedb_path'] = os.path.realpath(config.get('sincedb_path')) config['globs'] = {} return config def _section_parser(config, raise_exceptions=True): '''Parse a given INI-style config file using ConfigParser module. Stanza's names match file names, and properties are defaulted as in http://logstash.net/docs/1.1.1/inputs/file Config file example: [/var/log/syslog] type: syslog tags: sys,main [/var/log/auth] type: syslog ;tags: auth,main ''' fields = config.get('add_field', '') if type(fields) != dict: try: if type(fields) == str: fields = filter(None, fields.split(',')) if len(fields) == 0: config['fields'] = {} elif (len(fields) % 2) == 1: if raise_exceptions: raise Exception('Wrong number of values for add_field') else: fieldkeys = fields[0::2] fieldvalues = [[x] for x in fields[1::2]] config['fields'] = dict(zip(fieldkeys, fieldvalues)) except TypeError: config['fields'] = {} if 'add_field' in config: del config['add_field'] try: tags = config.get('tags', '') if type(tags) == str: tags = filter(None, tags.split(',')) if len(tags) == 0: tags = [] config['tags'] = tags except TypeError: config['tags'] = [] try: file_type = config.get('type', 'file') if not file_type: file_type = 'file' config['type'] = file_type except: config['type'] = 'file' if config['type']: if raise_exceptions: raise Exception('Missing mandatory config "type"') require_bool = ['debug', 'ignore_empty', 'ignore_truncate'] for k in require_bool: config[k] = bool(int(config[k])) require_int = ['sincedb_write_interval', 'stat_interval', 'tail_lines'] for k in require_int: config[k] = int(config[k]) return config conf = Configuration( name='beaver', path=self._configfile, main_defaults=self._main_defaults, section_defaults=self._section_defaults, main_parser=_main_parser, section_parser=_section_parser, ) config = conf.raw() self._beaver_config = config['beaver'] self._file_config = config['sections'] self._main_parser = _main_parser(self._main_defaults) self._section_defaults = _section_parser(self._section_defaults, raise_exceptions=False) self._files = {} for section in config['sections']: globs = eglob(section, config['sections'][section].get('exclude', '')) if not globs: self._logger.debug('Skipping glob due to no files found: %s' % section) continue for globbed_file in globs: self._files[os.path.realpath(globbed_file)] = config['sections'][section]
def _parse(self, args): def _main_parser(config): transpose = [ 'config', 'confd_path', 'debug', 'daemonize', 'files', 'format', 'fqdn', 'hostname', 'path', 'pid', 'transport' ] namspace_dict = vars(args) for key in transpose: if key not in namspace_dict or namspace_dict[ key] is None or namspace_dict[key] == '': continue config[key] = namspace_dict[key] if args.mode: config['zeromq_bind'] = args.mode # HACK: Python 2.6 ConfigParser does not properly # handle non-string values for key in config: if config[key] == '': config[key] = None require_bool = [ 'debug', 'daemonize', 'fqdn', 'rabbitmq_exchange_durable', 'rabbitmq_queue_durable', 'rabbitmq_ha_queue' ] for key in require_bool: config[key] = bool(int(config[key])) require_int = [ 'max_failure', 'max_queue_size', 'queue_timeout', 'rabbitmq_port', 'respawn_delay', 'subprocess_poll_sleep', 'tcp_port', 'udp_port', 'wait_timeout', 'zeromq_hwm', ] for key in require_int: if config[key] is not None: config[key] = int(config[key]) require_float = [ 'update_file_mapping_time', 'discover_interval', ] for key in require_float: if config[key] is not None: config[key] = float(config[key]) if config.get('format') == 'null': config['format'] = 'raw' if config['files'] is not None and type(config['files']) == str: config['files'] = config['files'].split(',') if config['path'] is not None: config['path'] = os.path.realpath(config['path']) if not os.path.isdir(config['path']): raise LookupError('{0} does not exist'.format( config['path'])) if config.get('hostname') is None: if config.get('fqdn') is True: config['hostname'] = socket.getfqdn() else: config['hostname'] = socket.gethostname() if config.get('sincedb_path'): config['sincedb_path'] = os.path.realpath( config.get('sincedb_path')) config['globs'] = {} return config def _section_parser(config, raise_exceptions=True): '''Parse a given INI-style config file using ConfigParser module. Stanza's names match file names, and properties are defaulted as in http://logstash.net/docs/1.1.1/inputs/file Config file example: [/var/log/syslog] type: syslog tags: sys,main [/var/log/auth] type: syslog ;tags: auth,main ''' fields = config.get('add_field', '') if type(fields) != dict: try: if type(fields) == str: fields = filter(None, fields.split(',')) if len(fields) == 0: config['fields'] = {} elif (len(fields) % 2) == 1: if raise_exceptions: raise Exception( 'Wrong number of values for add_field') else: fieldkeys = fields[0::2] fieldvalues = [[x] for x in fields[1::2]] config['fields'] = dict(zip(fieldkeys, fieldvalues)) except TypeError: config['fields'] = {} if 'add_field' in config: del config['add_field'] try: tags = config.get('tags', '') if type(tags) == str: tags = filter(None, tags.split(',')) if len(tags) == 0: tags = [] config['tags'] = tags except TypeError: config['tags'] = [] if config.get('format') == 'null': config['format'] = 'raw' file_type = config.get('type', None) if not file_type: config['type'] = 'file' require_bool = ['debug', 'ignore_empty', 'ignore_truncate'] for k in require_bool: config[k] = bool(int(config[k])) require_int = [ 'sincedb_write_interval', 'stat_interval', 'tail_lines' ] for k in require_int: config[k] = int(config[k]) return config conf = Configuration(name='beaver', path=self._configfile, main_defaults=self._main_defaults, section_defaults=self._section_defaults, main_parser=_main_parser, section_parser=_section_parser, path_from_main='confd_path') config = conf.raw() self._beaver_config = config['beaver'] self._file_config = config['sections'] self._main_parser = _main_parser(self._main_defaults) self._section_defaults = _section_parser(self._section_defaults, raise_exceptions=False) self._files = {} for section in config['sections']: globs = eglob(section, config['sections'][section].get('exclude', '')) if not globs: self._logger.debug('Skipping glob due to no files found: %s' % section) continue for globbed_file in globs: self._files[os.path.realpath( globbed_file)] = config['sections'][section]
def _parse(self, args): def _main_parser(config): transpose = [ "config", "confd_path", "debug", "daemonize", "files", "format", "fqdn", "hostname", "path", "pid", "transport", ] namspace_dict = vars(args) for key in transpose: if key not in namspace_dict or namspace_dict[key] is None or namspace_dict[key] == "": continue config[key] = namspace_dict[key] if args.mode: config["zeromq_bind"] = args.mode # HACK: Python 2.6 ConfigParser does not properly # handle non-string values for key in config: if config[key] == "": config[key] = None require_bool = [ "debug", "daemonize", "fqdn", "rabbitmq_exchange_durable", "rabbitmq_queue_durable", "rabbitmq_ha_queue", "rabbitmq_ssl", "tcp_ssl_enabled", "tcp_ssl_verify", ] for key in require_bool: config[key] = bool(int(config[key])) require_int = [ "max_failure", "max_queue_size", "queue_timeout", "rabbitmq_port", "rabbitmq_timeout", "rabbitmq_delivery_mode", "respawn_delay", "subprocess_poll_sleep", "refresh_worker_process", "tcp_port", "udp_port", "wait_timeout", "zeromq_hwm", "logstash_version", "kafka_batch_n", "kafka_batch_t", "kafka_ack_timeout", "number_of_consumer_processes", ] for key in require_int: if config[key] is not None: config[key] = int(config[key]) require_float = ["update_file_mapping_time", "discover_interval"] for key in require_float: if config[key] is not None: config[key] = float(config[key]) if config.get("format") == "null": config["format"] = "raw" if config["files"] is not None and type(config["files"]) == str: config["files"] = config["files"].split(",") if config["path"] is not None: config["path"] = os.path.realpath(config["path"]) if not os.path.isdir(config["path"]): raise LookupError("{0} does not exist".format(config["path"])) if config.get("hostname") is None: if config.get("fqdn") is True: config["hostname"] = socket.getfqdn() else: config["hostname"] = socket.gethostname() if config.get("sincedb_path"): config["sincedb_path"] = os.path.realpath(config.get("sincedb_path")) if config["zeromq_address"] and type(config["zeromq_address"]) == str: config["zeromq_address"] = [x.strip() for x in config.get("zeromq_address").split(",")] if config.get("ssh_options") is not None: csv = config.get("ssh_options") config["ssh_options"] = [] if type(csv) == str: for opt in csv.split(","): config["ssh_options"].append("-o %s" % opt.strip()) else: config["ssh_options"] = [] config["globs"] = {} return config def _section_parser(config, raise_exceptions=True): """Parse a given INI-style config file using ConfigParser module. Stanza's names match file names, and properties are defaulted as in http://logstash.net/docs/1.1.1/inputs/file Config file example: [/var/log/syslog] type: syslog tags: sys,main [/var/log/auth] type: syslog ;tags: auth,main """ fields = config.get("add_field", "") if type(fields) != dict: try: if type(fields) == str: fields = filter(None, fields.split(",")) if len(fields) == 0: config["fields"] = {} elif (len(fields) % 2) == 1: if raise_exceptions: raise Exception("Wrong number of values for add_field") else: fieldkeys = fields[0::2] fieldvalues = [[x] for x in fields[1::2]] config["fields"] = dict(zip(fieldkeys, fieldvalues)) except TypeError: config["fields"] = {} if "add_field" in config: del config["add_field"] envFields = config.get("add_field_env", "") if type(envFields) != dict: try: if type(envFields) == str: envFields = envFields.replace(" ", "") envFields = filter(None, envFields.split(",")) if len(envFields) == 0: config["envFields"] = {} elif (len(envFields) % 2) == 1: if raise_exceptions: raise Exception("Wrong number of values for add_field_env") else: envFieldkeys = envFields[0::2] envFieldvalues = [] for x in envFields[1::2]: envFieldvalues.append(os.environ.get(x)) config["fields"].update(dict(zip(envFieldkeys, envFieldvalues))) except TypeError: config["envFields"] = {} if "add_field_env" in config: del config["add_field_env"] try: tags = config.get("tags", "") if type(tags) == str: tags = filter(None, tags.split(",")) if len(tags) == 0: tags = [] config["tags"] = tags except TypeError: config["tags"] = [] if config.get("format") == "null": config["format"] = "raw" file_type = config.get("type", None) if not file_type: config["type"] = "file" require_bool = ["debug", "ignore_empty", "ignore_truncate"] for k in require_bool: config[k] = bool(int(config[k])) config["delimiter"] = config["delimiter"].decode("string-escape") if config["multiline_regex_after"]: config["multiline_regex_after"] = re.compile(config["multiline_regex_after"]) if config["multiline_regex_before"]: config["multiline_regex_before"] = re.compile(config["multiline_regex_before"]) require_int = ["sincedb_write_interval", "stat_interval", "tail_lines"] for k in require_int: config[k] = int(config[k]) return config conf = Configuration( name="beaver", path=self._configfile, main_defaults=self._main_defaults, section_defaults=self._section_defaults, main_parser=_main_parser, section_parser=_section_parser, path_from_main="confd_path", config_parser=self._config_parser, ) config = conf.raw() self._beaver_config = config["beaver"] self._file_config = config["sections"] self._main_parser = _main_parser(self._main_defaults) self._section_defaults = _section_parser(self._section_defaults, raise_exceptions=False) self._files = {} for section in config["sections"]: globs = eglob(section, config["sections"][section].get("exclude", "")) if not globs: self._logger.debug("Skipping glob due to no files found: %s" % section) continue for globbed_file in globs: self._files[os.path.realpath(globbed_file)] = config["sections"][section]