def _parse(self, args): def _main_parser(config): transpose = [ 'config', 'confd_path', 'debug', 'daemonize', 'files', 'format', 'fqdn', 'hostname', 'path', 'pid', 'transport' ] namspace_dict = vars(args) for key in transpose: if key not in namspace_dict or namspace_dict[ key] is None or namspace_dict[key] == '': continue config[key] = namspace_dict[key] if args.mode: config['zeromq_bind'] = args.mode # HACK: Python 2.6 ConfigParser does not properly # handle non-string values for key in config: if config[key] == '': config[key] = None require_bool = [ 'debug', 'daemonize', 'fqdn', 'rabbitmq_exchange_durable', 'rabbitmq_queue_durable', 'rabbitmq_ha_queue', 'rabbitmq_ssl', 'tcp_ssl_enabled', 'tcp_ssl_verify' ] for key in require_bool: config[key] = bool(int(config[key])) require_int = [ 'max_failure', 'max_queue_size', 'queue_timeout', 'rabbitmq_port', 'rabbitmq_timeout', 'rabbitmq_delivery_mode', 'respawn_delay', 'subprocess_poll_sleep', 'refresh_worker_process', 'tcp_port', 'udp_port', 'wait_timeout', 'zeromq_hwm', 'logstash_version', 'kafka_batch_n', 'kafka_batch_t', 'kafka_ack_timeout', 'number_of_consumer_processes', 'ignore_old_files' ] for key in require_int: if config[key] is not None: config[key] = int(config[key]) require_float = [ 'update_file_mapping_time', 'discover_interval', ] for key in require_float: if config[key] is not None: config[key] = float(config[key]) if config.get('format') == 'null': config['format'] = 'raw' if config['files'] is not None and type(config['files']) == str: config['files'] = config['files'].split(',') if config['path'] is not None: config['path'] = os.path.realpath(config['path']) if not os.path.isdir(config['path']): raise LookupError('{0} does not exist'.format( config['path'])) if config.get('hostname') is None: if config.get('fqdn') is True: config['hostname'] = socket.getfqdn() else: config['hostname'] = socket.gethostname() if config.get('sincedb_path'): config['sincedb_path'] = os.path.realpath( config.get('sincedb_path')) if config['zeromq_address'] and type( config['zeromq_address']) == str: config['zeromq_address'] = [ x.strip() for x in config.get('zeromq_address').split(',') ] if config.get('ssh_options') is not None: csv = config.get('ssh_options') config['ssh_options'] = [] if type(csv) == str: for opt in csv.split(','): config['ssh_options'].append('-o %s' % opt.strip()) else: config['ssh_options'] = [] config['globs'] = {} return config def _section_parser(config, raise_exceptions=True): '''Parse a given INI-style config file using ConfigParser module. Stanza's names match file names, and properties are defaulted as in http://logstash.net/docs/1.1.1/inputs/file Config file example: [/var/log/syslog] type: syslog tags: sys,main [/var/log/auth] type: syslog ;tags: auth,main ''' fields = config.get('add_field', '') if type(fields) != dict: try: if type(fields) == str: fields = filter(None, fields.split(',')) if len(fields) == 0: config['fields'] = {} elif (len(fields) % 2) == 1: if raise_exceptions: raise Exception( 'Wrong number of values for add_field') else: fieldkeys = fields[0::2] fieldvalues = [[x] for x in fields[1::2]] config['fields'] = dict(zip(fieldkeys, fieldvalues)) except TypeError: config['fields'] = {} if 'add_field' in config: del config['add_field'] envFields = config.get('add_field_env', '') if type(envFields) != dict: try: if type(envFields) == str: envFields = envFields.replace(" ", "") envFields = filter(None, envFields.split(',')) if len(envFields) == 0: config['envFields'] = {} elif (len(envFields) % 2) == 1: if raise_exceptions: raise Exception( 'Wrong number of values for add_field_env') else: envFieldkeys = envFields[0::2] envFieldvalues = [] for x in envFields[1::2]: envFieldvalues.append(os.environ.get(x)) config['fields'].update( dict(zip(envFieldkeys, envFieldvalues))) except TypeError: config['envFields'] = {} if 'add_field_env' in config: del config['add_field_env'] try: tags = config.get('tags', '') if type(tags) == str: tags = filter(None, tags.split(',')) if len(tags) == 0: tags = [] config['tags'] = tags except TypeError: config['tags'] = [] if config.get('format') == 'null': config['format'] = 'raw' file_type = config.get('type', None) if not file_type: config['type'] = 'file' require_bool = ['debug', 'ignore_empty', 'ignore_truncate'] for k in require_bool: config[k] = bool(int(config[k])) config['delimiter'] = config['delimiter'].decode('string-escape') if config['multiline_regex_after']: config['multiline_regex_after'] = re.compile( config['multiline_regex_after']) if config['multiline_regex_before']: config['multiline_regex_before'] = re.compile( config['multiline_regex_before']) require_int = [ 'sincedb_write_interval', 'stat_interval', 'tail_lines' ] for k in require_int: config[k] = int(config[k]) return config conf = Configuration(name='beaver', path=self._configfile, main_defaults=self._main_defaults, section_defaults=self._section_defaults, main_parser=_main_parser, section_parser=_section_parser, path_from_main='confd_path', config_parser=self._config_parser) config = conf.raw() self._beaver_config = config['beaver'] self._file_config = config['sections'] self._main_parser = _main_parser(self._main_defaults) self._section_defaults = _section_parser(self._section_defaults, raise_exceptions=False) self._files = {} for section in config['sections']: globs = eglob(section, config['sections'][section].get('exclude', '')) if not globs: self._logger.debug('Skipping glob due to no files found: %s' % section) continue for globbed_file in globs: self._files[os.path.realpath( globbed_file)] = config['sections'][section]
def test_readme(self): def digitize(config): for key in config: if not config[key].isdigit(): try: config[key] = float(config[key]) except ValueError: pass else: try: config[key] = int(config[key]) except ValueError: pass return config conf = Configuration( name='derp', path='./data/conf', main_defaults={ 'no': 'one', 'expected': 'the spanish inquisition', 'cats': '1', }, section_parser=digitize ) expected = { 'sections': {'herp': {'sleep': 1, 'wait': 5.0, 'timeout': 'seventy'}}, 'derp': {'expected': 'the spanish inquisition', 'til': 'brooklyn', 'cats': '1', 'no': 'sleep'} } self.assertEqual(expected, conf.raw()) actual = conf.get(section='derp', key='no', default="jumping") self.assertEqual('sleep', actual) actual = conf.get(section='derp', key='til') self.assertEqual('brooklyn', actual) actual = conf.get(section='derp', key='cats') self.assertEqual('1', actual) actual = conf.get(section='derp', key='dogs') self.assertEqual(None, actual) actual = conf.get(section='herp', key='sleep') self.assertEqual(1, actual) actual = conf.get(section='herp', key='wait') self.assertEqual(5.0, actual) actual = conf.get(section='herp', key='timeout') self.assertEqual('seventy', actual) actual = conf.has(section='derp') self.assertEqual(True, actual) actual = conf.has(section='derp', key='no') self.assertEqual(True, actual) expected = { 'sections': { 'herp': { 'sleep': 1, 'wait': 5.0, 'timeout': 'seventy' } }, 'derp': { 'expected': 'the spanish inquisition', 'til': 'brooklyn', 'cats': '1', 'no': 'sleep' } } self.assertEqual(expected, conf.raw())
def test_confd(self): conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', confd_path='./data/conf.d', main_defaults={'main_key': 'main_value'} ) self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections')) self.assertEqual({'til': 'brooklyn', 'no': 'sleep'}, conf.get('derp')) self.assertEqual('main_value', conf.get('multiple_sections', 'main_key')) self.assertEqual('brooklyn', conf.get('derp', 'til')) self.assertEqual(None, conf.get('derp', 'missing_key')) self.assertEqual(None, conf.get('derp', 'main_key')) self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1')) expected = { 'multiple_sections': {'main_key': 'main_value'}, 'sections': { 'another/conf': {'sleep': '1', 'wait': '15'}, 'derp': {'no': 'sleep', 'til': 'brooklyn'}, 'section': {'key': 'value'}, 'test': {'conf': 'path/to/another/conf', 'sleep': '15'} } } self.assertEqual(expected, conf.raw()) conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', confd_path='./data/conf.d', section_defaults={'sleep': '2', 'wait': '30'} ) self.assertEqual({}, conf.get('multiple_sections')) self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp')) self.assertEqual(None, conf.get('multiple_sections', 'main_key')) self.assertEqual('brooklyn', conf.get('derp', 'til')) self.assertEqual(None, conf.get('derp', 'missing_key')) self.assertEqual(None, conf.get('derp', 'main_key')) self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1')) expected = { 'multiple_sections': {}, 'sections': { 'another/conf': {'sleep': '1', 'wait': '15'}, 'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, 'section': {'key': 'value', 'sleep': '2', 'wait': '30'}, 'test': {'conf': 'path/to/another/conf', 'sleep': '15', 'wait': '30'} } } self.assertEqual(expected, conf.raw()) conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', confd_path='./data/conf.d', main_defaults={'main_key': 'main_value'}, section_defaults={'sleep': '2', 'wait': '30'} ) self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections')) self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp')) self.assertEqual('main_value', conf.get('multiple_sections', 'main_key')) self.assertEqual('brooklyn', conf.get('derp', 'til')) self.assertEqual(None, conf.get('derp', 'missing_key')) self.assertEqual(None, conf.get('derp', 'main_key')) self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1')) expected = { 'multiple_sections': {'main_key': 'main_value'}, 'sections': { 'another/conf': {'sleep': '1', 'wait': '15'}, 'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, 'section': {'key': 'value', 'sleep': '2', 'wait': '30'}, 'test': {'conf': 'path/to/another/conf', 'sleep': '15', 'wait': '30'} } } self.assertEqual(expected, conf.raw()) conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', confd_path='./data/conf.d', conf_ext='conf', main_defaults={'main_key': 'main_value'}, section_defaults={'sleep': '2', 'wait': '30'} ) self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections')) self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp')) self.assertEqual('main_value', conf.get('multiple_sections', 'main_key')) self.assertEqual('brooklyn', conf.get('derp', 'til')) self.assertEqual(None, conf.get('derp', 'missing_key')) self.assertEqual(None, conf.get('derp', 'main_key')) self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1')) expected = { 'multiple_sections': {'main_key': 'main_value'}, 'sections': { 'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, 'section': {'key': 'value', 'sleep': '2', 'wait': '30'}, } } self.assertEqual(expected, conf.raw()) conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', confd_path='./data/conf.d', conf_ext='.conf', main_defaults={'main_key': 'main_value'}, section_defaults={'sleep': '2', 'wait': '30'} ) self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections')) self.assertEqual({'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, conf.get('derp')) self.assertEqual('main_value', conf.get('multiple_sections', 'main_key')) self.assertEqual('brooklyn', conf.get('derp', 'til')) self.assertEqual(None, conf.get('derp', 'missing_key')) self.assertEqual(None, conf.get('derp', 'main_key')) self.assertEqual('1', conf.get('derp', 'missing_key_with_default', '1')) expected = { 'multiple_sections': {'main_key': 'main_value'}, 'sections': { 'derp': {'no': 'sleep', 'sleep': '2', 'til': 'brooklyn', 'wait': '30'}, 'section': {'key': 'value', 'sleep': '2', 'wait': '30'}, } } self.assertEqual(expected, conf.raw())
def test_parser(self): def safe_cast(val): try: return int(val) except ValueError: return val def all_as_bool(config): for key in config: config[key] = bool(config[key]) return config def all_as_int(config): for key in config: try: config[key] = int(config[key]) except ValueError: pass return config conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', confd_path='./data/conf.d', main_defaults={'main_key': 'main_value'}, section_defaults={'sleep': '2', 'wait': '30'}, main_parser=all_as_bool ) self.assertEqual({'main_key': True}, conf.get('multiple_sections')) self.assertEqual(True, conf.get('multiple_sections', 'main_key')) self.assertEqual('1', conf.get('another/conf', 'sleep')) conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', confd_path='./data/conf.d', main_defaults={'main_key': 'main_value'}, section_defaults={'sleep': '2', 'wait': '30'}, section_parser=all_as_bool ) self.assertEqual({'main_key': 'main_value'}, conf.get('multiple_sections')) self.assertEqual('main_value', conf.get('multiple_sections', 'main_key')) self.assertEqual(1, conf.get('another/conf', 'sleep')) conf = Configuration( name='multiple_sections', path='./data/multiple_sections.ini', confd_path='./data/conf.d', main_defaults={'main_key': 'main_value'}, section_defaults={'sleep': '2', 'wait': '30'}, main_parser=all_as_bool, section_parser=all_as_int ) self.assertEqual({'main_key': True}, conf.get('multiple_sections')) self.assertEqual(True, conf.get('multiple_sections', 'main_key')) self.assertEqual(None, conf.get('derp', 'missing_key')) self.assertEqual(1, conf.get('another/conf', 'sleep'))
def _parse(self, args): def _main_parser(config): transpose = [ 'config', 'debug', 'daemonize', 'files', 'format', 'fqdn', 'hostname', 'path', 'pid', 'transport' ] namspace_dict = vars(args) for key in transpose: if key not in namspace_dict or namspace_dict[ key] is None or namspace_dict[key] == '': continue config[key] = namspace_dict[key] if args.mode: config['zeromq_bind'] = args.mode # HACK: Python 2.6 ConfigParser does not properly # handle non-string values for key in config: if config[key] == '': config[key] = None require_bool = [ 'debug', 'daemonize', 'fqdn', 'rabbitmq_exchange_durable', 'rabbitmq_queue_durable', 'rabbitmq_ha_queue' ] for key in require_bool: config[key] = bool(int(config[key])) require_int = [ 'max_failure', 'max_queue_size', 'queue_timeout', 'rabbitmq_port', 'respawn_delay', 'subprocess_poll_sleep', 'udp_port', 'wait_timeout', 'zeromq_hwm', ] for key in require_int: if config[key] is not None: config[key] = int(config[key]) require_float = [ 'update_file_mapping_time', 'discover_interval', ] for key in require_float: if config[key] is not None: config[key] = float(config[key]) if config['files'] is not None and type(config['files']) == str: config['files'] = config['files'].split(',') config['path'] = os.path.realpath(config['path']) if not os.path.isdir(config['path']): raise LookupError('{0} does not exist'.format(config['path'])) if config.get('hostname') is None: if config.get('fqdn') is True: config['hostname'] = socket.getfqdn() else: config['hostname'] = socket.gethostname() if config.get('sincedb_path'): config['sincedb_path'] = os.path.realpath( config.get('sincedb_path')) config['globs'] = {} return config def _section_parser(config, raise_exceptions=True): '''Parse a given INI-style config file using ConfigParser module. Stanza's names match file names, and properties are defaulted as in http://logstash.net/docs/1.1.1/inputs/file Config file example: [/var/log/syslog] type: syslog tags: sys,main [/var/log/auth] type: syslog ;tags: auth,main ''' fields = config.get('add_field', '') if type(fields) != dict: try: if type(fields) == str: fields = filter(None, fields.split(',')) if len(fields) == 0: config['fields'] = {} elif (len(fields) % 2) == 1: if raise_exceptions: raise Exception( 'Wrong number of values for add_field') else: fieldkeys = fields[0::2] fieldvalues = [[x] for x in fields[1::2]] config['fields'] = dict(zip(fieldkeys, fieldvalues)) except TypeError: config['fields'] = {} if 'add_field' in config: del config['add_field'] try: tags = config.get('tags', '') if type(tags) == str: tags = filter(None, tags.split(',')) if len(tags) == 0: tags = [] config['tags'] = tags except TypeError: config['tags'] = [] try: file_type = config.get('type', 'file') if not file_type: file_type = 'file' config['type'] = file_type except: config['type'] = 'file' if config['type']: if raise_exceptions: raise Exception('Missing mandatory config "type"') require_bool = ['debug', 'ignore_empty', 'ignore_truncate'] for k in require_bool: config[k] = bool(int(config[k])) require_int = [ 'sincedb_write_interval', 'stat_interval', 'tail_lines' ] for k in require_int: config[k] = int(config[k]) return config conf = Configuration( name='beaver', path=self._configfile, main_defaults=self._main_defaults, section_defaults=self._section_defaults, main_parser=_main_parser, section_parser=_section_parser, ) config = conf.raw() self._beaver_config = config['beaver'] self._file_config = config['sections'] self._main_parser = _main_parser(self._main_defaults) self._section_defaults = _section_parser(self._section_defaults, raise_exceptions=False) self._files = {} for section in config['sections']: globs = eglob(section, config['sections'][section].get('exclude', '')) if not globs: self._logger.debug('Skipping glob due to no files found: %s' % section) continue for globbed_file in globs: self._files[os.path.realpath( globbed_file)] = config['sections'][section]