def __init__(self, **kwargs): config = Config(required={ "comparator": ("dbpath", "series_timeout", "cleanup_time") }) self.comparator_config = config["comparator"] self.comparator_config["dbpath"] = os.path.expanduser( self.comparator_config["dbpath"]) dbpath_dirname = os.path.dirname(self.comparator_config["dbpath"]) try: os.makedirs(dbpath_dirname, 0o700) except OSError: pass super(Comparator, self).__init__(**kwargs) # store dir doesn't exist, stop comparator if not os.path.isdir(dbpath_dirname): raise Exception( 'store dir does not exist, stop comparator, path:', self.comparator_config["dbpath"]) # store directory exists, but it has no rights to write if not os.access(dbpath_dirname, os.W_OK): raise Exception( 'stop comparator, remember to set the rights' ' for user, which runs comparator, path:', self.comparator_config["dbpath"]) self.state = ComparatorState( int(self.comparator_config["cleanup_time"])) self.db = ComparatorDataWrapper(self.comparator_config["dbpath"])
def __init__(self, **kwargs): config = Config(required={"aggregator": ("dbpath", "time_tolerance")}) self.aggregator_config = config["aggregator"] self.aggregator_config["dbpath"] = os.path.expanduser( self.aggregator_config["dbpath"]) dbpath_dirname = os.path.dirname(self.aggregator_config["dbpath"]) try: os.makedirs(dbpath_dirname, 0700) except OSError: pass super(Aggregator, self).__init__(**kwargs) # store dir doesn't exist, stop aggregator if not os.path.isdir(dbpath_dirname): raise Exception( 'store dir does not exist, stop aggregator, path:', self.aggregator_config["dbpath"]) # store directory exists, but it has no rights to write if not os.access(dbpath_dirname, os.W_OK): raise Exception( 'stop aggregator, remember to set the rights' ' for user, which runs aggregator, path:', self.aggregator_config["dbpath"]) self.db = AggregatorDataWrapper( self.aggregator_config["dbpath"], int(self.aggregator_config["time_tolerance"])) self.timeout_id = None # id of the 'tick' timeout that executes source cleanup
def __make_converter__list_of_template_loc(): def convert_template_loc(loc): if loc.startswith('@'): try: loc_parts = loc[1:].split(':', 1) try: package_name, package_subdir_path = loc_parts except ValueError: raise ValueError('the colon is missing') if isabs(package_subdir_path): raise ValueError( 'the subdirectory part is not a relative path') except ValueError as exc: raise ValueError( '{!a} is not a valid package-based template ' 'location ({})'.format(loc, ascii_str(exc))) converted_loc = package_name, package_subdir_path else: converted_loc = expanduser(loc) if not isabs(converted_loc): raise ValueError( '{!a} is not a valid absolute-path-based template ' 'location (is not an absolute path)'.format(loc)) return converted_loc return Config.make_list_converter(convert_template_loc, name='list_of_template_loc')
def __init__(self, config=None): """ Args: config: dict containing: mongohost, mongoport, mongodb, count_try_connection, time_sleep_between_try_connect, uri """ if config is None: config = Config(required={"archiveraw": ("mongohost", "mongoport", "mongodb", "count_try_connection", "time_sleep_between_try_connect", "uri")}) self.config = config["archiveraw"] else: self.config = config self.host = self.config['mongohost'] self.port = int(self.config['mongoport']) self._currdb = self.config['mongodb'] self.uri = self.config["uri"] self.connection = None self._currcoll = None self.conn_gridfs = None self.time_sleep_between_try_connect = int(self.config['time_sleep_between_try_connect']) self.count_try_connection = int(self.config['count_try_connection']) self.indexes_store = [] self.backup_msg = '.backup_msg' self.backup_msg_data = None self.backup_msg_headers = None
def __init__(self, **kwargs): LOGGER.info("Recorder Start") config = Config(required={"recorder": ("uri", )}) self.config = config["recorder"] self.record_dict = None self.records = None self.routing_key = None self.session_db = self._setup_db() self.dict_map_fun = { "event.filtered": (RecordDict.from_json, self.new_event), "bl-new.filtered": (BLRecordDict.from_json, self.blacklist_new), "bl-change.filtered": (BLRecordDict.from_json, self.blacklist_change), "bl-delist.filtered": (BLRecordDict.from_json, self.blacklist_delist), "bl-expire.filtered": (BLRecordDict.from_json, self.blacklist_expire), "bl-update.filtered": (BLRecordDict.from_json, self.blacklist_update), "suppressed.filtered": (RecordDict.from_json, self.suppressed_update), } # keys in each of the tuples being values of `dict_map_fun` self.FROM_JSON = 0 self.HANDLE_EVENT = 1 super(Recorder, self).__init__(**kwargs)
def from_predefined(cls, default_template_name=None, autoescape=None, settings=None, **kwargs): config = Config.section( cls.config_spec, settings=settings, custom_converters={ 'list_of_template_loc': cls.__make_converter__list_of_template_loc(), }) env = Environment(loader=PackageOrFileSystemLoader( locations=config['template_locations'], encoding=config['template_encoding'], fs_followlinks=config['follow_symlinks']), autoescape=(autoescape if autoescape is not None else DEFAULT_AUTOESCAPE)) jinja_extensions = sorted( set(config['jinja_extensions'] + list(kwargs.pop('jinja_extensions', [])))) return cls(jinja_env=env, jinja_extensions=jinja_extensions, default_template_name=default_template_name, **kwargs)
def __init__(self, **kwargs): LOGGER.info("Recorder Start") config = Config(required={"recorder": ("uri", "echo")}) self.config = config["recorder"] self.rows = None self.record_dict = None self.source = None self.dir_name = None self.wait_timeout = int(self.config.get("wait_timeout", 28800)) engine = create_engine(self.config["uri"], echo=bool((int(self.config["echo"])))) self.session_db = N6DataBackendAPI.configure_db_session(engine) self.set_session_wait_timeout() self.records = None self.routing_key = None self.dict_map_fun = { "event.filtered": (RecordDict.from_json, self.new_event), "bl-new.filtered": (BLRecordDict.from_json, self.blacklist_new), "bl-change.filtered": (BLRecordDict.from_json, self.blacklist_change), "bl-delist.filtered": (BLRecordDict.from_json, self.blacklist_delist), "bl-expire.filtered": (BLRecordDict.from_json, self.blacklist_expire), "bl-update.filtered": (BLRecordDict.from_json, self.blacklist_update), "suppressed.filtered": (RecordDict.from_json, self.suppressed_update), } # keys in each of the tuples being values of `dict_map_fun` self.FROM_JSON = 0 self.HANDLE_EVENT = 1 super(Recorder, self).__init__(**kwargs)
def get_amqp_connection_params_dict(rabbitmq_config_section='rabbitmq'): """ Get the AMQP connection parameters (as a dict) from config. Returns: A dict that can be used as **kwargs for pika.ConnectionParameters. """ # Config is imported here to avoid circular dependency from n6lib.config import Config config_spec = RABBITMQ_CONFIG_SPEC_PATTERN.format( rabbitmq_config_section=rabbitmq_config_section) queue_conf = Config.section(config_spec) params_dict = dict( host=queue_conf["host"], port=queue_conf["port"], ssl=queue_conf["ssl"], ssl_options={}, heartbeat_interval=queue_conf['heartbeat_interval'], ) if params_dict['ssl']: params_dict['credentials'] = pika.credentials.ExternalCredentials() params_dict['ssl_options'].update( ca_certs=os.path.expanduser(queue_conf["ssl_ca_certs"]), certfile=os.path.expanduser(queue_conf["ssl_certfile"]), keyfile=os.path.expanduser(queue_conf["ssl_keyfile"]), cert_reqs=ssl.CERT_REQUIRED, ) return params_dict
def get_connection_params_dict(cls): params_dict = super(BaseParser, cls).get_connection_params_dict() config = Config(required={ cls.rabbitmq_config_section: ("heartbeat_interval_parsers", ) }) queue_conf = config[cls.rabbitmq_config_section] params_dict['heartbeat_interval'] = int( queue_conf['heartbeat_interval_parsers']) return params_dict
def __init__(self, **fmt_args): self.config = Config.section(self.general_config_spec) if self.is_config_spec_or_group_declared(): self.config.update(self.get_config_section(**fmt_args)) self.parameters = self.config.copy() config_ttl = self.parameters.get(self.config_ttl_opt_name) if config_ttl: self.parameters[self.parameter_ttl_opt_name] = config_ttl del self.parameters[self.config_ttl_opt_name] if self.unused_options: self.parameters.update({key: None for key in self.unused_options}) if self.default_none_options: self.parameters.update({key: None for key in self.default_none_options if key not in self.parameters})
def __init__(self, **kwargs): self.gi_asn = None self.gi_cc = None self._resolver = None config = Config( required={ "enrich": ("dnshost", "dnsport", "geoippath", "asndatabasefilename", "citydatabasefilename") }) self._enrich_config = config["enrich"] self.excluded_ips = self._get_excluded_ips() self._setup_geodb() self._setup_dnsresolver(self._enrich_config["dnshost"], int(self._enrich_config["dnsport"])) super(Enricher, self).__init__(**kwargs)
def get_amqp_connection_params_dict(rabbitmq_config_section='rabbitmq'): """ Get the AMQP connection parameters (as a dict) from config. Returns: A dict that can be used as **kwargs for pika.ConnectionParameters. """ # Config is imported here to avoid circular dependency from n6lib.config import Config config_spec = RABBITMQ_CONFIG_SPEC_PATTERN.format( rabbitmq_config_section=rabbitmq_config_section) queue_conf = Config.section(config_spec) return get_amqp_connection_params_dict_from_args( host=queue_conf["host"], port=queue_conf["port"], heartbeat_interval=queue_conf["heartbeat_interval"], ssl=queue_conf["ssl"], ca_certs=queue_conf.get("ssl_ca_certs", None), certfile=queue_conf.get("ssl_certfile", None), keyfile=queue_conf.get("ssl_keyfile", None))
def get_pipeline_binding_states(pipeline_group, pipeline_name, pipeline_config_section='pipeline'): """ Get the list of "binding states" for the component, or its group, from the pipeline config. Pipeline config for an individual component has a priority over group's config. Args: `pipeline_group`: A group which the component is bound to. `pipeline_name`: Name of the component in the pipeline config format, which is, by default, a lowercase component's class' name. `pipeline_config_section`: Name of the pipeline config section, "pipeline" by default. Returns: The list of "binding states" for the component, or None, if no config option could be found. """ from n6lib.config import Config config_spec = PIPELINE_CONFIG_SPEC_PATTERN.format( pipeline_config_section=pipeline_config_section) pipeline_conf = Config.section(config_spec, default_converter='list_of_str') try: return pipeline_conf[pipeline_name] except KeyError: pass try: return pipeline_conf[pipeline_group] except KeyError: return None
class TestBaseParser(unittest.TestCase): def setUp(self): self.mock = Mock(__class__=BaseParser, allow_empty_results=False) self.meth = MethodProxy(BaseParser, self.mock) def _asserts_of_proper__new__instance_adjustment(self, instance): # BaseQueued.__new__() ensures that self.assertIsNot(instance.input_queue, BaseParser.input_queue) def _asserts_of_proper_preinit_hook_instance_adjustment( self, instance, binding_key): # for classes with `default_binding_key` # BaseParser.preinit_hook() ensures that self.assertEqual( instance.input_queue, { 'exchange': 'raw', 'exchange_type': 'topic', 'queue_name': binding_key, 'binding_keys': [binding_key], }) self.assertEqual(BaseParser.input_queue, { 'exchange': 'raw', 'exchange_type': 'topic', }) def _basic_init_related_asserts(self, instance, subclass, super_mock, super_cls_mock, expected_config, expected_config_full): # assert that an instance of the proper type has been returned self.assertIsInstance(instance, subclass) # assert that super used properly super_mock.assert_called_once_with(BaseParser, instance) super_cls_mock.__init__.assert_called_once_with(a=sentinel.a, bb=sentinel.bb) # assert that configuration stuff has been obtained properly self.assertEqual(instance.config, expected_config) self.assertIsInstance(instance.config, ConfigSection) self.assertEqual(instance.config_full, expected_config_full) self.assertIsInstance(instance.config_full, Config) def test_basics(self): self.assertTrue(issubclass(BaseParser, QueuedBase)) self.assertTrue(hasattr(BaseParser, 'default_binding_key')) self.assertTrue(hasattr(BaseParser, 'config_spec_pattern')) self.assertTrue(hasattr(BaseParser, 'constant_items')) self.assertTrue(hasattr(BaseParser, 'record_dict_class')) self.assertTrue(hasattr(BaseParser, 'event_type')) def test_config_spec_pattern(self): config_spec = BaseParser.config_spec_pattern.format( parser_class_name='example_foo') config_spec_parsed = parse_config_spec(config_spec) prefetch_count_opt_spec = config_spec_parsed.get_opt_spec( 'example_foo.prefetch_count') self.assertEqual(prefetch_count_opt_spec.name, 'prefetch_count') self.assertEqual(prefetch_count_opt_spec.converter_spec, 'int') def test_initialization_without_default_binding_key(self): class SomeParser(BaseParser): pass # no `default_binding_key` defined => it's an abstract class with self.assertRaises(NotImplementedError): SomeParser() unready_instance = SomeParser.__new__(SomeParser) self._asserts_of_proper__new__instance_adjustment(unready_instance) # for classes without `default_binding_key` # `queue_name` and `binding_keys` items are *not* added... self.assertEqual(unready_instance.input_queue, BaseParser.input_queue) self.assertEqual(BaseParser.input_queue, { 'exchange': 'raw', 'exchange_type': 'topic', }) @foreach( param( mocked_conf_from_files={}, expected_config=ConfigSection('SomeParser', {'prefetch_count': 1}), expected_config_full=Config.make( {'SomeParser': { 'prefetch_count': 1 }}), ), param( mocked_conf_from_files={ 'SomeParser': { 'prefetch_count': '42' }, 'another_section': { 'another_opt': '123.456' }, }, expected_config=ConfigSection('SomeParser', {'prefetch_count': 42}), expected_config_full=Config.make( {'SomeParser': { 'prefetch_count': 42 }}), ), param( custom_config_spec_pattern=concat_reducing_indent( BaseParser.config_spec_pattern, ''' some_opt = [-3, null] :: json [another_section] another_opt :: float yet_another_opt = Foo Bar Spam Ham ''', ), mocked_conf_from_files={ 'SomeParser': { 'prefetch_count': '42' }, 'another_section': { 'another_opt': '123.456' }, }, expected_config=ConfigSection('SomeParser', { 'prefetch_count': 42, 'some_opt': [-3, None], }), expected_config_full=Config.make({ 'SomeParser': { 'prefetch_count': 42, 'some_opt': [-3, None], }, 'another_section': { 'another_opt': 123.456, 'yet_another_opt': 'Foo Bar Spam Ham', }, }), ), ) @foreach( param(binding_key='foo.bar'), param(binding_key='foo.bar.33'), ) def test_initialization_with_default_binding_key( self, binding_key, mocked_conf_from_files, expected_config, expected_config_full, custom_config_spec_pattern=None): class SomeParser(BaseParser): default_binding_key = binding_key # => it's a concrete class if custom_config_spec_pattern is not None: SomeParser.config_spec_pattern = custom_config_spec_pattern unready_instance = SomeParser.__new__(SomeParser) self._asserts_of_proper__new__instance_adjustment(unready_instance) self._asserts_of_proper_preinit_hook_instance_adjustment( unready_instance, binding_key) super_cls_mock = SimpleNamespace(__init__=Mock()) with patch_always('n6.parsers.generic.super', return_value=super_cls_mock) as super_mock, \ patch('n6.parsers.generic.Config._load_n6_config_files', return_value=mocked_conf_from_files): # instantiation instance = SomeParser(a=sentinel.a, bb=sentinel.bb) self._asserts_of_proper__new__instance_adjustment(instance) self._asserts_of_proper_preinit_hook_instance_adjustment( instance, binding_key) self._basic_init_related_asserts(instance, SomeParser, super_mock, super_cls_mock, expected_config, expected_config_full) def test__make_binding_keys(self): self.mock.default_binding_key = 'fooo.barr' binding_keys = self.meth.make_binding_keys() self.assertEqual(binding_keys, ['fooo.barr']) self.assertEqual(self.mock.mock_calls, []) def test__make_binding_keys_with_raw_format_version_tag(self): self.mock.default_binding_key = 'fooo.barr.33' binding_keys = self.meth.make_binding_keys() self.assertEqual(binding_keys, ['fooo.barr.33']) self.assertEqual(self.mock.mock_calls, []) def test__get_script_init_kwargs(self): self.assertIsInstance( vars(BaseParser)['get_script_init_kwargs'], classmethod) init_kwargs = BaseParser.get_script_init_kwargs.__func__(self.mock) self.assertEqual(init_kwargs, {}) self.assertEqual(self.mock.mock_calls, []) def test__run_handling__interrupted(self): self.mock.configure_mock(**{'run.side_effect': KeyboardInterrupt}) self.meth.run_handling() self.mock.run.assert_called_once_with() self.mock.stop.assert_called_once_with() def test__run_handling__not_interrupted(self): self.meth.run_handling() self.mock.run.assert_called_once_with() self.assertEqual(self.mock.stop.mock_calls, []) @patch('n6.parsers.generic.FilePagedSequence') def test__input_callback(self, FilePagedSequence_mock): FilePagedSequence_mock.return_value = MagicMock() FilePagedSequence_mock.return_value.__enter__.return_value = sentinel.working_seq data = MagicMock(**{'get.return_value': sentinel.rid}) self.mock.configure_mock( **{ '_fix_body.return_value': sentinel.body, 'prepare_data.return_value': data, 'setting_error_event_info': MagicMock(), 'get_output_rk.return_value': sentinel.output_rk, 'get_output_bodies.return_value': [sentinel.output_body1, sentinel.output_body2], }) self.meth.input_callback(sentinel.routing_key, sentinel.body, sentinel.properties) self.assertEqual(self.mock.mock_calls, [ call._fix_body(sentinel.body), call.prepare_data(sentinel.routing_key, sentinel.body, sentinel.properties), call.prepare_data().get('properties.message_id'), call.setting_error_event_info(sentinel.rid), call.setting_error_event_info().__enter__(), call.get_output_rk(data), call.get_output_bodies(data, sentinel.working_seq), call.publish_output(routing_key=sentinel.output_rk, body=sentinel.output_body1), call.publish_output(routing_key=sentinel.output_rk, body=sentinel.output_body2), call.setting_error_event_info().__exit__(None, None, None), ]) self.assertEqual(FilePagedSequence_mock.mock_calls, [ call(page_size=1000), call().__enter__(), call().__exit__(None, None, None), ]) def test__prepare_data(self): data = self.meth.prepare_data(routing_key='ham.spam', body=sentinel.body, properties=SimpleNamespace( foo=sentinel.foo, bar=sentinel.bar, timestamp=1389348840, headers={'a': sentinel.a})) self.assertEqual( data, { 'a': sentinel.a, 'properties.foo': sentinel.foo, 'properties.bar': sentinel.bar, 'source': 'ham.spam', 'properties.timestamp': '2014-01-10 10:14:00', 'raw_format_version_tag': None, 'raw': sentinel.body, }) def test__prepare_data__rk__with_raw_format_version_tag(self): data = self.meth.prepare_data(routing_key='ham.spam.33', body=sentinel.body, properties=SimpleNamespace( foo=sentinel.foo, bar=sentinel.bar, timestamp=1389348840, headers={'a': sentinel.a})) self.assertEqual( data, { 'a': sentinel.a, 'properties.foo': sentinel.foo, 'properties.bar': sentinel.bar, 'source': 'ham.spam', 'properties.timestamp': '2014-01-10 10:14:00', 'raw_format_version_tag': '33', 'raw': sentinel.body, }) def test__get_output_rk(self): self.mock.configure_mock(**{ 'event_type': 'foobar', }) data = {'source': 'ham.spam'} output_rk = self.meth.get_output_rk(data) self.assertEqual(output_rk, 'foobar.parsed.ham.spam') def test__get_output_bodies(self): parsed = [ MagicMock( **{ '__class__': RecordDict, 'used_as_context_manager': True, 'get_ready_json.return_value': getattr(sentinel, 'output_body{}'.format(i)) }) for i in (1, 2) ] self.mock.configure_mock( **{ 'parse.return_value': parsed, 'get_output_message_id.side_effect': [ sentinel.msg_A, sentinel.msg_B, ], 'setting_error_event_info': MagicMock(), 'postprocess_parsed.side_effect': ( lambda data, parsed, total, item_no: parsed), }) seq_mock = FilePagedSequence._instance_mock() output_bodies = self.meth.get_output_bodies(sentinel.data, seq_mock) self.assertIs(output_bodies, seq_mock) self.assertEqual(seq_mock._list, [ sentinel.output_body1, sentinel.output_body2, ]) self.assertEqual(parsed[0].mock_calls, [ call.__setitem__('id', sentinel.msg_A), call.get_ready_json(), ]) self.assertEqual(parsed[1].mock_calls, [ call.__setitem__('id', sentinel.msg_B), call.get_ready_json(), ]) self.assertEqual(self.mock.mock_calls, [ call.parse(sentinel.data), call.get_output_message_id(parsed[0]), call.delete_too_long_address(parsed[0]), call.get_output_message_id(parsed[1]), call.delete_too_long_address(parsed[1]), call.setting_error_event_info(parsed[0]), call.setting_error_event_info().__enter__(), call.postprocess_parsed(sentinel.data, parsed[0], 2, item_no=1), call.setting_error_event_info().__exit__(None, None, None), call.setting_error_event_info(parsed[1]), call.setting_error_event_info().__enter__(), call.postprocess_parsed(sentinel.data, parsed[1], 2, item_no=2), call.setting_error_event_info().__exit__(None, None, None), ]) def test__get_output_bodies__record_dict_not_used_as_context_manager(self): parsed = [ MagicMock(**{ '__class__': RecordDict, 'used_as_context_manager': False }) for i in (1, 2) ] self.mock.configure_mock(**{'parse.return_value': parsed}) with self.assertRaises(AssertionError): self.meth.get_output_bodies(sentinel.data, FilePagedSequence._instance_mock()) self.assertEqual(self.mock.method_calls, [ call.parse(sentinel.data), ]) def test__get_output_bodies__parse_yielded_no_items(self): self.mock.configure_mock(**{'parse.return_value': iter([])}) with self.assertRaises(ValueError): self.meth.get_output_bodies(sentinel.data, FilePagedSequence._instance_mock()) self.assertEqual(self.mock.method_calls, [ call.parse(sentinel.data), ]) def test__get_output_bodies__parse_yielded_no_items__allow_empty_results( self): self.mock.configure_mock(**{ 'parse.return_value': iter([]), 'allow_empty_results': True }) seq_mock = FilePagedSequence._instance_mock() output_bodies = self.meth.get_output_bodies(sentinel.data, seq_mock) self.assertIs(output_bodies, seq_mock) self.assertEqual(seq_mock._list, []) # just empty self.assertEqual(self.mock.mock_calls, [ call.parse(sentinel.data), ]) def test__delete_too_long_address__address_is_ok(self): parsed = RecordDict() parsed['address'] = [{'ip': i + 1} for i in xrange(MAX_IPS_IN_ADDRESS)] expected = RecordDict() expected['address'] = [{ 'ip': i + 1 } for i in xrange(MAX_IPS_IN_ADDRESS)] self.meth.delete_too_long_address(parsed) self.assertEqual(parsed, expected) def test__delete_too_long_address__address_is_too_long(self): ips = MAX_IPS_IN_ADDRESS + 1 parsed = RecordDict() parsed['id'] = '0123456789abcdef0123456789abcdef' parsed['address'] = [{'ip': i + 1} for i in xrange(ips)] expected = RecordDict() expected['id'] = '0123456789abcdef0123456789abcdef' self.meth.delete_too_long_address(parsed) self.assertEqual(parsed, expected) def test__delete_too_long_address__address_is_empty(self): parsed = RecordDict() parsed.update({'source': 'foo.bar'}) expected = RecordDict() expected.update({'source': 'foo.bar'}) self.meth.delete_too_long_address(parsed) self.assertEqual(parsed, expected) def test__get_output_message_id(self): inputs_and_resultant_hash_bases = [ # basics ({ 'source': 'foo.bar' }, 'source,foo.bar'), ({ u'source': u'foo.bar' }, 'source,foo.bar'), # proper sorting of multiple values ({ 'key1': 2, 'key2': ['value2', 'value3', 'value1'] }, 'key1,2\nkey2,value1,value2,value3'), # ...and of keys + proper encoding of unicode keys/values ({ u'key2': [u'value3', u'value1', u'value2'], u'key1': 2L }, 'key1,2\nkey2,value1,value2,value3'), # ...as well as proper int/long normalization/representation ({ u'key2': [30, 10, 20L], u'key1': 9000111222333444555666777888999000L }, 'key1,9000111222333444555666777888999000\nkey2,10,20,30'), # non-ascii values ({ 'target': 'zażółć', u'client': [u'jaźń', u'gęślą'] }, 'client,gęślą,jaźń\ntarget,zażółć'), ({ u'target': u'zażółć', 'client': ['jaźń', 'gęślą'] }, 'client,gęślą,jaźń\ntarget,zażółć'), # subdicts ({ 'dip': u'3.3.3.3', u'address': [{ 'ip': '255.255.255.0' }, { 'ip': '127.0.0.1' }] }, "address,{'ip': '127.0.0.1'},{'ip': '255.255.255.0'}\ndip,3.3.3.3" ), # non-ascii subdict keys/values ({ u'key2': [{ 'ką2': 'vą2' }, { 'ką1': 'vą1' }], 'key1': { 'ką': 'vą' } }, "key1,{'k\\xc4\\x85': 'v\\xc4\\x85'}\n" + "key2,{'k\\xc4\\x851': 'v\\xc4\\x851'},{'k\\xc4\\x852': 'v\\xc4\\x852'}" ), # proper encoding of unicode keys/values + proper sorting of whole subdicts ({ 'key1': { u'ką': u'vą' }, 'key2': [{ u'ką2': 'vą2' }, { 'ką1': u'vą1' }] }, "key1,{'k\\xc4\\x85': 'v\\xc4\\x85'}\n" + "key2,{'k\\xc4\\x851': 'v\\xc4\\x851'},{'k\\xc4\\x852': 'v\\xc4\\x852'}" ), # ...as well as proper int/long normalization/representation ({ 'key1': { u'k': 2L }, 'key2': [{ 'k2': 2L }, { u'k1': 1 }] }, "key1,{'k': 2}\nkey2,{'k1': 1},{'k2': 2}"), ({ u'key2': [{ 'k2': 2 }, { 'k1': 1 }], 'key1': { 'k': 3 } }, "key1,{'k': 3}\nkey2,{'k1': 1},{'k2': 2}"), ({ u'key2': [{ 'k2': 2L }, { 'k1': 1L }], 'key1': { 'k': 9000111222333444555666777888999000L } }, "key1,{'k': 9000111222333444555666777888999000}\nkey2,{'k1': 1},{'k2': 2}" ), # proper sorting of multiple items in subdicts ({ 'key1': { 'c': 2, u'a': 3L, u'b': 1L }, 'key2': [{ 'c': 2, u'a': 3L, u'b': 1L }, { 'd': 3, u'a': 2L, u'b': 1L }] }, "key1,{'a': 3, 'b': 1, 'c': 2}\n" + "key2,{'a': 2, 'b': 1, 'd': 3},{'a': 3, 'b': 1, 'c': 2}"), ] class _RecordDict(RecordDict): adjust_key1 = adjust_key2 = None optional_keys = RecordDict.optional_keys | {'key1', 'key2'} parser = BaseParser.__new__(BaseParser) for input_dict, expected_base in inputs_and_resultant_hash_bases: record_dict = _RecordDict(input_dict) expected_result = hashlib.md5(expected_base).hexdigest() result = parser.get_output_message_id(record_dict) self.assertIsInstance(result, str) self.assertEqual(result, expected_result) def test__get_output_message_id__errors(self): inputs_and_exc_classes = [ # bad subdict key type ( { 'key1': { 32: 2 } }, TypeError, ), ( { 'key1': [{ 32: 2 }] }, TypeError, ), # bad subdict value type ( { 'key1': { 'k': 2.3 } }, TypeError, ), ( { 'key1': [{ 'k': 2.3 }] }, TypeError, ), ( { 'key1': { 'k': { 'k': 2 } } }, # nesting is illegal TypeError, ), ( { 'key1': [{ 'k': { 'k': 2 } }] }, # nesting is illegal TypeError, ), # bad value type ( { 'key1': 2.3 }, TypeError, ), ( { 'key1': [2.3] }, TypeError, ), ( { 'key1': [[2]] }, # nesting is illegal TypeError, ), ] class _RecordDict(RecordDict): adjust_key1 = adjust_key2 = None optional_keys = RecordDict.optional_keys | {'key1', 'key2'} parser = BaseParser.__new__(BaseParser) for input_dict, exc_class in inputs_and_exc_classes: record_dict = _RecordDict(input_dict) with self.assertRaises(exc_class): parser.get_output_message_id(record_dict) def test__postprocess_parsed__without__do_not_resolve_fqdn_to_ip(self): data = {} parsed = RecordDict() self.meth.postprocess_parsed(data, parsed, 1, item_no=1) self.assertEqual(parsed, {}) def test__postprocess_parsed__with__do_not_resolve_fqdn_to_ip__False(self): data = {'_do_not_resolve_fqdn_to_ip': False} parsed = RecordDict() self.meth.postprocess_parsed(data, parsed, 1, item_no=1) self.assertEqual(parsed, {}) def test__postprocess_parsed__with__do_not_resolve_fqdn_to_ip__True(self): data = {'_do_not_resolve_fqdn_to_ip': True} parsed = RecordDict() self.meth.postprocess_parsed(data, parsed, 1, item_no=1) self.assertEqual(parsed, {'_do_not_resolve_fqdn_to_ip': True})
def prepare_config_full(cls, pyramid_configurator): # type: (Configurator) -> Config config_constructor_kwargs = cls.prepare_config_constructor_kwargs(pyramid_configurator) return Config(cls.config_spec, **config_constructor_kwargs)