def test__enrich__with_fqdn_from_url_not_resolved(self): self.enricher._resolver.query = mock.MagicMock(side_effect=DNSException) data = self.enricher.enrich(RecordDict({"url": "http://www.nask.pl/asd"})) self.assertEqualIncludingTypes(data, RecordDict({ "enriched": (["fqdn"], {}), "url": "http://www.nask.pl/asd", "fqdn": "www.nask.pl"}))
def test__delete_too_long_address__address_is_empty(self): parsed = RecordDict() parsed.update({'source': 'foo.bar'}) expected = RecordDict() expected.update({'source': 'foo.bar'}) self.meth.delete_too_long_address(parsed) self.assertEqual(parsed, expected)
def test__delete_too_long_address__address_is_ok(self): parsed = RecordDict() parsed['address'] = [{'ip': i+1} for i in xrange(MAX_IPS_IN_ADDRESS)] expected = RecordDict() expected['address'] = [{'ip': i+1} for i in xrange(MAX_IPS_IN_ADDRESS)] self.meth.delete_too_long_address(parsed) self.assertEqual(parsed, expected)
def test__enrich__with_ip_url_given(self): data = self.enricher.enrich(RecordDict({"url": "http://192.168.0.1/asd"})) self.assertEqualIncludingTypes(data, RecordDict({ "enriched": ([], {"192.168.0.1": ["asn", "cc", "ip"]}), "url": "http://192.168.0.1/asd", "address": [{"ip": '192.168.0.1', "asn": '1234', "cc": 'PL'}]}))
def test__enrich__with_fqdn_given(self): data = self.enricher.enrich(RecordDict({"fqdn": "cert.pl"})) self.enricher._resolver.asert_called_once_with("cert.pl") self.assertEqualIncludingTypes(data, RecordDict({ "enriched": ([], {"127.0.0.1": ["asn", "cc", "ip"]}), "fqdn": "cert.pl", "address": [{"ip": '127.0.0.1', "asn": '1234', "cc": 'PL'}]}))
def test__delete_too_long_address__address_is_too_long(self): ips = MAX_IPS_IN_ADDRESS + 1 parsed = RecordDict() parsed['id'] = '0123456789abcdef0123456789abcdef' parsed['address'] = [{'ip': i + 1} for i in xrange(ips)] expected = RecordDict() expected['id'] = '0123456789abcdef0123456789abcdef' self.meth.delete_too_long_address(parsed) self.assertEqual(parsed, expected)
def test__enrich__with_fqdn_given__with_nodns_flag(self): data = self.enricher.enrich(RecordDict({ "fqdn": "cert.pl", "_do_not_resolve_fqdn_to_ip": True})) self.enricher._resolver.asert_called_once_with("cert.pl") self.assertEqualIncludingTypes(data, RecordDict({ "enriched": ([], {}), "fqdn": "cert.pl", "_do_not_resolve_fqdn_to_ip": True}))
def test__enrich__with_address_and_fqdn_given(self): data = self.enricher.enrich(RecordDict({ "fqdn": "cert.pl", "address": [{"ip": "10.20.30.40"}]})) self.assertEqualIncludingTypes(data, RecordDict({ "enriched": ([], {"10.20.30.40": ["asn", "cc"]}), "fqdn": "cert.pl", "address": [{"ip": '10.20.30.40', "asn": '1234', "cc": 'PL'}]}))
def test__enrich__with_url_given__with_nodns_flag(self): data = self.enricher.enrich(RecordDict({ "url": "http://www.nask.pl/asd", "_do_not_resolve_fqdn_to_ip": True})) self.enricher._resolver.asert_called_once_with("www.nask.pl") self.assertEqualIncludingTypes(data, RecordDict({ "enriched": (["fqdn"], {}), "url": "http://www.nask.pl/asd", "fqdn": "www.nask.pl", "_do_not_resolve_fqdn_to_ip": True}))
def test__enrich__with_url_given(self): data = self.enricher.enrich(RecordDict({"url": "http://www.nask.pl/asd"})) self.enricher._resolver.asert_called_once_with("www.nask.pl") self.assertEqualIncludingTypes(data, RecordDict({ "enriched": (["fqdn"], {"127.0.0.1": ["asn", "cc", "ip"]}), "url": "http://www.nask.pl/asd", "fqdn": "www.nask.pl", "address": [{"ip": '127.0.0.1', "asn": '1234', "cc": 'PL'}]}))
def test__enrich__with_address_and_url_given(self): data = self.enricher.enrich(RecordDict({ "url": "http://www.nask.pl/asd", "address": [{"ip": "10.20.30.40"}]})) self.assertEqualIncludingTypes(data, RecordDict({ "enriched": (["fqdn"], {"10.20.30.40": ["asn", "cc"]}), "url": "http://www.nask.pl/asd", "fqdn": "www.nask.pl", "address": [{"ip": '10.20.30.40', "asn": '1234', "cc": 'PL'}]}))
def test__enrich__with_excluded_ips_config__with_some_ip_to_exclude__2(self): self.enricher._enrich_config = {'dnshost': '8.8.8.8', 'dnsport': '53', 'geoippath': '/usr/share/GeoIP', 'excluded_ips': '127.0.0.1, 2.2.2.2, 3.3.3.3'} self.enricher.excluded_ips = self.enricher._get_excluded_ips() data = self.enricher.enrich(RecordDict({"url": "http://www.nask.pl/asd"})) self.enricher._resolver.asert_called_once_with("www.nask.pl") self.assertEqualIncludingTypes(data, RecordDict({ "enriched": (["fqdn"], {}), "url": "http://www.nask.pl/asd", "fqdn": "www.nask.pl"})) # (note: emptied `address` removed)
def test__enrich__with_ip_url_given__with_nodns_flag(self): data = self.enricher.enrich(RecordDict({ "url": "http://192.168.0.1/asd", "_do_not_resolve_fqdn_to_ip": True})) self.assertEqualIncludingTypes(data, RecordDict({ # (here the '_do_not_resolve_fqdn_to_ip' flag did *not* change behaviour) "enriched": ([], {"192.168.0.1": ["asn", "cc", "ip"]}), "url": "http://192.168.0.1/asd", "address": [{"ip": '192.168.0.1', "asn": '1234', "cc": 'PL'}], "_do_not_resolve_fqdn_to_ip": True}))
def test__enrich__with_address_and_fqdn_given__with_nodns_flag(self): data = self.enricher.enrich(RecordDict({ "fqdn": "cert.pl", "address": [{"ip": "10.20.30.40"}], "_do_not_resolve_fqdn_to_ip": True})) self.assertEqualIncludingTypes(data, RecordDict({ # (here the '_do_not_resolve_fqdn_to_ip' flag did *not* change behaviour) "enriched": ([], {"10.20.30.40": ["asn", "cc"]}), "fqdn": "cert.pl", "address": [{"ip": '10.20.30.40', "asn": '1234', "cc": 'PL'}], "_do_not_resolve_fqdn_to_ip": True}))
def test__filter_out_excluded_ips__with_excluded_ips_being_None(self): self.enricher.excluded_ips = None data = RecordDict({ "url": "http://www.nask.pl/asd", "address": [{'ip': "127.0.0.1"}], }) expected = RecordDict({ "url": "http://www.nask.pl/asd", "address": [{'ip': "127.0.0.1"}], }) ip_to_enr_mock = mock.MagicMock() ip_to_enr_expected_calls = [] self.enricher._filter_out_excluded_ips(data, ip_to_enr_mock) self.assertEqualIncludingTypes(expected, data) self.assertEqual(ip_to_enr_mock.mock_calls, ip_to_enr_expected_calls)
def test__enrich__with_excluded_ips_config__without_any_ip_to_exclude(self): self.enricher._enrich_config = {'dnshost': '8.8.8.8', 'dnsport': '53', 'geoippath': '/usr/share/GeoIP', 'excluded_ips': '2.2.2.2, 3.3.3.3'} self.enricher.excluded_ips = self.enricher._get_excluded_ips() data = self.enricher.enrich(RecordDict({"url": "http://www.nask.pl/asd"})) self.enricher._resolver.asert_called_once_with("www.nask.pl") self.assertEqualIncludingTypes(data, RecordDict({ "enriched": (["fqdn"], {"127.0.0.1": ["asn", "cc", "ip"]}), "url": "http://www.nask.pl/asd", "fqdn": "www.nask.pl", "address": [{"ip": '127.0.0.1', "asn": '1234', "cc": 'PL'}]}))
def test__filter_out_excluded_ips__with_no_ip_in_excluded_ips(self): self.enricher.excluded_ips = iptools.IpRangeList('1.1.1.1', '2.2.2.2', '3.3.3.3') data = RecordDict({ "url": "http://www.nask.pl/asd", "address": [{'ip': '1.1.1.5'}, {'ip': '2.1.1.1'}], }) expected = RecordDict({ "url": "http://www.nask.pl/asd", "address": [{'ip': '1.1.1.5'}, {'ip': '2.1.1.1'}], }) ip_to_enr_mock = mock.MagicMock() ip_to_enr_expected_calls = [] self.enricher._filter_out_excluded_ips(data, ip_to_enr_mock) self.assertEqualIncludingTypes(expected, data) self.assertEqual(ip_to_enr_mock.mock_calls, ip_to_enr_expected_calls)
def test_adding_asn_cc_if_possible(self): """Test if asn/cc are (maybe) added""" self.enricher.gi_asn.org_by_addr.side_effect = [ pygeoip.GeoIPError, "AS1234", "AS123456"] self.enricher.gi_cc.country_code_by_addr.side_effect = [ "PL", "UK", pygeoip.GeoIPError] data = RecordDict({ "address": [{"ip": "127.0.0.1"}, {"ip": "192.187.0.1"}, {"ip": "10.15.1.255"}]}) data.update(self.COMMON_DATA) self.enricher.enrich(data) self.assertEqual(data["address"], [ {"ip": "127.0.0.1", "cc": "PL"}, {"ip": "192.187.0.1", "asn": 1234, "cc": "UK"}, {"ip": "10.15.1.255", "asn": 123456}, ]) self.assertEqual(data["enriched"], ([], { "127.0.0.1": ["cc"], "192.187.0.1": ["asn", "cc"], "10.15.1.255": ["asn"], }))
def test__fqdn_to_ip__called(self): """Test if fqdn_to_ip is called if data does not contain address""" data = RecordDict({"fqdn": "cert.pl"}) data.update(self.COMMON_DATA) self.enricher.fqdn_to_ip = mock.MagicMock() self.enricher.enrich(data) self.enricher.fqdn_to_ip.assert_called_with("cert.pl")
def test__url_to_fqdn_or_ip__called_for_ip_url(self): """Test if url_to_fqdn_or_ip is called if data does not contain address and fqdn""" data = RecordDict({"url": "http://192.168.0.1"}) data.update(self.COMMON_DATA) self.enricher.url_to_fqdn_or_ip = mock.MagicMock(return_value="192.168.0.1") self.enricher.enrich(data) self.enricher.url_to_fqdn_or_ip.assert_called_with("http://192.168.0.1")
def actual_data_for_existing_asn_cc_always_dropped_and_new_ones_added_if_possible(self, mock_asn, mock_cc): self.enricher.gi_asn.asn = mock_asn self.enricher.gi_cc.city = mock_cc return RecordDict({ "address": [{"ip": "127.0.0.1", "cc": "JP"}, {"ip": "192.187.0.1", "cc": "US", "asn": 424242}, {"ip": "10.15.1.255", "asn": 434343}]})
def test_existing_asn_cc_always_dropped_and_new_ones_added_if_possible(self, LOGGER_mock): """Test if already existing asn/cc are removed and new ones are (maybe) added""" self.enricher.gi_asn.org_by_addr.side_effect = [ pygeoip.GeoIPError, pygeoip.GeoIPError, "AS12345"] self.enricher.gi_cc.country_code_by_addr.side_effect = [ pygeoip.GeoIPError, "PL", "UK"] data = RecordDict({ "address": [{"ip": "127.0.0.1", "cc": "JP"}, {"ip": "192.187.0.1", "cc": "US", "asn": 424242}, {"ip": "10.15.1.255", "asn": 434343}]}) data.update(self.COMMON_DATA) expected_num_of_warnings = 4 # 2 existing `cc` + 2 existing `asn` self.enricher.enrich(data) self.assertEqual(data["address"], [ {"ip": "127.0.0.1"}, {"ip": "192.187.0.1", "cc": "PL"}, {"ip": "10.15.1.255", "asn": 12345, "cc": "UK"}, ]) self.assertEqual(data["enriched"], ([], { "192.187.0.1": ["cc"], "10.15.1.255": ["asn", "cc"], })) self.assertEqual( len(LOGGER_mock.warning.mock_calls), expected_num_of_warnings)
def actual_data_for_adding_asn_cc_if_possible(self, asn_mock, cc_mock): self.enricher.gi_asn.asn = asn_mock self.enricher.gi_cc.city = cc_mock return RecordDict({ "address": [{"ip": "127.0.0.1"}, {"ip": "192.187.0.1"}, {"ip": "10.15.1.255"}]})
def test__fqdn_to_ip__not_called(self): """Test if fqdn_to_ip not called if address already present""" data = RecordDict({ "address": [{"ip": "127.0.0.1"}, {"ip": "192.187.0.1"}, {"ip": "10.15.1.255"}]}) data.update(self.COMMON_DATA) self.enricher.fqdn_to_ip = mock.MagicMock(return_value="127.0.0.1") self.enricher.enrich(data) self.assertFalse(self.enricher.fqdn_to_ip.called)
def test__filter_out_excluded_ips__with_range_of_ips(self): self.enricher.excluded_ips = iptools.IpRangeList('3.0.0.0/8') data = RecordDict({ "url": "http://www.nask.pl/asd", "address": [ { 'ip': '3.3.3.3', 'asn': 1234 }, { 'ip': '3.255.255.255', 'asn': 5632 }, { 'ip': '3.0.0.0', 'asn': 5631 }, { 'ip': '2.255.255.255', 'asn': 5632 }, ], }) expected = RecordDict({ "url": "http://www.nask.pl/asd", "address": [ { 'ip': '2.255.255.255', 'asn': 5632, }, ], }) ip_to_enr_mock = mock.MagicMock() ip_to_enr_expected_call_items = [ mock.call.pop('3.3.3.3', None), mock.call.pop('3.255.255.255', None), mock.call.pop('3.0.0.0', None), ] self.enricher._filter_out_excluded_ips(data, ip_to_enr_mock) self.assertEqualIncludingTypes(expected, data) self.assertItemsEqual(ip_to_enr_mock.mock_calls, ip_to_enr_expected_call_items)
def test__ip_to_cc__called(self): """Test if ip_to_cc was called for all ips""" data = RecordDict({ "address": [{"ip": "127.0.0.1"}, {"ip": "192.187.0.1"}, {"ip": "10.15.1.255"}]}) data.update(self.COMMON_DATA) self.enricher.ip_to_cc = mock.MagicMock(return_value="") self.enricher.enrich(data) for addr in data["address"]: self.enricher.ip_to_cc.assert_any_call(addr["ip"]) self.assertEqual(len(data["address"]), self.enricher.ip_to_cc.call_count)
def test__enrich__with_fqdn_given__resolved_to_various_ips_with_duplicates(self): self.enricher._resolver.query.return_value = [ '2.2.2.2', '127.0.0.1', '13.1.2.3', '1.1.1.1', '127.0.0.1', # duplicate '13.1.2.3', # duplicate '12.11.10.9', '13.1.2.3', # duplicate '1.0.1.1', ] data = self.enricher.enrich(RecordDict({"fqdn": "cert.pl"})) self.enricher._resolver.asert_called_once_with("cert.pl") self.assertEqualIncludingTypes(data, RecordDict({ "enriched": ([], {"1.0.1.1": ["asn", "cc", "ip"], "1.1.1.1": ["asn", "cc", "ip"], "12.11.10.9": ["asn", "cc", "ip"], "127.0.0.1": ["asn", "cc", "ip"], "13.1.2.3": ["asn", "cc", "ip"], "2.2.2.2": ["asn", "cc", "ip"]}), "fqdn": "cert.pl", "address": [{"ip": '1.0.1.1', # note: *removed IP duplicates* and "asn": '1234', # *ordered* by IP (textually) "cc": 'PL'}, {"ip": '1.1.1.1', "asn": '1234', "cc": 'PL'}, {"ip": '12.11.10.9', "asn": '1234', "cc": 'PL'}, {"ip": '127.0.0.1', "asn": '1234', "cc": 'PL'}, {"ip": '13.1.2.3', "asn": '1234', "cc": 'PL'}, {"ip": '2.2.2.2', "asn": '1234', "cc": 'PL'}]}))
def test__get_client_and_urls_matched__single_ip(self): test_criteria_local = [ { 'org_id': 'cli16bit', 'cc_seq': ['PL'], 'asn_seq': [21467], 'fqdn_seq': [u'ala.eu'], 'ip_min_max_seq': [(1292036523, 1292036523)], # '77.2.233.171' } ] data = self.prepare_mock(test_criteria_local) data['address'][0]['ip'] = '77.2.233.171' record_dict = RecordDict(data) self.assertEqual( self.filter.get_client_and_urls_matched(record_dict, self.fqdn_only_categories), (['cli16bit'], {})) for other_ip in [ '77.2.233.170', '77.2.233.172', '77.2.233.250', '77.2.233.0', '77.2.254.250', '77.3.100.1', '77.3.255.255', '77.1.233.171', '77.3.0.0', '77.4.0.0', '1.2.3.4', '10.20.30.40', '0.0.0.0', '255.255.255.255', ]: data['address'][0]['ip'] = other_ip record_dict = RecordDict(data) self.assertEqual( self.filter.get_client_and_urls_matched(record_dict, self.fqdn_only_categories), ([], {}))
def test_routing_key_modified(self): """Test if routing key after enrichement is set to "enriched.*" when publishing to output queue""" self.enricher.publish_output = mock.MagicMock() data = RecordDict({ "address": [{"ip": "127.0.0.1"}, {"ip": "192.187.0.1"}, {"ip": "10.15.1.255"}]}) data.update(self.COMMON_DATA) body = data.get_ready_json() initial_routing_key = "event.parsed.test.test-source" properties = None self.enricher.input_callback(initial_routing_key, body, properties) args, kwargs = self.enricher.publish_output.call_args self.assertIn("routing_key", kwargs) self.assertEqual(kwargs["routing_key"], "event.enriched.test.test-source")
def test__enrich__with_irrelevant_data(self): data = self.enricher.enrich(RecordDict(self.COMMON_DATA)) self.assertEqualIncludingTypes(data, RecordDict(dict(self.COMMON_DATA, **{ 'enriched': ([], {})})))