def test__delete_too_long_address__address_is_ok(self):
parsed = RecordDict()
parsed['address'] = [{'ip': i+1} for i in xrange(MAX_IPS_IN_ADDRESS)]
expected = RecordDict()
expected['address'] = [{'ip': i+1} for i in xrange(MAX_IPS_IN_ADDRESS)]
self.meth.delete_too_long_address(parsed)
self.assertEqual(parsed, expected)
def test_adding_asn_cc_if_possible(self):
"""Test if asn/cc are (maybe) added"""
self.enricher.gi_asn.org_by_addr.side_effect = [
pygeoip.GeoIPError,
"AS1234",
"AS123456"]
self.enricher.gi_cc.country_code_by_addr.side_effect = [
"PL",
"UK",
pygeoip.GeoIPError]
data = RecordDict({
"address": [{"ip": "127.0.0.1"},
{"ip": "192.187.0.1"},
{"ip": "10.15.1.255"}]})
data.update(self.COMMON_DATA)
self.enricher.enrich(data)
self.assertEqual(data["address"], [
{"ip": "127.0.0.1", "cc": "PL"},
{"ip": "192.187.0.1", "asn": 1234, "cc": "UK"},
{"ip": "10.15.1.255", "asn": 123456},
])
self.assertEqual(data["enriched"], ([], {
"127.0.0.1": ["cc"],
"192.187.0.1": ["asn", "cc"],
"10.15.1.255": ["asn"],
}))
def test__get_client_and_urls_matched__many_url_seq(self):
test_criteria_local = [
{'org_id': 'org4',
'asn_seq': [],
'fqdn_seq': [''],
'url_seq': [u'wąska.pl', 'example.com', 'http://aaa.pl/auth.php'], }]
body = self.prepare_mock(test_criteria_local)
# test glob mach
body[u'url_pattern'] = u'*.*'
json_msg = json.dumps(body)
record_dict = RecordDict.from_json(json_msg)
self.assertEqual(
self.filter.get_client_and_urls_matched(record_dict, self.fqdn_only_categories),
(['org4'], {'org4': ['example.com', 'http://aaa.pl/auth.php', u'wąska.pl']}))
# test regexp mach
body[u'url_pattern'] = u'.*aaa\\.pl\\/auth.*'
json_msg = json.dumps(body)
record_dict = RecordDict.from_json(json_msg)
self.assertEqual(
self.filter.get_client_and_urls_matched(record_dict, self.fqdn_only_categories),
(['org4'], {'org4': ['http://aaa.pl/auth.php']}))
# test regexp mach
body[u'url_pattern'] = u'.*example.*'
json_msg = json.dumps(body)
record_dict = RecordDict.from_json(json_msg)
self.assertEqual(
self.filter.get_client_and_urls_matched(record_dict, self.fqdn_only_categories),
(['org4'], {'org4': ['example.com']}))
# test regexp mach
body[u'url_pattern'] = u'wą[a-z][xkl]a\\..*'
json_msg = json.dumps(body)
record_dict = RecordDict.from_json(json_msg)
self.assertEqual(
self.filter.get_client_and_urls_matched(record_dict, self.fqdn_only_categories),
(['org4'], {'org4': [u'wąska.pl']}))
def test__enrich__with_fqdn_from_url_not_resolved(self):
self.enricher._resolver.query = mock.MagicMock(side_effect=DNSException)
data = self.enricher.enrich(RecordDict({"url": "http://www.nask.pl/asd"}))
self.assertEqualIncludingTypes(data, RecordDict({
"enriched": (["fqdn"], {}),
"url": "http://www.nask.pl/asd",
"fqdn": "www.nask.pl"}))
def test__url_to_fqdn_or_ip__called_for_ip_url(self):
"""Test if url_to_fqdn_or_ip is called if data does not contain address and fqdn"""
data = RecordDict({"url": "http://192.168.0.1"})
data.update(self.COMMON_DATA)
self.enricher.url_to_fqdn_or_ip = mock.MagicMock(return_value="192.168.0.1")
self.enricher.enrich(data)
self.enricher.url_to_fqdn_or_ip.assert_called_with("http://192.168.0.1")
def test__get_client_and_urls_matched__url_pattern_empty(self):
test_criteria_local = [
{'org_id': 'org4',
'asn_seq': [],
'fqdn_seq': [''],
'url_seq': ['wp.pl', u'wpą.pl'], }]
body = self.prepare_mock(test_criteria_local)
# test empty url_pattern unicode
body[u'url_pattern'] = u''
json_msg = json.dumps(body)
record_dict = RecordDict.from_json(json_msg)
self.assertEqual(
self.filter.get_client_and_urls_matched(record_dict, self.fqdn_only_categories),
([], {}))
# test empty url_pattern string
body[u'url_pattern'] = ''
json_msg = json.dumps(body)
record_dict = RecordDict.from_json(json_msg)
self.assertEqual(
self.filter.get_client_and_urls_matched(record_dict, self.fqdn_only_categories),
([], {}))
# # test empty url_pattern = None
body[u'url_pattern'] = None
json_msg = json.dumps(body)
record_dict = RecordDict.from_json(json_msg)
self.assertEqual(
self.filter.get_client_and_urls_matched(record_dict, self.fqdn_only_categories),
([], {}))
def test__fqdn_to_ip__called(self):
"""Test if fqdn_to_ip is called if data does not contain address"""
data = RecordDict({"fqdn": "cert.pl"})
data.update(self.COMMON_DATA)
self.enricher.fqdn_to_ip = mock.MagicMock()
self.enricher.enrich(data)
self.enricher.fqdn_to_ip.assert_called_with("cert.pl")
def test_existing_asn_cc_always_dropped_and_new_ones_added_if_possible(self, LOGGER_mock):
"""Test if already existing asn/cc are removed and new ones are (maybe) added"""
self.enricher.gi_asn.org_by_addr.side_effect = [
pygeoip.GeoIPError,
pygeoip.GeoIPError,
"AS12345"]
self.enricher.gi_cc.country_code_by_addr.side_effect = [
pygeoip.GeoIPError,
"PL",
"UK"]
data = RecordDict({
"address": [{"ip": "127.0.0.1", "cc": "JP"},
{"ip": "192.187.0.1", "cc": "US", "asn": 424242},
{"ip": "10.15.1.255", "asn": 434343}]})
data.update(self.COMMON_DATA)
expected_num_of_warnings = 4 # 2 existing `cc` + 2 existing `asn`
self.enricher.enrich(data)
self.assertEqual(data["address"], [
{"ip": "127.0.0.1"},
{"ip": "192.187.0.1", "cc": "PL"},
{"ip": "10.15.1.255", "asn": 12345, "cc": "UK"},
])
self.assertEqual(data["enriched"], ([], {
"192.187.0.1": ["cc"],
"10.15.1.255": ["asn", "cc"],
}))
self.assertEqual(
len(LOGGER_mock.warning.mock_calls),
expected_num_of_warnings)
def test__enrich__with_ip_url_given(self):
data = self.enricher.enrich(RecordDict({"url": "http://192.168.0.1/asd"}))
self.assertEqualIncludingTypes(data, RecordDict({
"enriched": ([], {"192.168.0.1": ["asn", "cc", "ip"]}),
"url": "http://192.168.0.1/asd",
"address": [{"ip": '192.168.0.1',
"asn": '1234',
"cc": 'PL'}]}))
def test__delete_too_long_address__address_is_too_long(self):
ips = MAX_IPS_IN_ADDRESS + 1
parsed = RecordDict()
parsed['id'] = '0123456789abcdef0123456789abcdef'
parsed['address'] = [{'ip': i + 1} for i in xrange(ips)]
expected = RecordDict()
expected['id'] = '0123456789abcdef0123456789abcdef'
self.meth.delete_too_long_address(parsed)
self.assertEqual(parsed, expected)
def test__enrich__with_fqdn_given(self):
data = self.enricher.enrich(RecordDict({"fqdn": "cert.pl"}))
self.enricher._resolver.asert_called_once_with("cert.pl")
self.assertEqualIncludingTypes(data, RecordDict({
"enriched": ([], {"127.0.0.1": ["asn", "cc", "ip"]}),
"fqdn": "cert.pl",
"address": [{"ip": '127.0.0.1',
"asn": '1234',
"cc": 'PL'}]}))
def test__enrich__with_fqdn_given__with_nodns_flag(self):
data = self.enricher.enrich(RecordDict({
"fqdn": "cert.pl",
"_do_not_resolve_fqdn_to_ip": True}))
self.enricher._resolver.asert_called_once_with("cert.pl")
self.assertEqualIncludingTypes(data, RecordDict({
"enriched": ([], {}),
"fqdn": "cert.pl",
"_do_not_resolve_fqdn_to_ip": True}))
def test__fqdn_to_ip__not_called(self):
"""Test if fqdn_to_ip not called if address already present"""
data = RecordDict({
"address": [{"ip": "127.0.0.1"},
{"ip": "192.187.0.1"},
{"ip": "10.15.1.255"}]})
data.update(self.COMMON_DATA)
self.enricher.fqdn_to_ip = mock.MagicMock(return_value="127.0.0.1")
self.enricher.enrich(data)
self.assertFalse(self.enricher.fqdn_to_ip.called)
def test__enrich__with_url_given(self):
data = self.enricher.enrich(RecordDict({"url": "http://www.nask.pl/asd"}))
self.enricher._resolver.asert_called_once_with("www.nask.pl")
self.assertEqualIncludingTypes(data, RecordDict({
"enriched": (["fqdn"], {"127.0.0.1": ["asn", "cc", "ip"]}),
"url": "http://www.nask.pl/asd",
"fqdn": "www.nask.pl",
"address": [{"ip": '127.0.0.1',
"asn": '1234',
"cc": 'PL'}]}))
def test__enrich__with_url_given__with_nodns_flag(self):
data = self.enricher.enrich(RecordDict({
"url": "http://www.nask.pl/asd",
"_do_not_resolve_fqdn_to_ip": True}))
self.enricher._resolver.asert_called_once_with("www.nask.pl")
self.assertEqualIncludingTypes(data, RecordDict({
"enriched": (["fqdn"], {}),
"url": "http://www.nask.pl/asd",
"fqdn": "www.nask.pl",
"_do_not_resolve_fqdn_to_ip": True}))
def test__enrich__with_address_and_fqdn_given(self):
data = self.enricher.enrich(RecordDict({
"fqdn": "cert.pl",
"address": [{"ip": "10.20.30.40"}]}))
self.assertEqualIncludingTypes(data, RecordDict({
"enriched": ([], {"10.20.30.40": ["asn", "cc"]}),
"fqdn": "cert.pl",
"address": [{"ip": '10.20.30.40',
"asn": '1234',
"cc": 'PL'}]}))
def test__enrich__with_address_and_url_given(self):
data = self.enricher.enrich(RecordDict({
"url": "http://www.nask.pl/asd",
"address": [{"ip": "10.20.30.40"}]}))
self.assertEqualIncludingTypes(data, RecordDict({
"enriched": (["fqdn"], {"10.20.30.40": ["asn", "cc"]}),
"url": "http://www.nask.pl/asd",
"fqdn": "www.nask.pl",
"address": [{"ip": '10.20.30.40',
"asn": '1234',
"cc": 'PL'}]}))
def test__ip_to_cc__called(self):
"""Test if ip_to_cc was called for all ips"""
data = RecordDict({
"address": [{"ip": "127.0.0.1"},
{"ip": "192.187.0.1"},
{"ip": "10.15.1.255"}]})
data.update(self.COMMON_DATA)
self.enricher.ip_to_cc = mock.MagicMock(return_value="")
self.enricher.enrich(data)
for addr in data["address"]:
self.enricher.ip_to_cc.assert_any_call(addr["ip"])
self.assertEqual(len(data["address"]), self.enricher.ip_to_cc.call_count)
def test__enrich__with_ip_url_given__with_nodns_flag(self):
data = self.enricher.enrich(RecordDict({
"url": "http://192.168.0.1/asd",
"_do_not_resolve_fqdn_to_ip": True}))
self.assertEqualIncludingTypes(data, RecordDict({
# (here the '_do_not_resolve_fqdn_to_ip' flag did *not* change behaviour)
"enriched": ([], {"192.168.0.1": ["asn", "cc", "ip"]}),
"url": "http://192.168.0.1/asd",
"address": [{"ip": '192.168.0.1',
"asn": '1234',
"cc": 'PL'}],
"_do_not_resolve_fqdn_to_ip": True}))
def test__enrich__with_excluded_ips_config__with_some_ip_to_exclude__2(self):
self.enricher._enrich_config = {'dnshost': '8.8.8.8',
'dnsport': '53',
'geoippath': '/usr/share/GeoIP',
'excluded_ips': '127.0.0.1, 2.2.2.2, 3.3.3.3'}
self.enricher.excluded_ips = self.enricher._get_excluded_ips()
data = self.enricher.enrich(RecordDict({"url": "http://www.nask.pl/asd"}))
self.enricher._resolver.asert_called_once_with("www.nask.pl")
self.assertEqualIncludingTypes(data, RecordDict({
"enriched": (["fqdn"], {}),
"url": "http://www.nask.pl/asd",
"fqdn": "www.nask.pl"})) # (note: emptied `address` removed)
def test__enrich__with_address_and_fqdn_given__with_nodns_flag(self):
data = self.enricher.enrich(RecordDict({
"fqdn": "cert.pl",
"address": [{"ip": "10.20.30.40"}],
"_do_not_resolve_fqdn_to_ip": True}))
self.assertEqualIncludingTypes(data, RecordDict({
# (here the '_do_not_resolve_fqdn_to_ip' flag did *not* change behaviour)
"enriched": ([], {"10.20.30.40": ["asn", "cc"]}),
"fqdn": "cert.pl",
"address": [{"ip": '10.20.30.40',
"asn": '1234',
"cc": 'PL'}],
"_do_not_resolve_fqdn_to_ip": True}))
def test__delete_too_long_address__address_is_empty(self):
parsed = RecordDict()
parsed.update({'source': 'foo.bar'})
expected = RecordDict()
expected.update({'source': 'foo.bar'})
self.meth.delete_too_long_address(parsed)
self.assertEqual(parsed, expected)
def test__filter_out_excluded_ips__with_no_ip_in_excluded_ips(self):
self.enricher.excluded_ips = iptools.IpRangeList('1.1.1.1', '2.2.2.2', '3.3.3.3')
data = RecordDict({
"url": "http://www.nask.pl/asd",
"address": [{'ip': '1.1.1.5'}, {'ip': '2.1.1.1'}],
})
expected = RecordDict({
"url": "http://www.nask.pl/asd",
"address": [{'ip': '1.1.1.5'}, {'ip': '2.1.1.1'}],
})
ip_to_enr_mock = mock.MagicMock()
ip_to_enr_expected_calls = []
self.enricher._filter_out_excluded_ips(data, ip_to_enr_mock)
self.assertEqualIncludingTypes(expected, data)
self.assertEqual(ip_to_enr_mock.mock_calls, ip_to_enr_expected_calls)
def test__filter_out_excluded_ips__with_excluded_ips_being_None(self):
self.enricher.excluded_ips = None
data = RecordDict({
"url": "http://www.nask.pl/asd",
"address": [{'ip': "127.0.0.1"}],
})
expected = RecordDict({
"url": "http://www.nask.pl/asd",
"address": [{'ip': "127.0.0.1"}],
})
ip_to_enr_mock = mock.MagicMock()
ip_to_enr_expected_calls = []
self.enricher._filter_out_excluded_ips(data, ip_to_enr_mock)
self.assertEqualIncludingTypes(expected, data)
self.assertEqual(ip_to_enr_mock.mock_calls, ip_to_enr_expected_calls)
def test__enrich__with_excluded_ips_config__without_any_ip_to_exclude(self):
self.enricher._enrich_config = {'dnshost': '8.8.8.8',
'dnsport': '53',
'geoippath': '/usr/share/GeoIP',
'excluded_ips': '2.2.2.2, 3.3.3.3'}
self.enricher.excluded_ips = self.enricher._get_excluded_ips()
data = self.enricher.enrich(RecordDict({"url": "http://www.nask.pl/asd"}))
self.enricher._resolver.asert_called_once_with("www.nask.pl")
self.assertEqualIncludingTypes(data, RecordDict({
"enriched": (["fqdn"], {"127.0.0.1": ["asn", "cc", "ip"]}),
"url": "http://www.nask.pl/asd",
"fqdn": "www.nask.pl",
"address": [{"ip": '127.0.0.1',
"asn": '1234',
"cc": 'PL'}]}))
def actual_data_for_adding_asn_cc_if_possible(self, asn_mock, cc_mock):
self.enricher.gi_asn.asn = asn_mock
self.enricher.gi_cc.city = cc_mock
return RecordDict({
"address": [{"ip": "127.0.0.1"},
{"ip": "192.187.0.1"},
{"ip": "10.15.1.255"}]})
def actual_data_for_existing_asn_cc_always_dropped_and_new_ones_added_if_possible(self, mock_asn, mock_cc):
self.enricher.gi_asn.asn = mock_asn
self.enricher.gi_cc.city = mock_cc
return RecordDict({
"address": [{"ip": "127.0.0.1", "cc": "JP"},
{"ip": "192.187.0.1", "cc": "US", "asn": 424242},
{"ip": "10.15.1.255", "asn": 434343}]})
def test__get_client_and_urls_matched__2(self):
body = {"category": "bots", "restriction": "public", "confidence": "medium",
"sha1": "023a00e7c2ef04ee5b0f767ba73ee39734323432", "name": "virut",
"proto": "tcp", "address": [{"cc": "XX", "ip": "1.1.1.1", "asn": "1"},
{"cc": "XX", "ip": "100.71.83.178", "asn": "1"},
{"cc": "XX", "ip": "102.71.83.178", "asn": "1"}],
"fqdn": "domain.com", "url": "http://onet.pl", "source": "hpfeeds.dionaea",
"time": "2013-07-01 20:37:20", "dport": "445",
"rid": "023a00e7c2ef04ee5b0f767ba73ee397",
"sport": "2147", "dip": "10.28.71.43", "id": "023a00e7c2ef04ee5b0f767ba73ee397"}
# tested key:[test_value,[valid value]]
input_data = {'ip': ['154.89.207.81', ['fdc']],
'cc': ['SU', ['decfba', 'fdc']],
'asn': ['45975', ['fdc']],
'fqdn': ['onetbrutalmikmokcert.eu', ['fdc']]}
self.auth_api_mock._get_inside_criteria.return_value = TEST_CRITERIA
for i in input_data:
body = self.reset_body(body)
if i == 'fqdn':
body['fqdn'] = input_data[i][0]
else:
body['address'][0][i] = input_data[i][0]
json_msg = json.dumps(body)
record_dict = RecordDict.from_json(json_msg)
self.assertItemsEqual(
self.filter.get_client_and_urls_matched(record_dict, self.fqdn_only_categories),
(input_data[i][1], {}))
def input_callback(self, routing_key, body, properties):
data = RecordDict.from_json(body)
with self.setting_error_event_info(data):
enriched = self.enrich(data)
rk = replace_segment(routing_key, 1, 'enriched')
body = enriched.get_ready_json()
self.publish_output(routing_key=rk, body=body)
def test_routing_key_modified(self):
"""Test if routing key after enrichement is set to "enriched.*"
when publishing to output queue"""
self.enricher.publish_output = mock.MagicMock()
data = RecordDict({
"address": [{"ip": "127.0.0.1"},
{"ip": "192.187.0.1"},
{"ip": "10.15.1.255"}]})
data.update(self.COMMON_DATA)
body = data.get_ready_json()
initial_routing_key = "event.parsed.test.test-source"
properties = None
self.enricher.input_callback(initial_routing_key, body, properties)
args, kwargs = self.enricher.publish_output.call_args
self.assertIn("routing_key", kwargs)
self.assertEqual(kwargs["routing_key"], "event.enriched.test.test-source")