Esempio n. 1
0
 def test__delete_too_long_address__address_is_ok(self):
     parsed = RecordDict()
     parsed['address'] = [{'ip': i+1} for i in xrange(MAX_IPS_IN_ADDRESS)]
     expected = RecordDict()
     expected['address'] = [{'ip': i+1} for i in xrange(MAX_IPS_IN_ADDRESS)]
     self.meth.delete_too_long_address(parsed)
     self.assertEqual(parsed, expected)
Esempio n. 2
0
 def test_adding_asn_cc_if_possible(self):
     """Test if asn/cc are (maybe) added"""
     self.enricher.gi_asn.org_by_addr.side_effect = [
         pygeoip.GeoIPError,
         "AS1234",
         "AS123456"]
     self.enricher.gi_cc.country_code_by_addr.side_effect = [
         "PL",
         "UK",
         pygeoip.GeoIPError]
     data = RecordDict({
         "address": [{"ip": "127.0.0.1"},
                     {"ip": "192.187.0.1"},
                     {"ip": "10.15.1.255"}]})
     data.update(self.COMMON_DATA)
     self.enricher.enrich(data)
     self.assertEqual(data["address"], [
         {"ip": "127.0.0.1", "cc": "PL"},
         {"ip": "192.187.0.1", "asn": 1234, "cc": "UK"},
         {"ip": "10.15.1.255", "asn": 123456},
     ])
     self.assertEqual(data["enriched"], ([], {
         "127.0.0.1": ["cc"],
         "192.187.0.1": ["asn", "cc"],
         "10.15.1.255": ["asn"],
     }))
Esempio n. 3
0
 def test__get_client_and_urls_matched__many_url_seq(self):
     test_criteria_local = [
         {'org_id': 'org4',
          'asn_seq': [],
          'fqdn_seq': [''],
          'url_seq': [u'wąska.pl', 'example.com', 'http://aaa.pl/auth.php'], }]
     body = self.prepare_mock(test_criteria_local)
     # test glob mach
     body[u'url_pattern'] = u'*.*'
     json_msg = json.dumps(body)
     record_dict = RecordDict.from_json(json_msg)
     self.assertEqual(
         self.filter.get_client_and_urls_matched(record_dict, self.fqdn_only_categories),
         (['org4'], {'org4': ['example.com', 'http://aaa.pl/auth.php', u'wąska.pl']}))
     # test regexp mach
     body[u'url_pattern'] = u'.*aaa\\.pl\\/auth.*'
     json_msg = json.dumps(body)
     record_dict = RecordDict.from_json(json_msg)
     self.assertEqual(
         self.filter.get_client_and_urls_matched(record_dict, self.fqdn_only_categories),
         (['org4'], {'org4': ['http://aaa.pl/auth.php']}))
     # test regexp mach
     body[u'url_pattern'] = u'.*example.*'
     json_msg = json.dumps(body)
     record_dict = RecordDict.from_json(json_msg)
     self.assertEqual(
         self.filter.get_client_and_urls_matched(record_dict, self.fqdn_only_categories),
         (['org4'], {'org4': ['example.com']}))
     # test regexp mach
     body[u'url_pattern'] = u'wą[a-z][xkl]a\\..*'
     json_msg = json.dumps(body)
     record_dict = RecordDict.from_json(json_msg)
     self.assertEqual(
         self.filter.get_client_and_urls_matched(record_dict, self.fqdn_only_categories),
         (['org4'], {'org4': [u'wąska.pl']}))
Esempio n. 4
0
 def test__enrich__with_fqdn_from_url_not_resolved(self):
     self.enricher._resolver.query = mock.MagicMock(side_effect=DNSException)
     data = self.enricher.enrich(RecordDict({"url": "http://www.nask.pl/asd"}))
     self.assertEqualIncludingTypes(data, RecordDict({
         "enriched": (["fqdn"], {}),
         "url": "http://www.nask.pl/asd",
         "fqdn": "www.nask.pl"}))
Esempio n. 5
0
 def test__url_to_fqdn_or_ip__called_for_ip_url(self):
     """Test if url_to_fqdn_or_ip is called if data does not contain address and fqdn"""
     data = RecordDict({"url": "http://192.168.0.1"})
     data.update(self.COMMON_DATA)
     self.enricher.url_to_fqdn_or_ip = mock.MagicMock(return_value="192.168.0.1")
     self.enricher.enrich(data)
     self.enricher.url_to_fqdn_or_ip.assert_called_with("http://192.168.0.1")
Esempio n. 6
0
 def test__get_client_and_urls_matched__url_pattern_empty(self):
     test_criteria_local = [
         {'org_id': 'org4',
          'asn_seq': [],
          'fqdn_seq': [''],
          'url_seq': ['wp.pl', u'wpą.pl'], }]
     body = self.prepare_mock(test_criteria_local)
     # test empty url_pattern unicode
     body[u'url_pattern'] = u''
     json_msg = json.dumps(body)
     record_dict = RecordDict.from_json(json_msg)
     self.assertEqual(
         self.filter.get_client_and_urls_matched(record_dict, self.fqdn_only_categories),
         ([], {}))
     # test empty url_pattern string
     body[u'url_pattern'] = ''
     json_msg = json.dumps(body)
     record_dict = RecordDict.from_json(json_msg)
     self.assertEqual(
         self.filter.get_client_and_urls_matched(record_dict, self.fqdn_only_categories),
         ([], {}))
     # # test empty url_pattern = None
     body[u'url_pattern'] = None
     json_msg = json.dumps(body)
     record_dict = RecordDict.from_json(json_msg)
     self.assertEqual(
         self.filter.get_client_and_urls_matched(record_dict, self.fqdn_only_categories),
         ([], {}))
Esempio n. 7
0
 def test__fqdn_to_ip__called(self):
     """Test if fqdn_to_ip is called if data does not contain address"""
     data = RecordDict({"fqdn": "cert.pl"})
     data.update(self.COMMON_DATA)
     self.enricher.fqdn_to_ip = mock.MagicMock()
     self.enricher.enrich(data)
     self.enricher.fqdn_to_ip.assert_called_with("cert.pl")
Esempio n. 8
0
 def test_existing_asn_cc_always_dropped_and_new_ones_added_if_possible(self, LOGGER_mock):
     """Test if already existing asn/cc are removed and new ones are (maybe) added"""
     self.enricher.gi_asn.org_by_addr.side_effect = [
         pygeoip.GeoIPError,
         pygeoip.GeoIPError,
         "AS12345"]
     self.enricher.gi_cc.country_code_by_addr.side_effect = [
         pygeoip.GeoIPError,
         "PL",
         "UK"]
     data = RecordDict({
         "address": [{"ip": "127.0.0.1", "cc": "JP"},
                     {"ip": "192.187.0.1", "cc": "US", "asn": 424242},
                     {"ip": "10.15.1.255", "asn": 434343}]})
     data.update(self.COMMON_DATA)
     expected_num_of_warnings = 4  # 2 existing `cc` + 2 existing `asn`
     self.enricher.enrich(data)
     self.assertEqual(data["address"], [
         {"ip": "127.0.0.1"},
         {"ip": "192.187.0.1", "cc": "PL"},
         {"ip": "10.15.1.255", "asn": 12345, "cc": "UK"},
     ])
     self.assertEqual(data["enriched"], ([], {
         "192.187.0.1": ["cc"],
         "10.15.1.255": ["asn", "cc"],
     }))
     self.assertEqual(
         len(LOGGER_mock.warning.mock_calls),
         expected_num_of_warnings)
Esempio n. 9
0
 def test__enrich__with_ip_url_given(self):
     data = self.enricher.enrich(RecordDict({"url": "http://192.168.0.1/asd"}))
     self.assertEqualIncludingTypes(data, RecordDict({
         "enriched": ([], {"192.168.0.1": ["asn", "cc", "ip"]}),
         "url": "http://192.168.0.1/asd",
         "address": [{"ip": '192.168.0.1',
                      "asn": '1234',
                      "cc": 'PL'}]}))
Esempio n. 10
0
 def test__delete_too_long_address__address_is_too_long(self):
     ips = MAX_IPS_IN_ADDRESS + 1
     parsed = RecordDict()
     parsed['id'] = '0123456789abcdef0123456789abcdef'
     parsed['address'] = [{'ip': i + 1} for i in xrange(ips)]
     expected = RecordDict()
     expected['id'] = '0123456789abcdef0123456789abcdef'
     self.meth.delete_too_long_address(parsed)
     self.assertEqual(parsed, expected)
Esempio n. 11
0
 def test__enrich__with_fqdn_given(self):
     data = self.enricher.enrich(RecordDict({"fqdn": "cert.pl"}))
     self.enricher._resolver.asert_called_once_with("cert.pl")
     self.assertEqualIncludingTypes(data, RecordDict({
         "enriched": ([], {"127.0.0.1": ["asn", "cc", "ip"]}),
         "fqdn": "cert.pl",
         "address": [{"ip": '127.0.0.1',
                      "asn": '1234',
                      "cc": 'PL'}]}))
Esempio n. 12
0
 def test__enrich__with_fqdn_given__with_nodns_flag(self):
     data = self.enricher.enrich(RecordDict({
         "fqdn": "cert.pl",
         "_do_not_resolve_fqdn_to_ip": True}))
     self.enricher._resolver.asert_called_once_with("cert.pl")
     self.assertEqualIncludingTypes(data, RecordDict({
         "enriched": ([], {}),
         "fqdn": "cert.pl",
         "_do_not_resolve_fqdn_to_ip": True}))
Esempio n. 13
0
 def test__fqdn_to_ip__not_called(self):
     """Test if fqdn_to_ip not called if address already present"""
     data = RecordDict({
         "address": [{"ip": "127.0.0.1"},
                     {"ip": "192.187.0.1"},
                     {"ip": "10.15.1.255"}]})
     data.update(self.COMMON_DATA)
     self.enricher.fqdn_to_ip = mock.MagicMock(return_value="127.0.0.1")
     self.enricher.enrich(data)
     self.assertFalse(self.enricher.fqdn_to_ip.called)
Esempio n. 14
0
 def test__enrich__with_url_given(self):
     data = self.enricher.enrich(RecordDict({"url": "http://www.nask.pl/asd"}))
     self.enricher._resolver.asert_called_once_with("www.nask.pl")
     self.assertEqualIncludingTypes(data, RecordDict({
         "enriched": (["fqdn"], {"127.0.0.1": ["asn", "cc", "ip"]}),
         "url": "http://www.nask.pl/asd",
         "fqdn": "www.nask.pl",
         "address": [{"ip": '127.0.0.1',
                      "asn": '1234',
                      "cc": 'PL'}]}))
Esempio n. 15
0
 def test__enrich__with_url_given__with_nodns_flag(self):
     data = self.enricher.enrich(RecordDict({
         "url": "http://www.nask.pl/asd",
         "_do_not_resolve_fqdn_to_ip": True}))
     self.enricher._resolver.asert_called_once_with("www.nask.pl")
     self.assertEqualIncludingTypes(data, RecordDict({
         "enriched": (["fqdn"], {}),
         "url": "http://www.nask.pl/asd",
         "fqdn": "www.nask.pl",
         "_do_not_resolve_fqdn_to_ip": True}))
Esempio n. 16
0
 def test__enrich__with_address_and_fqdn_given(self):
     data = self.enricher.enrich(RecordDict({
         "fqdn": "cert.pl",
         "address": [{"ip": "10.20.30.40"}]}))
     self.assertEqualIncludingTypes(data, RecordDict({
         "enriched": ([], {"10.20.30.40": ["asn", "cc"]}),
         "fqdn": "cert.pl",
         "address": [{"ip": '10.20.30.40',
                      "asn": '1234',
                      "cc": 'PL'}]}))
Esempio n. 17
0
 def test__enrich__with_address_and_url_given(self):
     data = self.enricher.enrich(RecordDict({
         "url": "http://www.nask.pl/asd",
         "address": [{"ip": "10.20.30.40"}]}))
     self.assertEqualIncludingTypes(data, RecordDict({
         "enriched": (["fqdn"], {"10.20.30.40": ["asn", "cc"]}),
         "url": "http://www.nask.pl/asd",
         "fqdn": "www.nask.pl",
         "address": [{"ip": '10.20.30.40',
                      "asn": '1234',
                      "cc": 'PL'}]}))
Esempio n. 18
0
 def test__ip_to_cc__called(self):
     """Test if ip_to_cc was called for all ips"""
     data = RecordDict({
         "address": [{"ip": "127.0.0.1"},
                     {"ip": "192.187.0.1"},
                     {"ip": "10.15.1.255"}]})
     data.update(self.COMMON_DATA)
     self.enricher.ip_to_cc = mock.MagicMock(return_value="")
     self.enricher.enrich(data)
     for addr in data["address"]:
         self.enricher.ip_to_cc.assert_any_call(addr["ip"])
     self.assertEqual(len(data["address"]), self.enricher.ip_to_cc.call_count)
Esempio n. 19
0
 def test__enrich__with_ip_url_given__with_nodns_flag(self):
     data = self.enricher.enrich(RecordDict({
         "url": "http://192.168.0.1/asd",
         "_do_not_resolve_fqdn_to_ip": True}))
     self.assertEqualIncludingTypes(data, RecordDict({
         # (here the '_do_not_resolve_fqdn_to_ip' flag did *not* change behaviour)
         "enriched": ([], {"192.168.0.1": ["asn", "cc", "ip"]}),
         "url": "http://192.168.0.1/asd",
         "address": [{"ip": '192.168.0.1',
                      "asn": '1234',
                      "cc": 'PL'}],
         "_do_not_resolve_fqdn_to_ip": True}))
Esempio n. 20
0
 def test__enrich__with_excluded_ips_config__with_some_ip_to_exclude__2(self):
     self.enricher._enrich_config = {'dnshost': '8.8.8.8',
                                     'dnsport': '53',
                                     'geoippath': '/usr/share/GeoIP',
                                     'excluded_ips': '127.0.0.1, 2.2.2.2, 3.3.3.3'}
     self.enricher.excluded_ips = self.enricher._get_excluded_ips()
     data = self.enricher.enrich(RecordDict({"url": "http://www.nask.pl/asd"}))
     self.enricher._resolver.asert_called_once_with("www.nask.pl")
     self.assertEqualIncludingTypes(data, RecordDict({
         "enriched": (["fqdn"], {}),
         "url": "http://www.nask.pl/asd",
         "fqdn": "www.nask.pl"}))  # (note: emptied `address` removed)
Esempio n. 21
0
 def test__enrich__with_address_and_fqdn_given__with_nodns_flag(self):
     data = self.enricher.enrich(RecordDict({
         "fqdn": "cert.pl",
         "address": [{"ip": "10.20.30.40"}],
         "_do_not_resolve_fqdn_to_ip": True}))
     self.assertEqualIncludingTypes(data, RecordDict({
         # (here the '_do_not_resolve_fqdn_to_ip' flag did *not* change behaviour)
         "enriched": ([], {"10.20.30.40": ["asn", "cc"]}),
         "fqdn": "cert.pl",
         "address": [{"ip": '10.20.30.40',
                      "asn": '1234',
                      "cc": 'PL'}],
         "_do_not_resolve_fqdn_to_ip": True}))
Esempio n. 22
0
 def test__delete_too_long_address__address_is_empty(self):
     parsed = RecordDict()
     parsed.update({'source': 'foo.bar'})
     expected = RecordDict()
     expected.update({'source': 'foo.bar'})
     self.meth.delete_too_long_address(parsed)
     self.assertEqual(parsed, expected)
Esempio n. 23
0
 def test__filter_out_excluded_ips__with_no_ip_in_excluded_ips(self):
     self.enricher.excluded_ips = iptools.IpRangeList('1.1.1.1', '2.2.2.2', '3.3.3.3')
     data = RecordDict({
         "url": "http://www.nask.pl/asd",
         "address": [{'ip': '1.1.1.5'}, {'ip': '2.1.1.1'}],
     })
     expected = RecordDict({
         "url": "http://www.nask.pl/asd",
         "address": [{'ip': '1.1.1.5'}, {'ip': '2.1.1.1'}],
     })
     ip_to_enr_mock = mock.MagicMock()
     ip_to_enr_expected_calls = []
     self.enricher._filter_out_excluded_ips(data, ip_to_enr_mock)
     self.assertEqualIncludingTypes(expected, data)
     self.assertEqual(ip_to_enr_mock.mock_calls, ip_to_enr_expected_calls)
Esempio n. 24
0
 def test__filter_out_excluded_ips__with_excluded_ips_being_None(self):
     self.enricher.excluded_ips = None
     data = RecordDict({
         "url": "http://www.nask.pl/asd",
         "address": [{'ip': "127.0.0.1"}],
     })
     expected = RecordDict({
         "url": "http://www.nask.pl/asd",
         "address": [{'ip': "127.0.0.1"}],
     })
     ip_to_enr_mock = mock.MagicMock()
     ip_to_enr_expected_calls = []
     self.enricher._filter_out_excluded_ips(data, ip_to_enr_mock)
     self.assertEqualIncludingTypes(expected, data)
     self.assertEqual(ip_to_enr_mock.mock_calls, ip_to_enr_expected_calls)
Esempio n. 25
0
 def test__enrich__with_excluded_ips_config__without_any_ip_to_exclude(self):
     self.enricher._enrich_config = {'dnshost': '8.8.8.8',
                                     'dnsport': '53',
                                     'geoippath': '/usr/share/GeoIP',
                                     'excluded_ips': '2.2.2.2, 3.3.3.3'}
     self.enricher.excluded_ips = self.enricher._get_excluded_ips()
     data = self.enricher.enrich(RecordDict({"url": "http://www.nask.pl/asd"}))
     self.enricher._resolver.asert_called_once_with("www.nask.pl")
     self.assertEqualIncludingTypes(data, RecordDict({
         "enriched": (["fqdn"], {"127.0.0.1": ["asn", "cc", "ip"]}),
         "url": "http://www.nask.pl/asd",
         "fqdn": "www.nask.pl",
         "address": [{"ip": '127.0.0.1',
                      "asn": '1234',
                      "cc": 'PL'}]}))
Esempio n. 26
0
 def actual_data_for_adding_asn_cc_if_possible(self, asn_mock, cc_mock):
     self.enricher.gi_asn.asn = asn_mock
     self.enricher.gi_cc.city = cc_mock
     return RecordDict({
         "address": [{"ip": "127.0.0.1"},
                     {"ip": "192.187.0.1"},
                     {"ip": "10.15.1.255"}]})
Esempio n. 27
0
 def actual_data_for_existing_asn_cc_always_dropped_and_new_ones_added_if_possible(self, mock_asn, mock_cc):
     self.enricher.gi_asn.asn = mock_asn
     self.enricher.gi_cc.city = mock_cc
     return RecordDict({
         "address": [{"ip": "127.0.0.1", "cc": "JP"},
                     {"ip": "192.187.0.1", "cc": "US", "asn": 424242},
                     {"ip": "10.15.1.255", "asn": 434343}]})
Esempio n. 28
0
    def test__get_client_and_urls_matched__2(self):
        body = {"category": "bots", "restriction": "public", "confidence": "medium",
                "sha1": "023a00e7c2ef04ee5b0f767ba73ee39734323432", "name": "virut",
                "proto": "tcp", "address": [{"cc": "XX", "ip": "1.1.1.1", "asn": "1"},
                                            {"cc": "XX", "ip": "100.71.83.178", "asn": "1"},
                                            {"cc": "XX", "ip": "102.71.83.178", "asn": "1"}],
                "fqdn": "domain.com", "url": "http://onet.pl", "source": "hpfeeds.dionaea",
                "time": "2013-07-01 20:37:20", "dport": "445",
                "rid": "023a00e7c2ef04ee5b0f767ba73ee397",
                "sport": "2147", "dip": "10.28.71.43", "id": "023a00e7c2ef04ee5b0f767ba73ee397"}

        # tested key:[test_value,[valid value]]
        input_data = {'ip': ['154.89.207.81', ['fdc']],
                      'cc': ['SU', ['decfba', 'fdc']],
                      'asn': ['45975', ['fdc']],
                      'fqdn': ['onetbrutalmikmokcert.eu', ['fdc']]}

        self.auth_api_mock._get_inside_criteria.return_value = TEST_CRITERIA

        for i in input_data:
            body = self.reset_body(body)
            if i == 'fqdn':
                body['fqdn'] = input_data[i][0]
            else:
                body['address'][0][i] = input_data[i][0]
            json_msg = json.dumps(body)
            record_dict = RecordDict.from_json(json_msg)
            self.assertItemsEqual(
                self.filter.get_client_and_urls_matched(record_dict, self.fqdn_only_categories),
                (input_data[i][1], {}))
Esempio n. 29
0
 def input_callback(self, routing_key, body, properties):
     data = RecordDict.from_json(body)
     with self.setting_error_event_info(data):
         enriched = self.enrich(data)
         rk = replace_segment(routing_key, 1, 'enriched')
         body = enriched.get_ready_json()
         self.publish_output(routing_key=rk, body=body)
Esempio n. 30
0
 def test_routing_key_modified(self):
     """Test if routing key after enrichement is set to "enriched.*"
     when publishing to output queue"""
     self.enricher.publish_output = mock.MagicMock()
     data = RecordDict({
         "address": [{"ip": "127.0.0.1"},
                     {"ip": "192.187.0.1"},
                     {"ip": "10.15.1.255"}]})
     data.update(self.COMMON_DATA)
     body = data.get_ready_json()
     initial_routing_key = "event.parsed.test.test-source"
     properties = None
     self.enricher.input_callback(initial_routing_key, body, properties)
     args, kwargs = self.enricher.publish_output.call_args
     self.assertIn("routing_key", kwargs)
     self.assertEqual(kwargs["routing_key"], "event.enriched.test.test-source")