def test_bad_domains(self): self._blacklist_data['blacklist_is_domains'] = True with patch('logging.warning', autospec=True) as patched_logging_warning: blacklist = create_blacklist(self._blacklist_data) T.assert_equal(4, patched_logging_warning.call_count) calls = [ call( 'Blacklisted value "apple" cannot be resolved as a domain name' ), call( 'Blacklisted value "banana" cannot be resolved as a domain name' ), call( 'Blacklisted value "corolla" cannot be resolved as a domain name' ), call( 'Blacklisted value "datsun" cannot be resolved as a domain name' ), ] T.assert_equal(calls, patched_logging_warning.call_args_list) blob = {'fruit_name': 'apple.com'} T.assert_equal(bool(blacklist.match_line(blob)), False)
def __init__(self, initial_domains=None, initial_ips=None, generations=2, related_when=None): """Initializes the RelatedDomainsFilter. Args: initial_domains: an enumerable of string domain names initial_ips: an enumerable of string IPs in the form '' generations: How many generations of related domains to retrieve. Passing 1 means just find the domains related to the initial input. Passing 2 means also find the domains related to the domains related to the initial input. related_when: A boolean function to call to decide whether to add the domains from a line to the list of related domains. """ super(RelatedDomainsFilter, self).__init__() self._whitelist = create_blacklist(config_get_deep('domain_whitelist')) cache_file_name = config_get_deep('opendns.RelatedDomainsFilter.cache_file_name', None) self._investigate = InvestigateApi(config_get_deep('api_key.opendns'), cache_file_name=cache_file_name) self._initial_domains = set(initial_domains) if initial_domains else set() self._initial_ips = set(initial_ips) if initial_ips else set() self._related_domains = set(initial_domains) if initial_domains else set() self._related_when = related_when self._generations = generations self._all_blobs = list()
def __init__(self, lookup_when=None, **kwargs): super(LookupDomainsFilter, self).__init__('osxcollector_domains', 'osxcollector_opendns', lookup_when=lookup_when, name_of_api_key='opendns', **kwargs) self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
def test_match_fruit_regex(self): good_blobs = [ { 'fruit_name': 'apple' }, ] bad_blobs = [ { 'fruit_name': 'banana' }, { 'car_name': 'corolla' }, { 'car_name': 'datsun' }, ] self._blacklist_data['blacklist_is_regex'] = True self._file_contents.return_value = ['app.*', 'ban.+org'] blacklist = create_blacklist(self._blacklist_data) for blob in good_blobs: T.assert_equal(bool(blacklist.match_line(blob)), True) for blob in bad_blobs: T.assert_equal(bool(blacklist.match_line(blob)), False)
def __init__(self, initial_domains=None, initial_ips=None, generations=2, related_when=None): """Initializes the RelatedDomainsFilter. Args: initial_domains: an enumerable of string domain names initial_ips: an enumerable of string IPs in the form '' generations: How many generations of related domains to retrieve. Passing 1 means just find the domains related to the initial input. Passing 2 means also find the domains related to the domains related to the initial input. related_when: A boolean function to call to decide whether to add the domains from a line to the list of related domains. """ super(RelatedDomainsFilter, self).__init__() self._whitelist = create_blacklist(config_get_deep('domain_whitelist')) cache_file_name = config_get_deep('opendns.RelatedDomainsFilter.cache_file_name', None) self._investigate = InvestigateApi(config_get_deep('api_key.opendns'), cache_file_name=cache_file_name) self._initial_domains = set(initial_domains) if initial_domains else set() self._initial_ips = set(initial_ips) if initial_ips else set() self._related_domains = set(initial_domains) if initial_domains else set() self._related_when = related_when self._generations = generations self._all_blobs = list()
def test_match_domains(self, blacklist_data, file_contents): good_blobs = [ { 'fruit_name': 'apple.com' }, { 'fruit_name': 'www.apple.com' }, { 'fruit_name': 'www.another-thing.apple.com' }, ] bad_blobs = [ { 'fruit_name': 'cran-apple.com' }, { 'fruit_name': 'apple.org' }, { 'fruit_name': 'apple.com.jp' }, { 'car_name': 'apple.com' }, ] blacklist_data['blacklist_is_domains'] = True file_contents.return_value = ['apple.com'] blacklist = create_blacklist(blacklist_data) for blob in good_blobs: assert blacklist.match_line(blob) for blob in bad_blobs: assert not blacklist.match_line(blob)
def test_match_fruit_regex(self, blacklist_data, file_contents): good_blobs = [ { 'fruit_name': 'apple' }, ] bad_blobs = [ { 'fruit_name': 'banana' }, { 'car_name': 'corolla' }, { 'car_name': 'datsun' }, ] blacklist_data['blacklist_is_regex'] = True file_contents.return_value = ['app.*', 'ban.+org'] blacklist = create_blacklist(blacklist_data) for blob in good_blobs: assert blacklist.match_line(blob) for blob in bad_blobs: assert not blacklist.match_line(blob)
def test_match_domains(self): good_blobs = [ { 'fruit_name': 'apple.com' }, { 'fruit_name': 'www.apple.com' }, { 'fruit_name': 'www.another-thing.apple.com' }, ] bad_blobs = [ { 'fruit_name': 'cran-apple.com' }, { 'fruit_name': 'apple.org' }, { 'fruit_name': 'apple.com.jp' }, { 'car_name': 'apple.com' }, ] self._blacklist_data['blacklist_is_domains'] = True self._file_contents.return_value = ['apple.com'] blacklist = create_blacklist(self._blacklist_data) for blob in good_blobs: T.assert_equal(bool(blacklist.match_line(blob)), True) for blob in bad_blobs: T.assert_equal(bool(blacklist.match_line(blob)), False)
def test_only_required_keys(self): blacklist = create_blacklist(self._blacklist_data) T.assert_equal(blacklist.name, self._blacklist_data['blacklist_name']) T.assert_equal(blacklist._blacklisted_keys, self._blacklist_data['blacklist_keys']) T.assert_equal(blacklist._is_regex, False) T.assert_equal(blacklist._is_domains, False)
def test_is_domains(self, blacklist_data, file_contents): file_contents.return_value = ['apple.com', 'banana.org'] # Setting 'blacklist_is_domains' overrides 'blacklist_is_regex' blacklist_data['blacklist_is_domains'] = True blacklist_data['blacklist_is_regex'] = False blacklist = create_blacklist(blacklist_data) assert blacklist._is_regex assert blacklist._is_domains
def __init__(self, lookup_when=None, **kwargs): super(LookupDomainsFilter, self).__init__( 'osxcollector_domains', 'osxcollector_vtdomain', lookup_when=lookup_when, name_of_api_key='virustotal', **kwargs ) self._whitelist = create_blacklist( config_get_deep('domain_whitelist'), kwargs.get('data_feeds', {}), )
def test_is_domains(self): self._file_contents.return_value = ['apple.com', 'banana.org'] # Setting 'blacklist_is_domains' overrides 'blacklist_is_regex' self._blacklist_data['blacklist_is_domains'] = True self._blacklist_data['blacklist_is_regex'] = False blacklist = create_blacklist(self._blacklist_data) T.assert_equal(blacklist._is_regex, True) T.assert_equal(blacklist._is_domains, True)
def test_is_domains(self): self._file_contents.return_value = ['apple.com', 'banana.org'] # Setting 'blacklist_is_domains' overrides 'blacklist_is_regex' self._blacklist_data['blacklist_is_domains'] = True self._blacklist_data['blacklist_is_regex'] = False blacklist = create_blacklist(self._blacklist_data) T.assert_equal(blacklist._is_regex, True) T.assert_equal(blacklist._is_domains, True)
def test_match_fruit_and_cars(self): good_blobs = [ {'fruit_name': 'apple'}, {'fruit_name': 'banana'}, {'car_name': 'corolla'}, {'car_name': 'datsun'}, ] self._blacklist_data['blacklist_keys'] = ['fruit_name', 'car_name'] blacklist = create_blacklist(self._blacklist_data) for blob in good_blobs: T.assert_equal(bool(blacklist.match_line(blob)), True)
def test_bad_domains_unicode(self, blacklist_data): unicode_domain_1 = 'yelp.公司' unicode_domain_2 = 'www.Yülp.tld' unicode_domain_3 = 'иelф.р' unicode_domains = [ unicode_domain_1, unicode_domain_2, unicode_domain_3 ] blacklist_data['blacklist_is_domains'] = True with patch( 'osxcollector.output_filters.util.blacklist._read_blacklist_file', return_value=unicode_domains, ): with patch('logging.warning', autospec=True) as patched_logging_warning: create_blacklist(blacklist_data) assert patched_logging_warning.call_count == 3 calls = [ call( u'Blacklisted value "{0}" cannot be resolved as a domain name'. format(unicode_domain), ) for unicode_domain in unicode_domains ] assert calls == patched_logging_warning.call_args_list
def test_bad_domains(self): self._blacklist_data['blacklist_is_domains'] = True with patch('logging.warning', autospec=True) as patched_logging_warning: blacklist = create_blacklist(self._blacklist_data) T.assert_equal(4, patched_logging_warning.call_count) calls = [ call('Blacklisted value "apple" cannot be resolved as a domain name'), call('Blacklisted value "banana" cannot be resolved as a domain name'), call('Blacklisted value "corolla" cannot be resolved as a domain name'), call('Blacklisted value "datsun" cannot be resolved as a domain name'), ] T.assert_equal(calls, patched_logging_warning.call_args_list) blob = {'fruit_name': 'apple.com'} T.assert_equal(bool(blacklist.match_line(blob)), False)
def test_match_fruit(self): good_blobs = [ {'fruit_name': 'apple'}, {'fruit_name': 'banana'}, ] bad_blobs = [ {'car_name': 'corolla'}, {'car_name': 'datsun'}, ] blacklist = create_blacklist(self._blacklist_data) for blob in good_blobs: T.assert_equal(bool(blacklist.match_line(blob)), True) for blob in bad_blobs: T.assert_equal(bool(blacklist.match_line(blob)), False)
def test_match_fruit_regex(self): good_blobs = [ {'fruit_name': 'apple'}, ] bad_blobs = [ {'fruit_name': 'banana'}, {'car_name': 'corolla'}, {'car_name': 'datsun'}, ] self._blacklist_data['blacklist_is_regex'] = True self._file_contents.return_value = ['app.*', 'ban.+org'] blacklist = create_blacklist(self._blacklist_data) for blob in good_blobs: T.assert_equal(bool(blacklist.match_line(blob)), True) for blob in bad_blobs: T.assert_equal(bool(blacklist.match_line(blob)), False)
def test_match_domains_data_feed(self, blacklist_data): good_blobs = [ { 'fruit_name': 'apple.com' }, { 'fruit_name': 'www.apple.com' }, { 'fruit_name': 'www.another-thing.apple.com' }, ] bad_blobs = [ { 'fruit_name': 'cran-apple.com' }, { 'fruit_name': 'apple.org' }, { 'fruit_name': 'apple.com.jp' }, { 'car_name': 'apple.com' }, ] blacklist_data['blacklist_is_domains'] = True blacklist_data['blacklist_data_feed'] = 'domain_list' blacklist_data.pop('blacklist_file_path') def mock_generator(): for domain in ['apple.com']: yield domain blacklist = create_blacklist( blacklist_data, {'domain_list': mock_generator}, ) for blob in good_blobs: assert blacklist.match_line(blob) for blob in bad_blobs: assert not blacklist.match_line(blob)
def test_match_domains(self): good_blobs = [ {'fruit_name': 'apple.com'}, {'fruit_name': 'www.apple.com'}, {'fruit_name': 'www.another-thing.apple.com'}, ] bad_blobs = [ {'fruit_name': 'cran-apple.com'}, {'fruit_name': 'apple.org'}, {'fruit_name': 'apple.com.jp'}, {'car_name': 'apple.com'}, ] self._blacklist_data['blacklist_is_domains'] = True self._file_contents.return_value = ['apple.com'] blacklist = create_blacklist(self._blacklist_data) for blob in good_blobs: T.assert_equal(bool(blacklist.match_line(blob)), True) for blob in bad_blobs: T.assert_equal(bool(blacklist.match_line(blob)), False)
def test_match_fruit_and_cars(self): good_blobs = [ { 'fruit_name': 'apple' }, { 'fruit_name': 'banana' }, { 'car_name': 'corolla' }, { 'car_name': 'datsun' }, ] self._blacklist_data['blacklist_keys'] = ['fruit_name', 'car_name'] blacklist = create_blacklist(self._blacklist_data) for blob in good_blobs: T.assert_equal(bool(blacklist.match_line(blob)), True)
def test_match_fruit_and_cars(self, blacklist_data): good_blobs = [ { 'fruit_name': 'apple' }, { 'fruit_name': 'banana' }, { 'car_name': 'corolla' }, { 'car_name': 'datsun' }, ] blacklist_data['blacklist_keys'] = ['fruit_name', 'car_name'] blacklist = create_blacklist(blacklist_data) for blob in good_blobs: assert blacklist.match_line(blob)
def test_match_fruit(self, blacklist_data): good_blobs = [ { 'fruit_name': 'apple' }, { 'fruit_name': 'banana' }, ] bad_blobs = [ { 'car_name': 'corolla' }, { 'car_name': 'datsun' }, ] blacklist = create_blacklist(blacklist_data) for blob in good_blobs: assert blacklist.match_line(blob) for blob in bad_blobs: assert not blacklist.match_line(blob)
def test_log_unicode_domain(self): config_chunk = { 'blacklist_name': 'Unicode domain', 'blacklist_keys': ['visited_domain'], 'blacklist_file_path': 'not_really_a_blacklist.txt', 'blacklist_is_domains': True, } file_contents = ['Bücher.tld', 'yelp.公司', 'www.Yülp.tld', 'иelф.р'] with patch( 'osxcollector.output_filters.util.blacklist._read_blacklist_file', return_value=file_contents, ), patch('logging.warning', autospec=True) as patched_logging_warning: blacklist = create_blacklist(config_chunk) assert patched_logging_warning.call_count == 4 calls = [ call( u'Blacklisted value "{0}" cannot be resolved as a domain name' .format(domain), ) for domain in file_contents ] assert calls == patched_logging_warning.call_args_list blob = {'visted_domain': 'Bücher.tld'} assert not blacklist.match_line(blob)
def test_bad_domains(self, blacklist_data): blacklist_data['blacklist_is_domains'] = True with patch('logging.warning', autospec=True) as patched_logging_warning: blacklist = create_blacklist(blacklist_data) assert patched_logging_warning.call_count == 4 calls = [ call( 'Blacklisted value "apple" cannot be resolved as a domain name' ), call( 'Blacklisted value "banana" cannot be resolved as a domain name' ), call( 'Blacklisted value "corolla" cannot be resolved as a domain name' ), call( 'Blacklisted value "datsun" cannot be resolved as a domain name' ), ] assert calls == patched_logging_warning.call_args_list blob = {'fruit_name': 'apple.com'} assert not blacklist.match_line(blob)
def test_match_fruit(self): good_blobs = [ { 'fruit_name': 'apple' }, { 'fruit_name': 'banana' }, ] bad_blobs = [ { 'car_name': 'corolla' }, { 'car_name': 'datsun' }, ] blacklist = create_blacklist(self._blacklist_data) for blob in good_blobs: T.assert_equal(bool(blacklist.match_line(blob)), True) for blob in bad_blobs: T.assert_equal(bool(blacklist.match_line(blob)), False)
def test_keys_not_list(self): self._blacklist_data['blacklist_keys'] = 'fruit_name' with T.assert_raises(MissingConfigError): create_blacklist(self._blacklist_data)
def test_missing_required_keys(self): for key in self._blacklist_data.keys(): blacklist_data = deepcopy(self._blacklist_data) del blacklist_data[key] with T.assert_raises(MissingConfigError): create_blacklist(blacklist_data)
def test_required_with_two_keys(self): self._blacklist_data['blacklist_keys'] = ['fruit_name', 'car_name'] blacklist = create_blacklist(self._blacklist_data) T.assert_equal(blacklist._blacklisted_keys, self._blacklist_data['blacklist_keys'])
def test_is_regex(self, blacklist_data): blacklist_data['blacklist_is_regex'] = True blacklist = create_blacklist(blacklist_data) assert blacklist._is_regex
def _init_blacklists(self): """Reads the config and builds a list of blacklists.""" return [create_blacklist(config_chunk) for config_chunk in config_get_deep('blacklists')]
def test_required_with_two_keys(self): self._blacklist_data['blacklist_keys'] = ['fruit_name', 'car_name'] blacklist = create_blacklist(self._blacklist_data) T.assert_equal(blacklist._blacklisted_keys, self._blacklist_data['blacklist_keys'])
def _init_blacklists(self): """Reads the config and builds a list of blacklists.""" return [ create_blacklist(config_chunk) for config_chunk in config_get_deep('blacklists') ]
def test_missing_required_keys(self, blacklist_data): for key in blacklist_data: _blacklist_data = deepcopy(blacklist_data) del _blacklist_data[key] with pytest.raises(MissingConfigError): create_blacklist(_blacklist_data)
def test_only_required_keys(self): blacklist = create_blacklist(self._blacklist_data) T.assert_equal(blacklist.name, self._blacklist_data['blacklist_name']) T.assert_equal(blacklist._blacklisted_keys, self._blacklist_data['blacklist_keys']) T.assert_equal(blacklist._is_regex, False) T.assert_equal(blacklist._is_domains, False)
def test_bad_domains(self): self._blacklist_data['blacklist_is_domains'] = True with T.assert_raises(BadDomainError): create_blacklist(self._blacklist_data)
def __init__(self, lookup_when=None, **kwargs): super(LookupDomainsFilter, self).__init__('osxcollector_domains', 'osxcollector_vtdomain', lookup_when=lookup_when, name_of_api_key='virustotal', **kwargs) self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
def test_required_with_two_keys(self, blacklist_data): blacklist_data['blacklist_keys'] = ['fruit_name', 'car_name'] blacklist = create_blacklist(blacklist_data) assert blacklist._blacklisted_keys == blacklist_data['blacklist_keys']
def test_is_regex(self): self._blacklist_data['blacklist_is_regex'] = True blacklist = create_blacklist(self._blacklist_data) T.assert_equal(blacklist._is_regex, True)
def test_keys_not_list(self): self._blacklist_data['blacklist_keys'] = 'fruit_name' with T.assert_raises(MissingConfigError): create_blacklist(self._blacklist_data)
def test_is_regex(self): self._blacklist_data['blacklist_is_regex'] = True blacklist = create_blacklist(self._blacklist_data) T.assert_equal(blacklist._is_regex, True)
def test_keys_not_list(self, blacklist_data): blacklist_data['blacklist_keys'] = 'fruit_name' with pytest.raises(MissingConfigError): create_blacklist(blacklist_data)
def test_bad_domains(self): self._blacklist_data['blacklist_is_domains'] = True with T.assert_raises(BadDomainError): create_blacklist(self._blacklist_data)
def test_missing_required_keys(self): for key in self._blacklist_data.keys(): blacklist_data = deepcopy(self._blacklist_data) del blacklist_data[key] with T.assert_raises(MissingConfigError): create_blacklist(blacklist_data)
def test_missing_data_input(self, blacklist_data): blacklist_data.pop('blacklist_file_path') with pytest.raises(MissingConfigError): create_blacklist(blacklist_data)
def __init__(self, lookup_when=None): super(LookupDomainsFilter, self).__init__('osxcollector_domains', 'osxcollector_opendns', lookup_when=lookup_when, name_of_api_key='opendns') self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))