def test_bad_domains(self):
self._blacklist_data['blacklist_is_domains'] = True
with patch('logging.warning',
autospec=True) as patched_logging_warning:
blacklist = create_blacklist(self._blacklist_data)
T.assert_equal(4, patched_logging_warning.call_count)
calls = [
call(
'Blacklisted value "apple" cannot be resolved as a domain name'
),
call(
'Blacklisted value "banana" cannot be resolved as a domain name'
),
call(
'Blacklisted value "corolla" cannot be resolved as a domain name'
),
call(
'Blacklisted value "datsun" cannot be resolved as a domain name'
),
]
T.assert_equal(calls, patched_logging_warning.call_args_list)
blob = {'fruit_name': 'apple.com'}
T.assert_equal(bool(blacklist.match_line(blob)), False)
def __init__(self, initial_domains=None, initial_ips=None, generations=2, related_when=None):
"""Initializes the RelatedDomainsFilter.
Args:
initial_domains: an enumerable of string domain names
initial_ips: an enumerable of string IPs in the form ''
generations: How many generations of related domains to retrieve. Passing 1
means just find the domains related to the initial input. Passing 2 means also find the
domains related to the domains related to the initial input.
related_when: A boolean function to call to decide whether to add the domains from a line to
the list of related domains.
"""
super(RelatedDomainsFilter, self).__init__()
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
cache_file_name = config_get_deep('opendns.RelatedDomainsFilter.cache_file_name', None)
self._investigate = InvestigateApi(config_get_deep('api_key.opendns'), cache_file_name=cache_file_name)
self._initial_domains = set(initial_domains) if initial_domains else set()
self._initial_ips = set(initial_ips) if initial_ips else set()
self._related_domains = set(initial_domains) if initial_domains else set()
self._related_when = related_when
self._generations = generations
self._all_blobs = list()
def __init__(self, lookup_when=None, **kwargs):
super(LookupDomainsFilter, self).__init__('osxcollector_domains',
'osxcollector_opendns',
lookup_when=lookup_when,
name_of_api_key='opendns',
**kwargs)
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
def test_match_fruit_regex(self):
good_blobs = [
{
'fruit_name': 'apple'
},
]
bad_blobs = [
{
'fruit_name': 'banana'
},
{
'car_name': 'corolla'
},
{
'car_name': 'datsun'
},
]
self._blacklist_data['blacklist_is_regex'] = True
self._file_contents.return_value = ['app.*', 'ban.+org']
blacklist = create_blacklist(self._blacklist_data)
for blob in good_blobs:
T.assert_equal(bool(blacklist.match_line(blob)), True)
for blob in bad_blobs:
T.assert_equal(bool(blacklist.match_line(blob)), False)
def __init__(self, initial_domains=None, initial_ips=None, generations=2, related_when=None):
"""Initializes the RelatedDomainsFilter.
Args:
initial_domains: an enumerable of string domain names
initial_ips: an enumerable of string IPs in the form ''
generations: How many generations of related domains to retrieve. Passing 1
means just find the domains related to the initial input. Passing 2 means also find the
domains related to the domains related to the initial input.
related_when: A boolean function to call to decide whether to add the domains from a line to
the list of related domains.
"""
super(RelatedDomainsFilter, self).__init__()
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
cache_file_name = config_get_deep('opendns.RelatedDomainsFilter.cache_file_name', None)
self._investigate = InvestigateApi(config_get_deep('api_key.opendns'), cache_file_name=cache_file_name)
self._initial_domains = set(initial_domains) if initial_domains else set()
self._initial_ips = set(initial_ips) if initial_ips else set()
self._related_domains = set(initial_domains) if initial_domains else set()
self._related_when = related_when
self._generations = generations
self._all_blobs = list()
def test_match_domains(self, blacklist_data, file_contents):
good_blobs = [
{
'fruit_name': 'apple.com'
},
{
'fruit_name': 'www.apple.com'
},
{
'fruit_name': 'www.another-thing.apple.com'
},
]
bad_blobs = [
{
'fruit_name': 'cran-apple.com'
},
{
'fruit_name': 'apple.org'
},
{
'fruit_name': 'apple.com.jp'
},
{
'car_name': 'apple.com'
},
]
blacklist_data['blacklist_is_domains'] = True
file_contents.return_value = ['apple.com']
blacklist = create_blacklist(blacklist_data)
for blob in good_blobs:
assert blacklist.match_line(blob)
for blob in bad_blobs:
assert not blacklist.match_line(blob)
def test_match_fruit_regex(self, blacklist_data, file_contents):
good_blobs = [
{
'fruit_name': 'apple'
},
]
bad_blobs = [
{
'fruit_name': 'banana'
},
{
'car_name': 'corolla'
},
{
'car_name': 'datsun'
},
]
blacklist_data['blacklist_is_regex'] = True
file_contents.return_value = ['app.*', 'ban.+org']
blacklist = create_blacklist(blacklist_data)
for blob in good_blobs:
assert blacklist.match_line(blob)
for blob in bad_blobs:
assert not blacklist.match_line(blob)
def test_match_domains(self):
good_blobs = [
{
'fruit_name': 'apple.com'
},
{
'fruit_name': 'www.apple.com'
},
{
'fruit_name': 'www.another-thing.apple.com'
},
]
bad_blobs = [
{
'fruit_name': 'cran-apple.com'
},
{
'fruit_name': 'apple.org'
},
{
'fruit_name': 'apple.com.jp'
},
{
'car_name': 'apple.com'
},
]
self._blacklist_data['blacklist_is_domains'] = True
self._file_contents.return_value = ['apple.com']
blacklist = create_blacklist(self._blacklist_data)
for blob in good_blobs:
T.assert_equal(bool(blacklist.match_line(blob)), True)
for blob in bad_blobs:
T.assert_equal(bool(blacklist.match_line(blob)), False)
def test_only_required_keys(self):
blacklist = create_blacklist(self._blacklist_data)
T.assert_equal(blacklist.name, self._blacklist_data['blacklist_name'])
T.assert_equal(blacklist._blacklisted_keys,
self._blacklist_data['blacklist_keys'])
T.assert_equal(blacklist._is_regex, False)
T.assert_equal(blacklist._is_domains, False)
def test_is_domains(self, blacklist_data, file_contents):
file_contents.return_value = ['apple.com', 'banana.org']
# Setting 'blacklist_is_domains' overrides 'blacklist_is_regex'
blacklist_data['blacklist_is_domains'] = True
blacklist_data['blacklist_is_regex'] = False
blacklist = create_blacklist(blacklist_data)
assert blacklist._is_regex
assert blacklist._is_domains
def __init__(self, lookup_when=None, **kwargs):
super(LookupDomainsFilter, self).__init__(
'osxcollector_domains', 'osxcollector_vtdomain',
lookup_when=lookup_when, name_of_api_key='virustotal', **kwargs
)
self._whitelist = create_blacklist(
config_get_deep('domain_whitelist'), kwargs.get('data_feeds', {}),
)
def test_is_domains(self):
self._file_contents.return_value = ['apple.com', 'banana.org']
# Setting 'blacklist_is_domains' overrides 'blacklist_is_regex'
self._blacklist_data['blacklist_is_domains'] = True
self._blacklist_data['blacklist_is_regex'] = False
blacklist = create_blacklist(self._blacklist_data)
T.assert_equal(blacklist._is_regex, True)
T.assert_equal(blacklist._is_domains, True)
def test_is_domains(self):
self._file_contents.return_value = ['apple.com', 'banana.org']
# Setting 'blacklist_is_domains' overrides 'blacklist_is_regex'
self._blacklist_data['blacklist_is_domains'] = True
self._blacklist_data['blacklist_is_regex'] = False
blacklist = create_blacklist(self._blacklist_data)
T.assert_equal(blacklist._is_regex, True)
T.assert_equal(blacklist._is_domains, True)
def test_match_fruit_and_cars(self):
good_blobs = [
{'fruit_name': 'apple'},
{'fruit_name': 'banana'},
{'car_name': 'corolla'},
{'car_name': 'datsun'},
]
self._blacklist_data['blacklist_keys'] = ['fruit_name', 'car_name']
blacklist = create_blacklist(self._blacklist_data)
for blob in good_blobs:
T.assert_equal(bool(blacklist.match_line(blob)), True)
def test_bad_domains_unicode(self, blacklist_data):
unicode_domain_1 = 'yelp.公司'
unicode_domain_2 = 'www.Yülp.tld'
unicode_domain_3 = 'иelф.р'
unicode_domains = [
unicode_domain_1, unicode_domain_2, unicode_domain_3
]
blacklist_data['blacklist_is_domains'] = True
with patch(
'osxcollector.output_filters.util.blacklist._read_blacklist_file',
return_value=unicode_domains,
):
with patch('logging.warning',
autospec=True) as patched_logging_warning:
create_blacklist(blacklist_data)
assert patched_logging_warning.call_count == 3
calls = [
call(
u'Blacklisted value "{0}" cannot be resolved as a domain name'.
format(unicode_domain), ) for unicode_domain in unicode_domains
]
assert calls == patched_logging_warning.call_args_list
def test_bad_domains(self):
self._blacklist_data['blacklist_is_domains'] = True
with patch('logging.warning', autospec=True) as patched_logging_warning:
blacklist = create_blacklist(self._blacklist_data)
T.assert_equal(4, patched_logging_warning.call_count)
calls = [
call('Blacklisted value "apple" cannot be resolved as a domain name'),
call('Blacklisted value "banana" cannot be resolved as a domain name'),
call('Blacklisted value "corolla" cannot be resolved as a domain name'),
call('Blacklisted value "datsun" cannot be resolved as a domain name'),
]
T.assert_equal(calls, patched_logging_warning.call_args_list)
blob = {'fruit_name': 'apple.com'}
T.assert_equal(bool(blacklist.match_line(blob)), False)
def test_match_fruit(self):
good_blobs = [
{'fruit_name': 'apple'},
{'fruit_name': 'banana'},
]
bad_blobs = [
{'car_name': 'corolla'},
{'car_name': 'datsun'},
]
blacklist = create_blacklist(self._blacklist_data)
for blob in good_blobs:
T.assert_equal(bool(blacklist.match_line(blob)), True)
for blob in bad_blobs:
T.assert_equal(bool(blacklist.match_line(blob)), False)
def test_match_fruit_regex(self):
good_blobs = [
{'fruit_name': 'apple'},
]
bad_blobs = [
{'fruit_name': 'banana'},
{'car_name': 'corolla'},
{'car_name': 'datsun'},
]
self._blacklist_data['blacklist_is_regex'] = True
self._file_contents.return_value = ['app.*', 'ban.+org']
blacklist = create_blacklist(self._blacklist_data)
for blob in good_blobs:
T.assert_equal(bool(blacklist.match_line(blob)), True)
for blob in bad_blobs:
T.assert_equal(bool(blacklist.match_line(blob)), False)
def test_match_domains_data_feed(self, blacklist_data):
good_blobs = [
{
'fruit_name': 'apple.com'
},
{
'fruit_name': 'www.apple.com'
},
{
'fruit_name': 'www.another-thing.apple.com'
},
]
bad_blobs = [
{
'fruit_name': 'cran-apple.com'
},
{
'fruit_name': 'apple.org'
},
{
'fruit_name': 'apple.com.jp'
},
{
'car_name': 'apple.com'
},
]
blacklist_data['blacklist_is_domains'] = True
blacklist_data['blacklist_data_feed'] = 'domain_list'
blacklist_data.pop('blacklist_file_path')
def mock_generator():
for domain in ['apple.com']:
yield domain
blacklist = create_blacklist(
blacklist_data,
{'domain_list': mock_generator},
)
for blob in good_blobs:
assert blacklist.match_line(blob)
for blob in bad_blobs:
assert not blacklist.match_line(blob)
def test_match_domains(self):
good_blobs = [
{'fruit_name': 'apple.com'},
{'fruit_name': 'www.apple.com'},
{'fruit_name': 'www.another-thing.apple.com'},
]
bad_blobs = [
{'fruit_name': 'cran-apple.com'},
{'fruit_name': 'apple.org'},
{'fruit_name': 'apple.com.jp'},
{'car_name': 'apple.com'},
]
self._blacklist_data['blacklist_is_domains'] = True
self._file_contents.return_value = ['apple.com']
blacklist = create_blacklist(self._blacklist_data)
for blob in good_blobs:
T.assert_equal(bool(blacklist.match_line(blob)), True)
for blob in bad_blobs:
T.assert_equal(bool(blacklist.match_line(blob)), False)
def test_match_fruit_and_cars(self):
good_blobs = [
{
'fruit_name': 'apple'
},
{
'fruit_name': 'banana'
},
{
'car_name': 'corolla'
},
{
'car_name': 'datsun'
},
]
self._blacklist_data['blacklist_keys'] = ['fruit_name', 'car_name']
blacklist = create_blacklist(self._blacklist_data)
for blob in good_blobs:
T.assert_equal(bool(blacklist.match_line(blob)), True)
def test_match_fruit_and_cars(self, blacklist_data):
good_blobs = [
{
'fruit_name': 'apple'
},
{
'fruit_name': 'banana'
},
{
'car_name': 'corolla'
},
{
'car_name': 'datsun'
},
]
blacklist_data['blacklist_keys'] = ['fruit_name', 'car_name']
blacklist = create_blacklist(blacklist_data)
for blob in good_blobs:
assert blacklist.match_line(blob)
def test_match_fruit(self, blacklist_data):
good_blobs = [
{
'fruit_name': 'apple'
},
{
'fruit_name': 'banana'
},
]
bad_blobs = [
{
'car_name': 'corolla'
},
{
'car_name': 'datsun'
},
]
blacklist = create_blacklist(blacklist_data)
for blob in good_blobs:
assert blacklist.match_line(blob)
for blob in bad_blobs:
assert not blacklist.match_line(blob)
def test_log_unicode_domain(self):
config_chunk = {
'blacklist_name': 'Unicode domain',
'blacklist_keys': ['visited_domain'],
'blacklist_file_path': 'not_really_a_blacklist.txt',
'blacklist_is_domains': True,
}
file_contents = ['Bücher.tld', 'yelp.公司', 'www.Yülp.tld', 'иelф.р']
with patch(
'osxcollector.output_filters.util.blacklist._read_blacklist_file',
return_value=file_contents,
), patch('logging.warning', autospec=True) as patched_logging_warning:
blacklist = create_blacklist(config_chunk)
assert patched_logging_warning.call_count == 4
calls = [
call(
u'Blacklisted value "{0}" cannot be resolved as a domain name'
.format(domain), ) for domain in file_contents
]
assert calls == patched_logging_warning.call_args_list
blob = {'visted_domain': 'Bücher.tld'}
assert not blacklist.match_line(blob)
def test_bad_domains(self, blacklist_data):
blacklist_data['blacklist_is_domains'] = True
with patch('logging.warning',
autospec=True) as patched_logging_warning:
blacklist = create_blacklist(blacklist_data)
assert patched_logging_warning.call_count == 4
calls = [
call(
'Blacklisted value "apple" cannot be resolved as a domain name'
),
call(
'Blacklisted value "banana" cannot be resolved as a domain name'
),
call(
'Blacklisted value "corolla" cannot be resolved as a domain name'
),
call(
'Blacklisted value "datsun" cannot be resolved as a domain name'
),
]
assert calls == patched_logging_warning.call_args_list
blob = {'fruit_name': 'apple.com'}
assert not blacklist.match_line(blob)
def test_match_fruit(self):
good_blobs = [
{
'fruit_name': 'apple'
},
{
'fruit_name': 'banana'
},
]
bad_blobs = [
{
'car_name': 'corolla'
},
{
'car_name': 'datsun'
},
]
blacklist = create_blacklist(self._blacklist_data)
for blob in good_blobs:
T.assert_equal(bool(blacklist.match_line(blob)), True)
for blob in bad_blobs:
T.assert_equal(bool(blacklist.match_line(blob)), False)
def test_keys_not_list(self):
self._blacklist_data['blacklist_keys'] = 'fruit_name'
with T.assert_raises(MissingConfigError):
create_blacklist(self._blacklist_data)
def test_missing_required_keys(self):
for key in self._blacklist_data.keys():
blacklist_data = deepcopy(self._blacklist_data)
del blacklist_data[key]
with T.assert_raises(MissingConfigError):
create_blacklist(blacklist_data)
def test_required_with_two_keys(self):
self._blacklist_data['blacklist_keys'] = ['fruit_name', 'car_name']
blacklist = create_blacklist(self._blacklist_data)
T.assert_equal(blacklist._blacklisted_keys,
self._blacklist_data['blacklist_keys'])
def test_is_regex(self, blacklist_data):
blacklist_data['blacklist_is_regex'] = True
blacklist = create_blacklist(blacklist_data)
assert blacklist._is_regex
def _init_blacklists(self):
"""Reads the config and builds a list of blacklists."""
return [create_blacklist(config_chunk) for config_chunk in config_get_deep('blacklists')]
def test_required_with_two_keys(self):
self._blacklist_data['blacklist_keys'] = ['fruit_name', 'car_name']
blacklist = create_blacklist(self._blacklist_data)
T.assert_equal(blacklist._blacklisted_keys, self._blacklist_data['blacklist_keys'])
def _init_blacklists(self):
"""Reads the config and builds a list of blacklists."""
return [
create_blacklist(config_chunk)
for config_chunk in config_get_deep('blacklists')
]
def test_missing_required_keys(self, blacklist_data):
for key in blacklist_data:
_blacklist_data = deepcopy(blacklist_data)
del _blacklist_data[key]
with pytest.raises(MissingConfigError):
create_blacklist(_blacklist_data)
def test_only_required_keys(self):
blacklist = create_blacklist(self._blacklist_data)
T.assert_equal(blacklist.name, self._blacklist_data['blacklist_name'])
T.assert_equal(blacklist._blacklisted_keys, self._blacklist_data['blacklist_keys'])
T.assert_equal(blacklist._is_regex, False)
T.assert_equal(blacklist._is_domains, False)
def test_bad_domains(self):
self._blacklist_data['blacklist_is_domains'] = True
with T.assert_raises(BadDomainError):
create_blacklist(self._blacklist_data)
def __init__(self, lookup_when=None, **kwargs):
super(LookupDomainsFilter, self).__init__('osxcollector_domains', 'osxcollector_vtdomain',
lookup_when=lookup_when, name_of_api_key='virustotal', **kwargs)
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
def test_required_with_two_keys(self, blacklist_data):
blacklist_data['blacklist_keys'] = ['fruit_name', 'car_name']
blacklist = create_blacklist(blacklist_data)
assert blacklist._blacklisted_keys == blacklist_data['blacklist_keys']
def test_is_regex(self):
self._blacklist_data['blacklist_is_regex'] = True
blacklist = create_blacklist(self._blacklist_data)
T.assert_equal(blacklist._is_regex, True)
def test_keys_not_list(self):
self._blacklist_data['blacklist_keys'] = 'fruit_name'
with T.assert_raises(MissingConfigError):
create_blacklist(self._blacklist_data)
def test_is_regex(self):
self._blacklist_data['blacklist_is_regex'] = True
blacklist = create_blacklist(self._blacklist_data)
T.assert_equal(blacklist._is_regex, True)
def test_keys_not_list(self, blacklist_data):
blacklist_data['blacklist_keys'] = 'fruit_name'
with pytest.raises(MissingConfigError):
create_blacklist(blacklist_data)
def test_bad_domains(self):
self._blacklist_data['blacklist_is_domains'] = True
with T.assert_raises(BadDomainError):
create_blacklist(self._blacklist_data)
def test_missing_required_keys(self):
for key in self._blacklist_data.keys():
blacklist_data = deepcopy(self._blacklist_data)
del blacklist_data[key]
with T.assert_raises(MissingConfigError):
create_blacklist(blacklist_data)
def test_missing_data_input(self, blacklist_data):
blacklist_data.pop('blacklist_file_path')
with pytest.raises(MissingConfigError):
create_blacklist(blacklist_data)
def __init__(self, lookup_when=None):
super(LookupDomainsFilter, self).__init__('osxcollector_domains', 'osxcollector_opendns',
lookup_when=lookup_when, name_of_api_key='opendns')
self._whitelist = create_blacklist(config_get_deep('domain_whitelist'))
