def test_extract_custom_iocs_excepts_on_bad_regex(self):
# Note: have to use list() here because exceptions are only raised when
# the generator is executed.
with self.assertRaises(re.error):
list(iocextract.extract_custom_iocs('', [r'(mismatched paren']))
list(iocextract.extract_custom_iocs('', [r'[mismatched bracket']))
with self.assertRaises(IndexError):
list(iocextract.extract_custom_iocs('', [r'no capture group']))
list(iocextract.extract_custom_iocs('', [r'']))
def test_extract_custom_extracts_from_list(self):
regex_list = [
r'(test\d)',
r't(..)t',
r'^(str.ng)$',
]
self.assertEqual(
list(iocextract.extract_custom_iocs('test1', regex_list)),
['test1', 'es'])
self.assertEqual(
list(iocextract.extract_custom_iocs('a test2 string', regex_list)),
['test2', 'es'])
self.assertEqual(
list(iocextract.extract_custom_iocs('just testing', regex_list)),
['es'])
self.assertEqual(
list(iocextract.extract_custom_iocs('string', regex_list)),
['string'])
self.assertEqual(
list(iocextract.extract_custom_iocs('strong', regex_list)),
['strong'])
self.assertEqual(
list(iocextract.extract_custom_iocs('strange', regex_list)), [])
self.assertEqual(
list(iocextract.extract_custom_iocs('another one', regex_list)),
[])
def find_iocs(text, blacklist=None):
iocs = deepcopy(default_values)
if not blacklist:
blacklist = []
# Custom IOCs
custom_funcs = {
"bitcoin_addresses": [CRYPTO_WALLET_BITCOIN],
"bitcoincash_addresses": [CRYPTO_WALLET_BITCOIN_CASH],
"ethereum_addresses": [CRYPTO_WALLET_ETHEREUM],
"litecoin_addresses": [CRYPTO_WALLET_LITECOIN],
"dogecoin_addresses": [CRYPTO_WALLET_DOGECOIN],
"dash_addresses": [CRYPTO_WALLET_DASH],
"monero_addresses": [CRYPTO_WALLET_MONERO],
"neo_addresses": [CRYPTO_WALLET_NEO],
"ripple_addresses": [CRYPTO_WALLET_RIPPLE],
"onion_addresses": [TOR_ONION_V2_ADDR, TOR_ONION_V3_ADDR],
}
for ioc_type, ioc_regexes in custom_funcs.items():
if ioc_type in blacklist:
continue
validator = get_validator(ioc_type)
for ioc in extract_custom_iocs(text, ioc_regexes):
if validator and not validator(ioc):
continue
# IOC Extract
no_refang = ["ipv6s", "md5s", "sha1s", "sha256s", "sha512s"]
iocextract_funcs = {
"ipv4s": extract_ipv4s,
"ipv6s": extract_ipv6s,
"urls": extract_urls,
"email_addresses": extract_emails,
"md5s": extract_md5_hashes,
"sha1s": extract_sha1_hashes,
"sha256s": extract_sha256_hashes,
"sha512s": extract_sha512_hashes,
}
for ioc_type, ioc_func in iocextract_funcs.items():
if ioc_type in blacklist:
continue
validator = get_validator(ioc_type)
if ioc_type in no_refang:
ioc_values = ioc_func(text)
else:
ioc_values = ioc_func(text, refang=True)
for ioc in ioc_values:
if validator and not validator(ioc):
continue
iocs[ioc_type].append(ioc)
# IOC Finder
ioc_finder_res = if_find_iocs(text)
for ioc_type, ioc_values in ioc_finder_res.items():
if ioc_type in blacklist:
continue
validator = get_validator(ioc_type)
for ioc in ioc_values:
if validator and not validator(ioc):
continue
iocs[ioc_type].append(ioc)
return {k: list(set(v)) for k, v in iocs.items() if len(v) > 0}
def test_extract_custom_iocs_no_match_extracts_nothing(self):
self.assertEqual(
list(iocextract.extract_custom_iocs('words', [r'egex'])), [])
def test_extract_custom_iocs_empty_content_extracts_nothing(self):
self.assertEqual(list(iocextract.extract_custom_iocs('', [r'egex'])),
[])