def test_extract_custom_iocs_excepts_on_bad_regex(self): # Note: have to use list() here because exceptions are only raised when # the generator is executed. with self.assertRaises(re.error): list(iocextract.extract_custom_iocs('', [r'(mismatched paren'])) list(iocextract.extract_custom_iocs('', [r'[mismatched bracket'])) with self.assertRaises(IndexError): list(iocextract.extract_custom_iocs('', [r'no capture group'])) list(iocextract.extract_custom_iocs('', [r'']))
def test_extract_custom_extracts_from_list(self): regex_list = [ r'(test\d)', r't(..)t', r'^(str.ng)$', ] self.assertEqual( list(iocextract.extract_custom_iocs('test1', regex_list)), ['test1', 'es']) self.assertEqual( list(iocextract.extract_custom_iocs('a test2 string', regex_list)), ['test2', 'es']) self.assertEqual( list(iocextract.extract_custom_iocs('just testing', regex_list)), ['es']) self.assertEqual( list(iocextract.extract_custom_iocs('string', regex_list)), ['string']) self.assertEqual( list(iocextract.extract_custom_iocs('strong', regex_list)), ['strong']) self.assertEqual( list(iocextract.extract_custom_iocs('strange', regex_list)), []) self.assertEqual( list(iocextract.extract_custom_iocs('another one', regex_list)), [])
def find_iocs(text, blacklist=None): iocs = deepcopy(default_values) if not blacklist: blacklist = [] # Custom IOCs custom_funcs = { "bitcoin_addresses": [CRYPTO_WALLET_BITCOIN], "bitcoincash_addresses": [CRYPTO_WALLET_BITCOIN_CASH], "ethereum_addresses": [CRYPTO_WALLET_ETHEREUM], "litecoin_addresses": [CRYPTO_WALLET_LITECOIN], "dogecoin_addresses": [CRYPTO_WALLET_DOGECOIN], "dash_addresses": [CRYPTO_WALLET_DASH], "monero_addresses": [CRYPTO_WALLET_MONERO], "neo_addresses": [CRYPTO_WALLET_NEO], "ripple_addresses": [CRYPTO_WALLET_RIPPLE], "onion_addresses": [TOR_ONION_V2_ADDR, TOR_ONION_V3_ADDR], } for ioc_type, ioc_regexes in custom_funcs.items(): if ioc_type in blacklist: continue validator = get_validator(ioc_type) for ioc in extract_custom_iocs(text, ioc_regexes): if validator and not validator(ioc): continue # IOC Extract no_refang = ["ipv6s", "md5s", "sha1s", "sha256s", "sha512s"] iocextract_funcs = { "ipv4s": extract_ipv4s, "ipv6s": extract_ipv6s, "urls": extract_urls, "email_addresses": extract_emails, "md5s": extract_md5_hashes, "sha1s": extract_sha1_hashes, "sha256s": extract_sha256_hashes, "sha512s": extract_sha512_hashes, } for ioc_type, ioc_func in iocextract_funcs.items(): if ioc_type in blacklist: continue validator = get_validator(ioc_type) if ioc_type in no_refang: ioc_values = ioc_func(text) else: ioc_values = ioc_func(text, refang=True) for ioc in ioc_values: if validator and not validator(ioc): continue iocs[ioc_type].append(ioc) # IOC Finder ioc_finder_res = if_find_iocs(text) for ioc_type, ioc_values in ioc_finder_res.items(): if ioc_type in blacklist: continue validator = get_validator(ioc_type) for ioc in ioc_values: if validator and not validator(ioc): continue iocs[ioc_type].append(ioc) return {k: list(set(v)) for k, v in iocs.items() if len(v) > 0}
def test_extract_custom_iocs_no_match_extracts_nothing(self): self.assertEqual( list(iocextract.extract_custom_iocs('words', [r'egex'])), [])
def test_extract_custom_iocs_empty_content_extracts_nothing(self): self.assertEqual(list(iocextract.extract_custom_iocs('', [r'egex'])), [])