def ValhallaReporter(values):
vallaha_val = []
for usrInput in values:
valFramework = copy.deepcopy(empty)
chk_256 = list(iocextract.extract_sha256_hashes(usrInput))
if chk_256:
usrInput = chk_256[0]
v = ValhallaAPI(api_key=valhallaKey)
response = v.get_hash_info(hash=usrInput)
if response.get('status') == 'empty':
pass
else:
dataReport = response.get('results')[0]
if len(dataReport) > 0:
valFramework.update({'Action On': usrInput})
valFramework.update(
{'Positives': dataReport.get('positives')})
valFramework.update(
{'Rule Name': dataReport.get('rulename')})
valFramework.update({'Tags': dataReport.get('tags')})
valFramework.update(
{'Timestamp': dataReport.get('timestamp')})
valFramework.update({'Total': dataReport.get('total')})
valFramework.update({
'Score':
str(dataReport.get('positives')) + '/' +
str(dataReport.get('total'))
})
vallaha_val.append({'Vallaha': valFramework})
return vallaha_val
def _utility_ioc_extractor_function(self, event, *args, **kwargs):
results = {}
results["was_successful"] = False
try:
# Get the function parameters:
incident_id = kwargs.get("incident_id") # number
text_string = kwargs.get("text_string") # text
log = logging.getLogger(__name__) # Establish logging
text_string = unicodedata.normalize(
"NFKD",
BeautifulSoup(text_string, "html.parser").get_text(
' ')) # Strip HTML and normalize text
# Parse IOCs by type from text_string - OrderedDict.fromkeys() preserves order and removes duplicates.
results["ipv4s"] = list(
OrderedDict.fromkeys(
list(iocextract.extract_ipv4s(text_string, refang=True))))
results["ipv6s"] = list(
OrderedDict.fromkeys(
list(iocextract.extract_ipv6s(text_string))))
results["urls"] = list(
OrderedDict.fromkeys(
list(iocextract.extract_urls(
text_string, refang=True)))) # URLs and domains
results["domains"] = list(
OrderedDict.fromkeys([
urlparse(url).netloc for url in results["urls"]
])) # domains only
results["email_addresses"] = list(
OrderedDict.fromkeys(
list(iocextract.extract_emails(text_string, refang=True))))
results["email_domains"] = list(
OrderedDict.fromkeys([
email.split('@')[1] for email in results["email_addresses"]
])) # domains only
results["md5_hashes"] = list(
OrderedDict.fromkeys(
list(iocextract.extract_md5_hashes(text_string))))
results["sha256_hashes"] = list(
OrderedDict.fromkeys(
list(iocextract.extract_sha256_hashes(text_string))))
results["was_successful"] = True
# Produce a FunctionResult with the results
yield FunctionResult(results)
except Exception:
yield FunctionError()
def test_sha256_extract(self):
content = '01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b'
self.assertEquals(list(iocextract.extract_sha256_hashes(content))[0], content)
self.assertEquals(list(iocextract.extract_sha256_hashes(_wrap_spaces(content)))[0], content)
self.assertEquals(list(iocextract.extract_sha256_hashes(_wrap_tabs(content)))[0], content)
self.assertEquals(list(iocextract.extract_sha256_hashes(_wrap_newlines(content)))[0], content)
self.assertEquals(list(iocextract.extract_sha256_hashes(_wrap_words(content)))[0], content)
self.assertEquals(list(iocextract.extract_sha256_hashes(_wrap_nonwords(content)))[0], content)
def test_sha256_not_in_sha512(self):
content = 'be688838ca8686e5c90689bf2ab585cef1137c999b48c70b92f67a5c34dc15697b5d11c982ed6d71be1e1e7f7b4e0733884aa97c3f7a339a8ed03577cf74be09'
self.assertEquals(len(list(iocextract.extract_sha256_hashes(content))), 0)
self.assertEquals(len(list(iocextract.extract_sha256_hashes(_wrap_spaces(content)))), 0)
self.assertEquals(len(list(iocextract.extract_sha256_hashes(_wrap_tabs(content)))), 0)
self.assertEquals(len(list(iocextract.extract_sha256_hashes(_wrap_newlines(content)))), 0)
self.assertEquals(len(list(iocextract.extract_sha256_hashes(_wrap_words(content)))), 0)
self.assertEquals(len(list(iocextract.extract_sha256_hashes(_wrap_nonwords(content)))), 0)
def create_group_pulse(input_text):
# Create the pulse title
unix_time = str(int(time.time()))
pulse_title = 'SlackIOCs - ' + unix_time
API_KEY = ''
otx = OTXv2(API_KEY)
group_id = 840
# Create a list of indicators
indicators = []
for url in iocextract.extract_urls(input_text):
indicators.append({'indicator': url, 'type': 'URL'})
for ip in iocextract.extract_ips(input_text):
indicators.append({'indicator': ip, 'type': 'IPv4'})
for sha256 in iocextract.extract_sha256_hashes(input_text):
indicators.append({'indicator': sha256, 'type': 'FileHash-SHA256'})
for sha1 in iocextract.extract_sha1_hashes(input_text):
indicators.append({'indicator': sha1, 'type': 'FileHash-SHA1'})
for md5 in iocextract.extract_md5_hashes(input_text):
indicators.append({'indicator': md5, 'type': 'FileHash-MD5'})
for email in iocextract.extract_emails(input_text):
indicators.append({'indicator': email, 'type': 'EMAIL'})
print('Adding ' + str(indicators))
response = otx.create_pulse(name=pulse_title,
public=True,
indicators=indicators,
tags=['covid19'],
references=[],
group_ids=[group_id],
tlp='White')
print('Response: ' + str(response))
def CapeReporter(values):
cape_val = []
for usrInput in values:
chk_ip = list(iocextract.extract_ipv4s(usrInput))
chk_url = list(iocextract.extract_urls(usrInput))
chk_md5 = list(iocextract.extract_md5_hashes(usrInput))
chk_sha1 = list(iocextract.extract_sha1_hashes(usrInput))
chk_256 = list(iocextract.extract_sha256_hashes(usrInput))
if chk_url:
usrInput = chk_url[0]
argType = 'url'
stream = allReport(usrInput, argType)
for data in stream:
cape_val.append({'Cape Sandbox': data})
elif chk_ip:
usrInput = chk_ip[0]
argType = 'ip'
stream = allReport(usrInput, argType)
for data in stream:
cape_val.append({'Cape Sandbox': data})
elif chk_md5:
usrInput = chk_md5[0]
argType = 'md5'
stream = allReport(usrInput, argType)
for data in stream:
cape_val.append({'Cape Sandbox': data})
elif chk_sha1:
usrInput = chk_sha1[0]
argType = 'sha1'
stream = allReport(usrInput, argType)
for data in stream:
cape_val.append({'Cape Sandbox': data})
elif chk_256:
usrInput = chk_256[0]
argType = 'sha256'
stream = allReport(usrInput, argType)
for data in stream:
cape_val.append({'Cape Sandbox': data})
else:
pass
return cape_val
def test_sha256(self):
content = "c4ed9a5c3798260ebc2c43c02428cae33fe3dd59129ec82f50374b82a4e4907d"
result = list(iocextract.extract_sha256_hashes(content))
self.assertEqual(len(result), 1)
self.assertEqual(result[0], content)
continue
# Extract text from pdf
filepath = os.path.join(path, filename)
content = convert_pdf_txt(filepath)
# Extract Indicators of Compromise from text, recording time
extracted_files[filename] = {}
extract_start_time = time.time()
extracted_files[filename]["urls"] = list(iocextract.extract_urls(content, refang=True))
extracted_files[filename]["email_addresses"] = list(iocextract.extract_emails(content, refang=True))
extracted_files[filename]["ipv4s"] = list(iocextract.extract_ipv4s(content, refang=True))
extracted_files[filename]["ipv6s"] = list(iocextract.extract_ipv6s(content))
extracted_files[filename]["md5s"] = list(iocextract.extract_md5_hashes(content))
extracted_files[filename]["sha1s"] = list(iocextract.extract_sha1_hashes(content))
extracted_files[filename]["sha256s"] = list(iocextract.extract_sha256_hashes(content))
extracted_files[filename]["sha512s"] = list(iocextract.extract_sha512_hashes(content))
extracted_files[filename]["yara"] = list(iocextract.extract_yara_rules(content))
extract_avg_numerator += time.time() - extract_start_time
count += 1
process_end_time = time.time()
# add some meta info on process run time
extracted_files["meta"] = {
"tool": "iocextract",
"files_examined": count,
"elapsed_time": process_end_time - process_start_time,
"average_time": extract_avg_numerator / count,
}