def selftest_function(opts):
"""
Placeholder for selftest function. An example use would be to test package api connectivity.
Suggested return values are be unimplemented, success, or failure.
"""
TEST_IP = "8.8.8.8"
options = opts.get("fn_virustotal", {})
reason = ""
try:
vt = VirusTotal(options['api_token'], options['proxies'])
response = vt.get_ip_report(TEST_IP)
if response and type(response) is not dict:
state = "failure"
reason = "no response"
else:
status = response.get('response_code', -1)
if status != HTTP_OK:
state = "failure"
reason = state
else:
state = "success"
except Exception as err:
state = "failure"
reason = str(err)
result = {"state": state, "reason": reason}
log.info(result)
return result
def virustotal(conf, ipaddress, results):
"""This method updates the network results
with the Virustotal reports.
Args:
conf (dict): dict of configuration
ipaddress (string): ip address to analyze
results (dict): dict where will put the results
Returns:
This method updates the results dict given
"""
if conf["enabled"]:
from virus_total_apis import PublicApi as VirusTotalPublicApi
vt = VirusTotalPublicApi(conf["api_key"])
# Error: {u'virustotal': {'error': SSLError(SSLEOFError(8, u'EOF
# occurred in violation of protocol (_ssl.c:590)'),)}}')
# TypeError: SSLError(SSLEOFError(8, u'EOF occurred in violation of
# protocol (_ssl.c:590)'),) is not JSON serializable')
try:
r = vt.get_ip_report(ipaddress)
report = json.dumps(r, ensure_ascii=False)
except TypeError:
log.error(
"TypeError in VirusTotal report for ip {!r}".format(ipaddress))
else:
if report:
results["virustotal"] = report
def test_get_ip_report(self):
vt = PublicApi(API_KEY)
try:
print(json.dumps(vt.get_ip_report('23.6.113.133'), sort_keys=False, indent=4))
except Exception as e:
self.fail(e)
def test_get_ip_report(self):
vt = PublicApi(API_KEY)
try:
print json.dumps(vt.get_ip_report('23.6.113.133'),
sort_keys=False,
indent=4)
except Exception as e:
self.fail(e)
def handle_ip(actapi: act.api.Act,
vtapi: VirusTotalApi,
ip: Text,
output_format: Text = "json") -> None:
"""Read IP address from stdin, query VirusTotal and
output a JSON text readable by generic_uploaderr.py"""
# To figure out what kind of IP address we have, let the ipaddress module
# parse the string and test for instance type as the platform distinguishes
# between IPv4 and IPv6 addresses.
try:
(ip_type, ip) = act.api.helpers.ip_obj(ip)
except ValueError:
return # invalid address
with no_ssl_verification():
response = vtapi.get_ip_report(ip)
try:
results = response['results']
except KeyError:
logging.error("%s in handle_ip for %s", response, ip)
sys.exit(1)
# create a dictionary of url that is observed in relation to the address.
urls: collections.defaultdict = collections.defaultdict(list)
if 'detected_urls' in results:
for u in map(urllib.parse.urlparse,
[x['url'] for x in results['detected_urls']]):
urls[u.netloc].append(u)
if 'undetected_urls' in results:
for u in map(urllib.parse.urlparse,
[x[0] for x in results['undetected_urls']]):
urls[u.netloc].append(u)
if 'resolutions' in results:
for resolution in results['resolutions']:
act.api.helpers.handle_fact(actapi.fact('resolvesTo').source(
'fqdn', resolution['hostname']).destination(ip_type, ip),
output_format=output_format)
# add all detected and undetected urls related to a given resolved hostname
if resolution['hostname'] in urls:
for u in urls[resolution['hostname']]:
add_uri(actapi, 'fqdn', resolution['hostname'], list(u))
# if the actuall ip is part of the url, add the urls directly connected to the
# ip.
if ip in urls:
for u in urls[ip]:
add_uri(actapi, ip_type, ip, list(u))
if 'detected_downloaded_samples' in results:
for sample in results['detected_downloaded_samples']:
my_uri = add_uri(actapi, ip_type, ip,
['network', ip, '', '', '', ''])
act.api.helpers.handle_fact(actapi.fact('at').source(
'content', sample['sha256']).destination('uri', my_uri),
output_format=output_format)
act.api.helpers.handle_fact(actapi.fact('represents').source(
'hash', sample['sha256']).destination('content',
sample['sha256']),
output_format=output_format)
handle_hexdigest(actapi,
vtapi,
sample['sha256'],
output_format=output_format)
if 'detected_communicating_samples' in results:
for sample in results['detected_communicating_samples']:
my_uri = add_uri(actapi, ip_type, ip,
['network', ip, '', '', '', ''])
act.api.helpers.handle_fact(actapi.fact('connectsTo').source(
'content', sample['sha256']).destination('uri', my_uri),
output_format=output_format)
act.api.helpers.handle_fact(actapi.fact('represents').source(
'hash', sample['sha256']).destination('content',
sample['sha256']),
output_format=output_format)
handle_hexdigest(actapi,
vtapi,
sample['sha256'],
output_format=output_format)
if x == 1:
# try ip
# def (ip):
ip = raw_input("Enter Ip adress :")
print " OTX_REPORT " * 80
pprint.pprint(threatcrowd.ip_report(ip))
print "**" * 80
print "VIRUSTOTAL_REPORT " * 80
vt = VirusTotalPublicApi(API_KEY_VT)
response = vt.get_ip_report(ip)
print(json.dumps(response, sort_keys=False, indent=4))
# ###################################################################################################
#hybrid analysis module
elif x == 2:
# email_report(address)
email = raw_input("Enter Email :")
pprint.pprint(threatcrowd.email_report(email))
print "/n"
print "virustotal report"
print "**" * 80
def _virustotal_function(self, event, *args, **kwargs):
"""Function: perform different scans on the following types:
ip addresses
hash - this will attempt to find an existing file report on the hash
domain
url - this will attempt to find an existing file report on the url. If none exist, a new scan is queued
file - this will start a new scan for the file and queue for a report later.
"""
try:
validateFields(('incident_id', 'vt_type'), kwargs) # required
# Init RequestsCommon with app.config options
rc = RequestsCommon(opts=self.opts, function_opts=self.options)
# Create a VirusTotal instance with the API Token and any proxies gathered by RequestsCommon
vt = VirusTotal(self.options['api_token'], rc.get_proxies())
# Get the function parameters:
incident_id = kwargs.get("incident_id") # number
artifact_id = kwargs.get("artifact_id") # number
attachment_id = kwargs.get("attachment_id") # number
vt_type = kwargs.get("vt_type") # text
vt_data = kwargs.get("vt_data") # text
self.log = logging.getLogger(__name__)
self.log.info("incident_id: %s", incident_id)
self.log.info("artifact_id: %s", artifact_id)
self.log.info("attachment_id: %s", attachment_id)
self.log.info("vt_type: %s", vt_type)
self.log.info("vt_data: %s", vt_data)
yield StatusMessage("starting...")
# determine next steps based on the API call to make
if vt_type.lower() == 'file':
entity = get_input_entity(get_resilient_client(self.resilient),
incident_id, attachment_id,
artifact_id)
# Create a temporary file to write the binary data to.
with tempfile.NamedTemporaryFile(
'w+b', delete=False) as temp_file_binary:
# Write binary data to a temporary file. Make sure to close the file here...this
# code must work on Windows and on Windows the file cannot be opened a second time
# While open. Floss will open the file again to read the data, so close before
# calling Floss.
temp_file_binary.write(entity["data"])
temp_file_binary.close()
try:
response = vt.scan_file(temp_file_binary.name,
filename=entity["name"])
except Exception as err:
raise err
finally:
os.unlink(temp_file_binary.name)
file_result = self.return_response(response,
vt.get_file_report,
time.time())
## was a sha-256 returned? try an existing report first
if file_result.get("sha256"):
response = vt.get_file_report(file_result.get("sha256"))
report_result = self.return_response(
response, None, time.time())
if report_result.get(
"response_code") and report_result.get(
"response_code") == 1:
result = report_result
else:
result = file_result
elif vt_type.lower() == 'url':
# attempt to see if a report already exists
response = vt.get_url_report(vt_data)
result = self.return_response(response, None, time.time())
# check if result is not found, meaning no report exists
if result['response_code'] == RC_NOT_FOUND:
response = vt.scan_url(vt_data)
result = self.return_response(response, vt.get_url_report,
time.time())
elif vt_type.lower() == 'ip':
response = vt.get_ip_report(vt_data)
result = self.return_response(response, None, time.time())
elif vt_type.lower() == 'domain':
response = vt.get_domain_report(vt_data)
result = self.return_response(response, None, time.time())
elif vt_type.lower() == 'hash':
response = vt.get_file_report(vt_data)
result = self.return_response(response, None, time.time())
else:
raise ValueError(
"Unknown type field: {}. Check workflow pre-processor script."
.format(vt_type))
results = {"scan": result}
self.log.debug("scan: {}".format(results))
# Produce a FunctionResult with the results
yield FunctionResult(results)
except Exception:
yield FunctionError()
def handle_ip(actapi: act.api.Act,
vtapi: VirusTotalApi,
ip: Text,
output_format: Text = "json") -> None:
"""Read IP address from stdin, query VirusTotal and
output a JSON text readable by generic_uploaderr.py"""
# To figure out what kind of IP address we have, let the ipaddress module
# parse the string and test for instance type as the platform distinguishes
# between IPv4 and IPv6 addresses.
try:
(ip_type, ip) = act.api.helpers.ip_obj(ip)
except ValueError:
return # invalid address
with no_ssl_verification():
response = vtapi.get_ip_report(ip)
try:
results = response["results"]
except KeyError:
logging.error("%s in handle_ip for %s", response, ip)
sys.exit(1)
# create a dictionary of url that is observed in relation to the address.
urls: collections.defaultdict = collections.defaultdict(list)
if "detected_urls" in results:
for u in map(urllib.parse.urlparse,
[x["url"] for x in results["detected_urls"]]):
urls[u.netloc].append(u)
if "undetected_urls" in results:
for u in map(urllib.parse.urlparse,
[x[0] for x in results["undetected_urls"]]):
urls[u.netloc].append(u)
if "resolutions" in results:
for resolution in results["resolutions"]:
if not resolution["hostname"]:
warning("Empty hostname in resolution: ip=%s, result=%s", ip,
results)
continue
act.api.helpers.handle_fact(
actapi.fact("resolvesTo").source(
"fqdn", resolution["hostname"]).destination(ip_type, ip),
output_format=output_format,
)
# add all detected and undetected urls related to a given resolved hostname
if resolution["hostname"] in urls:
for u in urls[resolution["hostname"]]:
add_uri(actapi, "fqdn", resolution["hostname"], list(u))
# if the actuall ip is part of the url, add the urls directly connected to the
# ip.
if ip in urls:
for u in urls[ip]:
add_uri(actapi, ip_type, ip, list(u))
if "detected_downloaded_samples" in results:
for sample in results["detected_downloaded_samples"]:
my_uri = add_uri(actapi, ip_type, ip,
["network", ip, "", "", "", ""])
act.api.helpers.handle_fact(
actapi.fact("at").source("content",
sample["sha256"]).destination(
"uri", my_uri),
output_format=output_format,
)
act.api.helpers.handle_fact(
actapi.fact("represents").source("hash",
sample["sha256"]).destination(
"content",
sample["sha256"]),
output_format=output_format,
)
handle_hexdigest(actapi,
vtapi,
sample["sha256"],
output_format=output_format)
if "detected_communicating_samples" in results:
for sample in results["detected_communicating_samples"]:
my_uri = add_uri(actapi, ip_type, ip,
["network", ip, "", "", "", ""])
act.api.helpers.handle_fact(
actapi.fact("connectsTo").source("content",
sample["sha256"]).destination(
"uri", my_uri),
output_format=output_format,
)
act.api.helpers.handle_fact(
actapi.fact("represents").source("hash",
sample["sha256"]).destination(
"content",
sample["sha256"]),
output_format=output_format,
)
handle_hexdigest(actapi,
vtapi,
sample["sha256"],
output_format=output_format)
class VirusTotal:
OutputDir = None
HTMLHeader = None
APIKeys = None
APIKEY = None
APIKeysNumber = None
APIKeyIndex = None
DisabledAttr = None
MaxResults = None
AttrSubstitution = None
Order = None
VTInstance = None
Persistence = None
Host = None
Database = None
UsernameR = None
PasswordR = None
UsernameRW = None
PasswordRW = None
Input = None
IP = None
GeneralOutput = None
DBR = None
DBRW = None
CursorR = None
CursorRW = None
def init(self):
config = None
try:
with open('config.json', 'r') as configFile:
configContent = configFile.read()
config = json.loads(configContent)
except json.decoder.JSONDecodeError:
print("Problem occured while parsing the config.json file")
exit()
if config == None:
print("Problem occured while parsing the config.json file")
exit()
self.OutputDir = config['General']['OutputDir']
self.HTMLHeader = config['General']['HTMLHeader']
self.TablesClass = config['General']['TablesClass']
self.APIKeys = config['VirusTotal']['APIKeys']
self.loadRecordsTmp()
self.APIKEY = self.APIKeys[self.APIKeyIndex]
self.APIKeysNumber = len(self.APIKeys)
self.DisabledAttr = config['VirusTotal']['DisabledAttr']
self.MaxResults = config['VirusTotal']['MaxResults']
self.AttrSubstitution = config['VirusTotal']['AttrSubstitution']
self.Order = config['VirusTotal']['Order']
self.VTInstance = VirusTotalPublicApi(self.APIKEY)
self.Persistence = config['VirusTotal']['Persistence']
self.Host = config['VirusTotal']['PersistenceCredentials']['host']
self.Database = config['VirusTotal']['PersistenceCredentials'][
'database']
self.UsernameR = config['VirusTotal']['PersistenceCredentials'][
'username_r']
self.PasswordR = config['VirusTotal']['PersistenceCredentials'][
'password_r']
self.UsernameRW = config['VirusTotal']['PersistenceCredentials'][
'username_rw']
self.PasswordRW = config['VirusTotal']['PersistenceCredentials'][
'password_rw']
self.Input = config['VirusTotal']['Input']
self.GeneralOutput = config['VirusTotal']['GeneralOutput']
self.HTML = ""
self.IMG = ""
if self.Persistence == PERSISTENCE_TYPE_SQL:
self.initSQL()
def initSQL(self):
self.DBR = mysql.connector.connect(host=self.Host,
user=self.UsernameR,
passwd=self.PasswordR,
database=self.Database)
self.CursorR = self.DBR.cursor(dictionary=True)
self.DBRW = mysql.connector.connect(host=self.Host,
user=self.UsernameRW,
passwd=self.PasswordRW,
database=self.Database)
self.CursorRW = self.DBRW.cursor(dictionary=True)
def resetSQL(self):
self.CursorR.close()
self.CursorRW.close()
self.DBR.close()
self.DBRW.close()
self.initSQL()
def preHandling(self):
self.number = {}
self.history = {}
def loadRecordsTmp(self):
try:
with open('.records.tmp', 'r') as recordsTmpFile:
recordsTmpContent = recordsTmpFile.read()
recordsTmp = json.loads(recordsTmpContent)
self.APIKeyIndex = recordsTmp["APIKeyIndex"]
except Exception:
self.APIKeyIndex = 0
recordsTmp = {"APIKeyIndex": self.APIKeyIndex}
with open('.records.tmp', 'w') as recordsTmpFile:
json.dump(recordsTmp, recordsTmpFile)
def updateAPIKeyIndex(self):
try:
with open('.records.tmp', 'r') as recordsTmpFile:
recordsTmpContent = recordsTmpFile.read()
recordsTmp = json.loads(recordsTmpContent)
except Exception:
recordsTmp = {"APIKeyIndex": self.APIKeyIndex}
recordsTmp["APIKeyIndex"] = self.APIKeyIndex
with open('.records.tmp', 'w') as recordsTmpFile:
json.dump(recordsTmp, recordsTmpFile)
def updateVTInstance(self):
self.APIKeyIndex = (self.APIKeyIndex + 1) % self.APIKeysNumber
self.updateAPIKeyIndex()
self.APIKEY = self.APIKeys[self.APIKeyIndex]
self.VTInstance = VirusTotalPublicApi(self.APIKEY)
def getIPReportAPI(self):
if self.Input == INPUT_TYPE_FILE:
return self.getIPReportAPIFile()
elif self.Input == INPUT_TYPE_ARGUMENT:
return self.getIPReportAPIArgument()
return {}
def getIPReportAPIArgument(self):
ips = "\n".join(self.IP.split(" ")).strip()
return self.getVTReport(ips)
def getIPReportAPIFile(self):
result = {}
with open("input_ip.txt") as file:
ips = file.read().strip()
result = self.getVTReport(ips)
return result
def getVTReport(self, ips):
result = {}
for ip in ips.split("\n"):
response = self.VTInstance.get_ip_report(ip)
self.updateVTInstance()
if response['response_code'] == 200:
result[ip] = response['results']
return result
def setHTMLDomain(self, domain):
return '<a href="https://www.virustotal.com/gui/domain/' + domain + '/details">' + domain + '</a>'
def formatArrayDateDomain(self, ip_report_api, attr):
result = []
count = self.MaxResults[attr] if self.MaxResults[attr] else -1
tmp = sorted(ip_report_api[attr], key=lambda i: i['last_resolved'])
for elem in list(reversed(tmp)):
if count == 0:
break
obj = {'Date resolved': "", 'Domain': ""}
obj['Date resolved'] = elem['last_resolved']
obj['Domain'] = self.setHTMLDomain(elem['hostname'])
result.append(obj)
count = count - 1
return result
def setHTMLFileHash(self, hash):
return '<a href="https://www.virustotal.com/gui/file/' + hash + '/detection">' + hash + '</a>'
def formatArrayDateScoreHash(self, ip_report_api, attr):
result = []
count = self.MaxResults[attr] if self.MaxResults[attr] else -1
for elem in list(ip_report_api[attr]):
if count == 0:
break
obj = {'Scanned': "", 'Detections': "", 'File Hash (sha256)': ""}
obj['Scanned'] = elem['date']
if elem['positives'] == 0:
color = "green"
else:
color = "red"
obj['Detections'] = '<span style="color:' + color + '">' + str(
elem['positives']) + "</span>/" + str(elem['total'])
#response=self.VTInstance.get_file_report(elem['sha256'])
#print(response)
obj['File Hash (sha256)'] = self.setHTMLFileHash(elem['sha256'])
result.append(obj)
count = count - 1
return result
def setHTMLURL(self, url, url_hash):
if url_hash == False:
url_hash = hashlib.sha256(url.encode('utf-8')).hexdigest()
return '<a href="https://www.virustotal.com/gui/url/' + url_hash + '/detection">' + url + '</a>'
def formatArrayDateScoreURL(self, ip_report_api, attr):
result = []
count = self.MaxResults[attr] if self.MaxResults[attr] else -1
for elem in list(ip_report_api[attr]):
if count == 0:
break
obj = {'Scanned': "", 'Detections': "", 'URL': ""}
obj['Scanned'] = elem['scan_date']
if elem['positives'] == 0:
color = "green"
else:
color = "red"
obj['Detections'] = '<span style="color:' + color + '">' + str(
elem['positives']) + "</span>/" + str(elem['total'])
#response=self.VTInstance.get_file_report(elem['sha256'])
#print(response)
obj['URL'] = self.setHTMLURL(elem['url'], False)
result.append(obj)
count = count - 1
return result
def formatArrayDateScoreURLnum(self, ip_report_api, attr):
result = []
count = self.MaxResults[attr] if self.MaxResults[attr] else -1
for elem in list(ip_report_api[attr]):
if count == 0:
break
obj = {'Scanned': "", 'Detections': "", 'URL': ""}
obj['Scanned'] = elem[4]
if elem[2] == 0:
color = "green"
else:
color = "red"
obj['Detections'] = '<span style="color:' + color + '">' + str(
elem[2]) + "</span>/" + str(elem[3])
#response=self.VTInstance.get_file_report(elem['sha256'])
#print(response)
obj['URL'] = self.setHTMLURL(elem[0], elem[1])
result.append(obj)
count = count - 1
return result
def getNumberMalicious(self, ip_report_api, attr):
result = [
d for d in ip_report_api[attr]
if ('positives' in d and d['positives'] > 0) or (
'positives' not in d and d[2] > 0)
]
return result
def getNumberBenign(self, ip_report_api, attr):
result = [
d for d in ip_report_api[attr]
if ('positives' in d and d['positives'] == 0) or (
'positives' not in d and d[2] == 0)
]
return result
def updateScoredAttr(self, attr, verdict, result):
self.number[attr] = self.number[attr] if attr in self.number else {}
self.number[attr][verdict] = result
def getIPReportFiltered(self, ip_report_api):
result = {}
for attr in list(ip_report_api):
if (attr in self.DisabledAttr):
None #del result[attr]
elif attr == 'resolutions':
newAttr = self.AttrSubstitution[
attr] if attr in self.AttrSubstitution else attr
self.history[newAttr] = len(ip_report_api[attr])
result[newAttr] = self.formatArrayDateDomain(
ip_report_api, attr)
elif attr == 'detected_referrer_samples':
newAttr = self.AttrSubstitution[
attr] if attr in self.AttrSubstitution else attr
tmp = self.formatArrayDateScoreHash(ip_report_api, attr)
self.updateScoredAttr(
newAttr, "malicious",
len(self.getNumberMalicious(ip_report_api, attr)))
if newAttr in result:
result[newAttr] = tmp + result[newAttr]
else:
result[newAttr] = tmp
elif attr == 'undetected_referrer_samples':
newAttr = self.AttrSubstitution[
attr] if attr in self.AttrSubstitution else attr
tmp = self.formatArrayDateScoreHash(ip_report_api, attr)
self.updateScoredAttr(
newAttr, "benign",
len(self.getNumberBenign(ip_report_api, attr)))
if newAttr in result:
result[newAttr] = result[newAttr] + tmp
else:
result[newAttr] = tmp
elif attr == 'detected_downloaded_samples':
newAttr = self.AttrSubstitution[
attr] if attr in self.AttrSubstitution else attr
tmp = self.formatArrayDateScoreHash(ip_report_api, attr)
self.updateScoredAttr(
newAttr, "malicious",
len(self.getNumberMalicious(ip_report_api, attr)))
if newAttr in result:
result[newAttr] = tmp + result[newAttr]
else:
result[newAttr] = tmp
elif attr == 'undetected_downloaded_samples':
newAttr = self.AttrSubstitution[
attr] if attr in self.AttrSubstitution else attr
tmp = self.formatArrayDateScoreHash(ip_report_api, attr)
self.updateScoredAttr(
newAttr, "benign",
len(self.getNumberBenign(ip_report_api, attr)))
if newAttr in result:
result[newAttr] = result[newAttr] + tmp
else:
result[newAttr] = tmp
elif attr == 'detected_communicating_samples':
newAttr = self.AttrSubstitution[
attr] if attr in self.AttrSubstitution else attr
tmp = self.formatArrayDateScoreHash(ip_report_api, attr)
self.updateScoredAttr(
newAttr, "malicious",
len(self.getNumberMalicious(ip_report_api, attr)))
if newAttr in result:
result[newAttr] = tmp + result[newAttr]
else:
result[newAttr] = tmp
elif attr == 'undetected_communicating_samples':
newAttr = self.AttrSubstitution[
attr] if attr in self.AttrSubstitution else attr
tmp = self.formatArrayDateScoreHash(ip_report_api, attr)
self.updateScoredAttr(
newAttr, "benign",
len(self.getNumberBenign(ip_report_api, attr)))
if newAttr in result:
result[newAttr] = result[newAttr] + tmp
else:
result[newAttr] = tmp
elif attr == 'detected_urls':
newAttr = self.AttrSubstitution[
attr] if attr in self.AttrSubstitution else attr
tmp = self.formatArrayDateScoreURL(ip_report_api, attr)
self.updateScoredAttr(
newAttr, "malicious",
len(self.getNumberMalicious(ip_report_api, attr)))
if newAttr in result:
result[newAttr] = tmp + result[newAttr]
else:
result[newAttr] = tmp
elif attr == 'undetected_urls':
newAttr = self.AttrSubstitution[
attr] if attr in self.AttrSubstitution else attr
tmp = self.formatArrayDateScoreURLnum(ip_report_api, attr)
self.updateScoredAttr(
newAttr, "benign",
len(self.getNumberBenign(ip_report_api, attr)))
if newAttr in result:
result[newAttr] = result[newAttr] + tmp
else:
result[newAttr] = tmp
else:
result[attr] = ip_report_api[attr]
return result
def getOrdered(self, ip_report_filtered):
result = {}
#We order the Order's elements in the begining of the result list
for elem in self.Order:
#If the substitued element index exists in the Order list, then it should be ordered
if elem in self.AttrSubstitution and self.AttrSubstitution[
elem] and self.AttrSubstitution[
elem] in ip_report_filtered and ip_report_filtered[
self.AttrSubstitution[elem]]:
#Ordered elements should not duplicated
if self.AttrSubstitution[elem] not in result:
result[self.AttrSubstitution[elem]] = ip_report_filtered[
self.AttrSubstitution[elem]]
#If the index is not substitutable and if the index exists in the Order list, then it should be ordered
elif elem in ip_report_filtered and ip_report_filtered[elem]:
#Ordered elements should not duplicated
if elem not in result:
result[elem] = ip_report_filtered[elem]
#Then, we add the non ordered elements since they are not blacklisted so of the Order list is missing some elements, they will be added in the end of the result list
for attr in list(ip_report_filtered):
#Ordered elements should not duplicated
if attr not in result:
result[attr] = ip_report_filtered[attr]
return result
def getHTML(self, ip_report_filtered, ip):
html = ""
html = html + "<h3>IP Address: " + ip + "</h3>"
for elem in list(ip_report_filtered):
html = html + "<h4>" + elem + "</h4>"
if elem in self.number and self.number[elem]:
malicious = self.number[elem][
"malicious"] if "malicious" in self.number[elem] else 0
benign = self.number[elem][
"benign"] if "benign" in self.number[elem] else 0
html = html + "<h5>(<span style='color:red'>" + str(
malicious
) + " malicious</span> and <span style='color:green'>" + str(
benign) + " benign</span>)</h5>"
if elem in self.history and self.history[elem]:
history = self.history[elem]
html = html + "<h5>(" + str(history) + " found)</h5>"
html = html + json2html.convert(
json=ip_report_filtered[elem],
table_attributes='class="' + self.TablesClass + '"',
escape=False)
self.HTML = self.HTML + html
html = '<html><head>' + self.HTMLHeader + '</head><body>' + html
html = html + '</body></html>'
output = self.OutputDir + "/"
#print(html)
#imgkit.from_string(html, output+ip+'-VirusTotal.jpg')
with open(output + ip + '-VirusTotal.html', 'w') as HTMLFile:
HTMLFile.write(html)
#self.IMG=self.IMG+"<img src='"+ip+"-VirusTotal.jpg'><br/>"
def updateGeneralHTML(self):
HTMLPrefix = '<html><head>' + self.HTMLHeader + '</head><body>'
self.HTML = HTMLPrefix + self.HTML + '</body></html>'
self.IMG = HTMLPrefix + self.IMG + '</body></html>'
output = self.OutputDir + "/"
with open(output + 'latest-HTML-VirusTotal.html', 'w') as HTMLFile:
HTMLFile.write(self.HTML)
with open(output + 'latest-IMG-VirusTotal.html', 'w') as HTMLFile:
HTMLFile.write(self.IMG)
def findPersistedIP(self, ip_id, table_name):
self.CursorR.execute("SELECT * FROM " + table_name + " where ip_id='" +
str(ip_id) + "'")
return self.CursorR.fetchall()
def persistResolutions(self, selected_ips, ip_report_filtered):
attr = "resolutions"
table_name = "vt_scanned_resolutions_table"
newAttr = self.AttrSubstitution[
attr] if attr in self.AttrSubstitution else attr
selected_domains = self.findPersistedIP(selected_ips[0]['id'],
table_name)
selected_domains_filtered = []
for selected_domain in selected_domains:
selected_domains_filtered.append(selected_domain['domain'])
if newAttr in ip_report_filtered:
for domain in ip_report_filtered[newAttr]:
if domain['Domain'] not in selected_domains_filtered:
try:
self.CursorRW.execute(
"INSERT INTO " + table_name +
" (ip_id,domain,scanned_time) VALUES ('" +
str(selected_ips[0]['id']) + "','" +
domain['Domain'] + "','" +
domain['Date resolved'] + "')")
self.DBRW.commit()
self.resetSQL()
except Exception as e:
print("INSERT INTO " + table_name +
" (ip_id,domain,scanned_time) VALUES ('" +
str(selected_ips[0]['id']) + "','" +
domain['Domain'] + "','" +
domain['Date resolved'] + "')")
print("EXCEPTION: ", e)
self.resetSQL()
def persistURLs(self, selected_ips, ip_report_filtered):
attr = "detected_urls"
table_name = "vt_scanned_urls_table"
newAttr = self.AttrSubstitution[
attr] if attr in self.AttrSubstitution else attr
selected_urls = self.findPersistedIP(selected_ips[0]['id'], table_name)
selected_urls_filtered = []
for selected_url in selected_urls:
selected_urls_filtered.append(selected_url['url'])
if newAttr in ip_report_filtered:
for url in ip_report_filtered[newAttr]:
print(url['URL'])
if url['URL'] not in selected_urls_filtered:
try:
self.CursorRW.execute(
"INSERT INTO " + table_name +
" (ip_id,url,detections,scanned_time) VALUES ('" +
str(selected_ips[0]['id']) + "','" + url['URL'] +
"','" + url['Detections'] + "','" +
url['Scanned'] + "')")
self.DBRW.commit()
self.resetSQL()
except Exception as e:
print(
"INSERT INTO " + table_name +
" (ip_id,url,detections,scanned_time) VALUES ('" +
str(selected_ips[0]['id']) + "','" + url['URL'] +
"','" + url['Detections'] + "','" +
url['Scanned'] + "')")
print("EXCEPTION: ", e)
self.resetSQL()
def persistHashs(self, selected_ips, ip_report_filtered, attr, table_name):
newAttr = self.AttrSubstitution[
attr] if attr in self.AttrSubstitution else attr
selected_hashs = self.findPersistedIP(selected_ips[0]['id'],
table_name)
selected_hashs_filtered = []
for selected_hash in selected_hashs:
selected_hashs_filtered.append(selected_hash['hash'])
if newAttr in ip_report_filtered:
for hash in ip_report_filtered[newAttr]:
if hash['File Hash (sha256)'] not in selected_hashs_filtered:
try:
self.CursorRW.execute(
"INSERT INTO " + table_name +
" (ip_id,hash,detections,scanned_time) VALUES ('" +
str(selected_ips[0]['id']) + "','" +
hash['File Hash (sha256)'] + "','" +
hash['Detections'] + "','" + hash['Scanned'] +
"')")
self.DBRW.commit()
self.resetSQL()
except Exception as e:
print(
"INSERT INTO " + table_name +
" (ip_id,hash,detections,scanned_time) VALUES ('" +
str(selected_ips[0]['id']) + "','" +
hash['File Hash (sha256)'] + "','" +
hash['Detections'] + "','" + hash['Scanned'] +
"')")
print("EXCEPTION: ", e)
self.resetSQL()
def persist(self, ip_report_api, ip_report_filtered):
self.CursorR.execute(
"SELECT * FROM vt_scanned_ips_table where scanned_ip='" +
ip_report_api + "'")
selected_ips = self.CursorR.fetchall()
if len(selected_ips) == 0:
self.CursorRW.execute(
"INSERT INTO vt_scanned_ips_table (scanned_ip,last_scanned_time) VALUES ('"
+ ip_report_api + "','" + str(int(time.time())) + "')")
self.DBRW.commit()
self.resetSQL()
self.CursorR.execute(
"SELECT * FROM vt_scanned_ips_table where scanned_ip='" +
ip_report_api + "'")
selected_ips = self.CursorR.fetchall()
self.persistResolutions(selected_ips, ip_report_filtered)
self.persistURLs(selected_ips, ip_report_filtered)
attr = "detected_referrer_samples"
table_name = "vt_scanned_referring_files_table"
self.persistHashs(selected_ips, ip_report_filtered, attr, table_name)
attr = "detected_downloaded_samples"
table_name = "vt_scanned_downloads_table"
self.persistHashs(selected_ips, ip_report_filtered, attr, table_name)
attr = "detected_communicating_samples"
table_name = "vt_scanned_communicating_files_table"
self.persistHashs(selected_ips, ip_report_filtered, attr, table_name)
class VirusTotalAnalyzer(Analyzer):
def __init__(self):
Analyzer.__init__(self)
self.service = self.get_param('config.service', None, 'Service parameter is missing')
self.virustotal_key = self.get_param('config.key', None, 'Missing VirusTotal API key')
self.polling_interval = self.get_param('config.polling_interval', 60)
self.proxies = self.get_param('config.proxy', None)
self.vt = VirusTotalPublicApi(self.virustotal_key, self.proxies)
def wait_file_report(self, id):
results = self.check_response(self.vt.get_file_report(id))
code = results.get('response_code', None)
if code == 1:
self.report(results)
else:
time.sleep(self.polling_interval)
self.wait_file_report(id)
def wait_url_report(self, id):
results = self.check_response(self.vt.get_url_report(id))
code = results.get('response_code', None)
if code == 1 and (results.get('scan_id') == id):
self.report(results)
else:
time.sleep(self.polling_interval)
self.wait_url_report(id)
def check_response(self, response):
if type(response) is not dict:
self.error('Bad response : ' + str(response))
status = response.get('response_code', -1)
if status == 204:
self.error('VirusTotal api rate limit exceeded (Status 204).')
if status != 200:
self.error('Bad status : ' + str(status))
results = response.get('results', {})
if 'Missing IP address' in results.get('verbose_msg', ''):
results['verbose_msg'] = 'IP address not available in VirusTotal'
return results
# 0 => not found
# -2 => in queue
# 1 => ready
def read_scan_response(self, response, func):
results = self.check_response(response)
code = results.get('response_code', None)
scan_id = results.get('scan_id', None)
if code == 1 and scan_id is not None:
func(scan_id)
else:
self.error('Scan not found')
def summary(self, raw):
taxonomies = []
level = "info"
namespace = "VT"
predicate = "GetReport"
value = "0"
if self.service == "scan":
predicate = "Scan"
result = {
"has_result": True
}
if raw["response_code"] != 1:
result["has_result"] = False
result["positives"] = raw.get("positives", 0)
result["total"] = raw.get("total", 0)
if "scan_date" in raw:
result["scan_date"] = raw["scan_date"]
if self.service == "get":
if "scans" in raw:
result["scans"] = len(raw["scans"])
value = "{}/{}".format(result["positives"], result["total"])
if result["positives"] == 0:
level = "safe"
elif result["positives"] < 5:
level = "suspicious"
else:
level = "malicious"
if "resolutions" in raw:
result["resolutions"] = len(raw["resolutions"])
value = "{} resolution(s)".format(result["resolutions"])
if result["resolutions"] == 0:
level = "safe"
elif result["resolutions"] < 5:
level = "suspicious"
else:
level = "malicious"
if "detected_urls" in raw:
result["detected_urls"] = len(raw["detected_urls"])
value = "{} detected_url(s)".format(result["detected_urls"])
if result["detected_urls"] == 0:
level = "safe"
elif result["detected_urls"] < 5:
level = "suspicious"
else:
level = "malicious"
if "detected_downloaded_samples" in raw:
result["detected_downloaded_samples"] = len(
raw["detected_downloaded_samples"])
if self.service == "scan":
if "scans" in raw:
result["scans"] = len(raw["scans"])
value = "{}/{}".format(result["positives"], result["total"])
if result["positives"] == 0:
level = "safe"
elif result["positives"] < 5:
level = "suspicious"
else:
level = "malicious"
taxonomies.append(self.build_taxonomy(level, namespace, predicate, value))
return {"taxonomies": taxonomies}
def run(self):
if self.service == 'scan':
if self.data_type == 'file':
filename = self.get_param('filename', 'noname.ext')
filepath = self.get_param('file', None, 'File is missing')
self.read_scan_response(
self.vt.scan_file(filepath, from_disk=True, filename=filename),
self.wait_file_report
)
elif self.data_type == 'url':
data = self.get_param('data', None, 'Data is missing')
self.read_scan_response(
self.vt.scan_url(data), self.wait_url_report)
else:
self.error('Invalid data type')
elif self.service == 'get':
if self.data_type == 'domain':
data = self.get_param('data', None, 'Data is missing')
self.report(self.check_response(
self.vt.get_domain_report(data)))
elif self.data_type == 'fqdn':
data = self.get_param('data', None, 'Data is missing')
self.report(self.check_response(
self.vt.get_domain_report(data)))
elif self.data_type == 'ip':
data = self.get_param('data', None, 'Data is missing')
self.report(self.check_response(self.vt.get_ip_report(data)))
elif self.data_type == 'file':
hashes = self.get_param('attachment.hashes', None)
if hashes is None:
filepath = self.get_param('file', None, 'File is missing')
hash = hashlib.sha256(open(filepath, 'rb').read()).hexdigest()
else:
# find SHA256 hash
hash = next(h for h in hashes if len(h) == 64)
self.report(self.check_response(self.vt.get_file_report(hash)))
elif self.data_type == 'hash':
data = self.get_param('data', None, 'Data is missing')
self.report(self.check_response(self.vt.get_file_report(data)))
elif self.data_type == 'url':
data = self.get_param('data', None, 'Data is missing')
self.report(self.check_response(self.vt.get_url_report(data)))
else:
self.error('Invalid data type')
else:
self.error('Invalid service')
class VirusTotalAnalyzer(Analyzer):
def __init__(self):
Analyzer.__init__(self)
self.service = self.get_param("config.service", None,
"Service parameter is missing")
self.virustotal_key = self.get_param("config.key", None,
"Missing VirusTotal API key")
self.polling_interval = self.get_param("config.polling_interval", 60)
self.rescan_hash_older_than_days = self.get_param(
"config.rescan_hash_older_than_days", None)
self.highlighted_antivirus = self.get_param(
"config.highlighted_antivirus", None)
self.download_sample = self.get_param("config.download_sample", False)
self.download_sample_if_highlighted = self.get_param(
"config.download_sample_if_highlighted", False)
self.obs_path = None
self.proxies = self.get_param("config.proxy", None)
if (self.download_sample or self.download_sample_if_highlighted
or self.service == "download"):
self.vt_pay = PrivateApi(self.virustotal_key, self.proxies)
self.vt = PublicApi(self.virustotal_key, self.proxies)
def get_file(self, hash):
self.obs_path = "{}/{}".format(tempfile.gettempdir(), hash)
response = self.vt_pay.get_file(hash)
if response.get("response_code", None) == 200:
with open(self.obs_path, "wb") as f:
f.write(response["results"])
kind = filetype.guess(self.obs_path)
if kind and kind.extension != None:
os.rename(self.obs_path,
"{}.{}".format(self.obs_path, kind.extension))
self.obs_path = "{}.{}".format(self.obs_path, kind.extension)
def wait_file_report(self, id):
results = self.check_response(self.vt.get_file_report(id))
code = results.get("response_code", None)
if code == 1:
if self.data_type == "hash" and (
self.download_sample or
(self.download_sample_if_highlighted
and self.highlighted_antivirus and any([
results.get("scans", {}).get(av, {}).get(
"detected", None) == False
for av in self.highlighted_antivirus
]))):
self.get_file(self.get_param("data", None, "Data is missing"))
self.report(results)
else:
time.sleep(self.polling_interval)
self.wait_file_report(id)
def wait_url_report(self, id):
results = self.check_response(self.vt.get_url_report(id))
code = results.get("response_code", None)
if code == 1 and (results.get("scan_id") == id):
self.report(results)
else:
time.sleep(self.polling_interval)
self.wait_url_report(id)
def check_response(self, response):
if type(response) is not dict:
self.error("Bad response : " + str(response))
status = response.get("response_code", -1)
if status == 204:
self.error("VirusTotal api rate limit exceeded (Status 204).")
if status != 200:
self.error("Bad status : " + str(status))
results = response.get("results", {})
if "Missing IP address" in results.get("verbose_msg", ""):
results["verbose_msg"] = "IP address not available in VirusTotal"
return results
# 0 => not found
# -2 => in queue
# 1 => ready
def read_scan_response(self, response, func):
results = self.check_response(response)
code = results.get("response_code", None)
scan_id = results.get("scan_id", None)
if code == 1 and scan_id is not None:
func(scan_id)
else:
self.error("Scan not found")
def artifacts(self, raw):
artifacts = []
if self.obs_path:
tags = []
# This will work only in scan/rescan workflow, not in download only
if self.highlighted_antivirus:
for av in self.highlighted_antivirus:
detected = raw.get("scans",
{}).get(av, {}).get("detected", None)
if detected == False:
tags.append("to_{}".format(av))
artifacts.append(
self.build_artifact("file", self.obs_path, tags=tags))
return artifacts
def summary(self, raw):
taxonomies = []
level = "info"
namespace = "VT"
predicate = "GetReport"
value = "0"
if self.service == "scan":
predicate = "Scan"
elif self.service == "rescan":
predicate = "Rescan"
elif self.service == "download":
return {"taxonomies": taxonomies}
result = {"has_result": True}
if raw["response_code"] != 1:
result["has_result"] = False
result["positives"] = raw.get("positives", 0)
result["total"] = raw.get("total", 0)
if "scan_date" in raw:
result["scan_date"] = raw["scan_date"]
if self.service == "get":
if "scans" in raw:
result["scans"] = len(raw["scans"])
value = "{}/{}".format(result["positives"], result["total"])
if result["positives"] == 0:
level = "safe"
elif result["positives"] < 5:
level = "suspicious"
else:
level = "malicious"
if "resolutions" in raw:
result["resolutions"] = len(raw["resolutions"])
value = "{} resolution(s)".format(result["resolutions"])
if result["resolutions"] == 0:
level = "safe"
elif result["resolutions"] < 5:
level = "suspicious"
else:
level = "malicious"
if "detected_urls" in raw:
result["detected_urls"] = len(raw["detected_urls"])
value = "{} detected_url(s)".format(result["detected_urls"])
if result["detected_urls"] == 0:
level = "safe"
elif result["detected_urls"] < 5:
level = "suspicious"
else:
level = "malicious"
if "detected_downloaded_samples" in raw:
result["detected_downloaded_samples"] = len(
raw["detected_downloaded_samples"])
if self.service in ["scan", "rescan"]:
if "scans" in raw:
result["scans"] = len(raw["scans"])
value = "{}/{}".format(result["positives"], result["total"])
if result["positives"] == 0:
level = "safe"
elif result["positives"] < 5:
level = "suspicious"
else:
level = "malicious"
taxonomies.append(
self.build_taxonomy(level, namespace, predicate, value))
if self.highlighted_antivirus:
for av in self.highlighted_antivirus:
detected = raw.get("scans", {}).get(av,
{}).get("detected", None)
if detected == False:
taxonomies.append(
self.build_taxonomy("info", namespace, av,
"Not detected!"))
return {"taxonomies": taxonomies}
def run(self):
if self.service == "scan":
if self.data_type == "file":
filename = self.get_param("filename", "noname.ext")
filepath = self.get_param("file", None, "File is missing")
self.read_scan_response(
self.vt.scan_file(filepath,
from_disk=True,
filename=filename),
self.wait_file_report,
)
elif self.data_type == "url":
data = self.get_param("data", None, "Data is missing")
self.read_scan_response(self.vt.scan_url(data),
self.wait_url_report)
else:
self.error("Invalid data type")
elif self.service == "rescan":
if self.data_type == "hash":
data = self.get_param("data", None, "Data is missing")
self.read_scan_response(self.vt.rescan_file(data),
self.wait_file_report)
else:
self.error("Invalid data type")
elif self.service == "download":
if self.data_type == "hash":
data = self.get_param("data", None, "Data is missing")
self.get_file(data)
self.report({"message": "file downloaded"})
elif self.service == "get":
if self.data_type == "domain":
data = self.get_param("data", None, "Data is missing")
results = self.check_response(self.vt.get_domain_report(data))
elif self.data_type == "fqdn":
data = self.get_param("data", None, "Data is missing")
results = self.check_response(self.vt.get_domain_report(data))
elif self.data_type == "ip":
data = self.get_param("data", None, "Data is missing")
results = self.check_response(self.vt.get_ip_report(data))
elif self.data_type == "file":
hashes = self.get_param("attachment.hashes", None)
if hashes is None:
filepath = self.get_param("file", None, "File is missing")
hash = hashlib.sha256(open(filepath,
"rb").read()).hexdigest()
else:
hash = next(h for h in hashes if len(h) == 64)
results = self.check_response(self.vt.get_file_report(hash))
elif self.data_type == "hash":
data = self.get_param("data", None, "Data is missing")
results = self.check_response(self.vt.get_file_report(data))
elif self.data_type == "url":
data = self.get_param("data", None, "Data is missing")
results = self.check_response(self.vt.get_url_report(data))
else:
self.error("Invalid data type")
# if aged and enabled rescan
if self.data_type == "hash" and self.rescan_hash_older_than_days:
if (datetime.strptime(results["scan_date"],
"%Y-%m-%d %H:%M:%S") - datetime.now()
).days > self.rescan_hash_older_than_days:
self.read_scan_response(self.vt.rescan_file(data),
self.wait_file_report)
# download if hash, dangerous and not seen by av
if (self.data_type == "hash"
and (results.get("response_code", None) == 1)
and (results.get("positives", 0) >= 5)
and (self.download_sample or
(self.download_sample_if_highlighted
and self.highlighted_antivirus and any([
results.get("scans", {}).get(av, {}).get(
"detected", None) == False
for av in self.highlighted_antivirus
])))):
self.get_file(data)
self.report(results)
else:
self.error("Invalid service")
class VT(object):
"""
Class to hold VirusTotal items.
"""
#
# FUNCTIONS
#
"""
Sets up a VirusTotal object with the public api.
"""
def __init__(self, vtpublicapi):
self.vtpublicapi = vtpublicapi
self.vt = VirusTotalPublicApi(self.vtpublicapi)
def add_headers(self, inputheaders):
"""
Adds appropriate headers to input list.
"""
inputheaders.append('VirusTotal Detected URLs')
inputheaders.append('VirusTotal Detected Communicating Samples')
inputheaders.append('VirusTotal Detected Downloaded Samples')
inputheaders.append('VirusTotal Link')
def add_row(self, host, inputrow):
"""
Adds the pulled data to the input row.
"""
vtdetectedurls = vtdetectedcommunicatingsamples = \
vtdetecteddownloadedsamples = vturl = ''
if libs.network.IsIPv4(host):
vtresponse = self.vt.get_ip_report(host)
while "response_code" not in vtresponse or \
(vtresponse["response_code"] != 200 and
vtresponse["response_code"] != 403):
time.sleep(60) # Sleep for the API throttling
vtresponse = self.vt.get_ip_report(host)
if "results" not in vtresponse:
vtdetectedurls = "INVALID API KEY"
elif "detected_urls" in vtresponse["results"]:
vtdetectedurls = str(
len(vtresponse["results"]["detected_urls"]))
else:
vtdetectedurls = str(0)
if "results" not in vtresponse:
vtdetectedcommunicatingsamples = "INVALID API KEY"
elif "detected_communicating_samples" in vtresponse["results"]:
vtdetectedcommunicatingsamples = str(
len(vtresponse["results"]["detected_"
"communicating_"
"samples"]))
else:
vtdetectedcommunicatingsamples = str(0)
vturl = "https://www.virustotal.com/en/ip-address/{}/information/"\
.format(host)
else:
vtresponse = self.vt.get_domain_report(host)
while "response_code" not in vtresponse or \
(vtresponse["response_code"] != 200 and
vtresponse["response_code"] != 403):
time.sleep(60) # Sleep for the API throttling
vtresponse = self.vt.get_domain_report(host)
if "results" not in vtresponse:
vtdetectedurls = "INVALID API KEY"
elif "detected_urls" in vtresponse["results"]:
vtdetectedurls = str(
len(vtresponse["results"]["detected_urls"]))
else:
vtdetectedurls = str(0)
if "results" not in vtresponse:
vtdetectedcommunicatingsamples = "INVALID API KEY"
elif "detected_communicating_samples" in vtresponse["results"]:
vtdetectedcommunicatingsamples = str(
len(vtresponse["results"]["detected_"
"communicating_"
"samples"]))
else:
vtdetectedcommunicatingsamples = str(0)
if "results" not in vtresponse:
vtdetecteddownloadedsamples = "INVALID API KEY"
elif "detected_downloaded_samples" in vtresponse["results"]:
vtdetecteddownloadedsamples = str(
len(vtresponse["results"]["detected_"
"downloaded_"
"samples"]))
else:
vtdetecteddownloadedsamples = str(0)
vturl = "https://www.virustotal.com/en/domain/{}/information/"\
.format(host)
inputrow.append(vtdetectedurls)
inputrow.append(vtdetectedcommunicatingsamples)
inputrow.append(vtdetecteddownloadedsamples)
inputrow.append(vturl)
