def vuln_parse(vuln, fromapi=False):
"""Parses Nexpose vulnerability XML"""
if vuln is None: return False, False
vulnfields = {
'f_vulnid': vuln.attrib['id'].lower(),
'f_title': vuln.attrib['title'],
'f_severity': vuln.attrib['severity'],
'f_pci_sev': vuln.attrib['pciSeverity']
}
if 'published' in vuln.keys():
vulnfields['f_dt_published'] = vuln_time_convert(vuln.attrib['published'])
vulnfields['f_dt_added'] = vuln_time_convert(vuln.attrib['added'])
vulnfields['f_dt_modified'] = vuln_time_convert(vuln.attrib['modified'])
if 'cvssScore' in vuln.keys():
vulnfields['f_cvss_score'] = vuln.attrib['cvssScore']
cvss_vectors = vuln.attrib['cvssVector'] # cvssVector="(AV:N/AC:M/Au:N/C:P/I:P/A:P)"
vulnfields['f_cvss_av'] = cvss_vectors[4]
vulnfields['f_cvss_ac'] = cvss_vectors[9]
vulnfields['f_cvss_au'] = cvss_vectors[14]
vulnfields['f_cvss_c'] = cvss_vectors[18]
vulnfields['f_cvss_i'] = cvss_vectors[22]
vulnfields['f_cvss_a'] = cvss_vectors[26]
# parse the first description field, since there can only be one
d = vuln.find("description")
if d is not None:
if fromapi:
result = etree.tostring(d)
result = result.replace("<description>", "")
result = result.replace("</description>", "")
vulnfields['f_description'] = html_to_markmin(result)
else:
d = StringIO(etree.tostring(d))
vulnfields['f_description'] = html_to_markmin(nx_xml_to_html(d))
references = []
for d in vuln.findall("references/reference"):
references.append([d.attrib['source'], d.text])
# right now we don't do anything with tags
#tags = []
#for d in vuln.findall("tags/tag"):
# tags.append(d.text)
# parse the first solution field, since there can only be one
d = vuln.find("solution")
if d is not None:
if fromapi:
result = etree.tostring(d)
result = result.replace("<solution>", "")
result = result.replace("</solution>", "")
vulnfields['f_solution'] = html_to_markmin(result)
else:
d = StringIO(etree.tostring(d))
vulnfields['f_solution'] = html_to_markmin(nx_xml_to_html(d))
vulnfields['f_source'] = 'Nexpose'
return vulnfields, references
def process_xml(
filename=None,
asset_group=None,
engineer=None,
msf_settings={},
ip_ignore_list=None,
ip_include_list=None,
update_hosts=False,
):
# Upload and process Nexpose XML Scan file
from skaldship.cpe import lookup_cpe
from skaldship.hosts import get_host_record
from gluon.validators import IS_IPADDRESS
import os
db = current.globalenv['db']
session = current.globalenv['session']
parser = html.parser.HTMLParser()
user_id = db.auth_user(engineer)
# build the hosts only/exclude list
ip_exclude = []
if ip_ignore_list:
ip_exclude = ip_ignore_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
ip_only = []
if ip_include_list:
ip_only = ip_include_list.split('\r\n')
# TODO: check for ip subnet/range and break it out to individuals
log(" [*] Processing Nexpose scan file %s" % filename)
try:
nexpose_xml = etree.parse(filename)
except etree.ParseError as e:
msg = " [!] Invalid Nexpose XML file (%s): %s " % (filename, e)
log(msg, logging.ERROR)
return msg
root = nexpose_xml.getroot()
existing_vulnids = db(db.t_vulndata()).select(
db.t_vulndata.id, db.t_vulndata.f_vulnid).as_dict(key='f_vulnid')
log(" [*] Found %d vulnerabilities in the database already." %
len(existing_vulnids))
# start with the vulnerability details
vulns_added = 0
vulns_skipped = 0
vulns = root.findall("VulnerabilityDefinitions/vulnerability")
log(" [*] Parsing %d vulnerabilities" % len(vulns))
for vuln in vulns:
# nexpose identifiers are always lower case in kvasir. UPPER CASE IS FOR SHOUTING!!!
vulnid = vuln.attrib['id'].lower()
if vulnid in existing_vulnids:
#log(" [-] Skipping %s - It's in the db already" % vulnid)
vulns_skipped += 1
else:
# add the vulnerability to t_vulndata - any duplicates are errored out
(vulnfields, references) = vuln_parse(vuln, fromapi=False)
try:
vulnid = db.t_vulndata.update_or_insert(**vulnfields)
if not vulnid:
vulnid = db(db.t_vulndata.f_vulnid ==
vulnfields['f_vulnid']).select().first().id
vulns_added += 1
db.commit()
except Exception as e:
log(
" [!] Error inserting %s to vulndata: %s" %
(vulnfields['f_vulnid'], e), logging.ERROR)
vulnid = None
db.commit()
continue
# add the references
if vulnid is not None:
for reference in references:
# check to see if reference exists first
ref_id = db(db.t_vuln_refs.f_text == reference[1])
if ref_id.count() == 0:
# add because it doesn't
ref_id = db.t_vuln_refs.insert(f_source=reference[0],
f_text=reference[1])
db.commit()
else:
# pick the first reference as the ID
ref_id = ref_id.select()[0].id
# make many-to-many relationship with t_vuln_data
res = db.t_vuln_references.insert(f_vuln_ref_id=ref_id,
f_vulndata_id=vulnid)
db.commit()
log(" [*] %d Vulnerabilities added, %d skipped" %
(vulns_added, vulns_skipped))
# re-make the existing_vulnids dict() since we've updated the system
existing_vulnids = db(db.t_vulndata()).select(
db.t_vulndata.id, db.t_vulndata.f_vulnid).as_dict(key='f_vulnid')
# parse the nodes now
nodes = root.findall("nodes/node")
log(" [-] Parsing %d nodes" % len(nodes))
hoststats = {'added': 0, 'skipped': 0, 'updated': 0, 'errored': 0}
hosts = [] # array of host_id fields
for node in nodes:
log(" [-] Node %s status is: %s" %
(node.attrib['address'], node.attrib['status']))
#sys.stderr.write(msg)
if node.attrib['status'] != "alive":
hoststats['skipped'] += 1
continue
if node.attrib['address'] in ip_exclude:
log(" [-] Node is in exclude list... skipping")
hoststats['skipped'] += 1
continue
nodefields = {}
if len(ip_only) > 0 and node.attrib['address'] not in ip_only:
log(" [-] Node is not in the only list... skipping")
hoststats['skipped'] += 1
continue
# we'll just take the last hostname in the names list since it'll usually be the full dns name
names = node.findall("names/name")
for name in names:
nodefields['f_hostname'] = name.text
ip = node.attrib['address']
if IS_IPADDRESS()(ip):
nodefields['f_ipaddr'] = ip
else:
log(" [!] Invalid IP Address: %s" % ip, logging.ERROR)
nodefields['f_engineer'] = user_id
nodefields['f_asset_group'] = asset_group
nodefields['f_confirmed'] = False
if 'hardware-address' in node.attrib:
nodefields['f_macaddr'] = node.attrib['hardware-address']
if node.find('names/name') is not None:
# XXX: for now just take the first hostname
nodefields['f_hostname'] = node.find('names/name').text
# check to see if IP exists in DB already
query = (db.t_hosts.f_ipaddr == ip)
host_rec = db(query).select().first()
if host_rec is None:
host_id = db.t_hosts.insert(**nodefields)
db.commit()
hoststats['added'] += 1
log(" [-] Adding IP: %s" % ip)
elif update_hosts:
db.commit()
db(db.t_hosts.f_ipaddr == nodefields['f_ipaddr']).update(
**nodefields)
db.commit()
host_id = get_host_record(nodefields['f_ipaddr'])
host_id = host_id.id
hoststats['updated'] += 1
log(" [-] Updating IP: %s" % ip)
else:
hoststats['skipped'] += 1
db.commit()
log(" [-] Skipped IP: %s" % ip)
continue
hosts.append(host_id)
# tests that aren't specific to any port we wrap up into a meta service
# called "INFO"
tests = node.findall("tests/test")
if len(tests) > 0:
svc_id = db.t_services.update_or_insert(f_proto="info",
f_number="0",
f_status="info",
f_hosts_id=host_id)
db.commit()
for test in tests:
d = {}
vulnid = test.get('id').lower()
# we may have valid username.
if "cifs-acct-" in vulnid:
username = test.get('key')
if username is not None:
d['f_services_id'] = svc_id
d['f_username'] = username
d['f_active'] = True
d['f_source'] = vulnid
query = (db.t_accounts.f_services_id == d['f_services_id']) &\
(db.t_accounts.f_username == d['f_username'])
db.t_accounts.update_or_insert(query, **d)
db.commit()
if test.attrib['status'] == 'vulnerable-exploited' or \
test.attrib['status'] == 'potential' or \
test.attrib['status'] == 'exception-vulnerable-exploited' or \
test.attrib['status'] == 'exception-vulnerable-version' or \
test.attrib['status'] == 'exception-vulnerable-potential' or \
test.attrib['status'] == 'vulnerable-version':
if vulnid in existing_vulnids:
vuln_id = existing_vulnids[vulnid]['id']
else:
continue
if vulnid == 'cifs-nt-0001':
# Windows users, local groups, and global groups
infotext = nx_xml_to_html(
StringIO(etree.tostring(test, xml_declaration=False)))
try:
unames = re.search(
"Found user\(s\): (?P<unames>.+?) </li>",
infotext).group('unames')
except AttributeError as e:
# regex not found
continue
for uname in unames.split():
# add account
d['f_username'] = uname
d['f_services_id'] = svc_id
d['f_source'] = 'cifs-nt-0001'
db.t_accounts.update_or_insert(**d)
db.commit()
test_str = etree.tostring(test,
xml_declaration=False,
encoding=str)
test_str = test_str.encode('ascii', 'xmlcharrefreplace')
proof = nx_xml_to_html(StringIO(test_str))
proof = html_to_markmin(proof)
if vulnid == 'cifs-insecure-acct-lockout-limit':
d['f_hosts_id'] = host_id
try:
d['f_lockout_limit'] = re.search(
"contains: (?P<l>\d+)", proof).group('l')
except AttributeError:
d['f_lockout_limit'] = 0
query = (db.t_netbios.f_hosts_id == host_id)
db.t_netbios.update_or_insert(query, **d)
db.commit()
# Check for CIFS uid/pw
if "cifs-" in vulnid:
try:
uid = re.search("uid\[(?P<u>.*?)\]", proof).group('u')
pw = re.search("pw\[(?P<p>.*?)\]", proof).group('p')
realm = re.search("realm\[(?P<r>.*?)\]",
proof).group('r')
d = {
'f_services_id': svc_id,
'f_username': uid,
'f_password': pw,
'f_description': realm,
'f_active': True,
'f_compromised': True,
'f_source': vulnid
}
query = (db.t_accounts.f_services_id
== svc_id) & (db.t_accounts.f_username == uid)
db.t_accounts.update_or_insert(query, **d)
db.commit()
except AttributeError:
db.commit()
except Exception as e:
log("Error inserting account (%s): %s" % (uid, e),
logging.ERROR)
db.commit()
# solaris-kcms-readfile shadow file
if vulnid.lower() == "rpc-solaris-kcms-readfile":
# funky chicken stuff, if they mess with this output then we've got to
# change this around as well. thems the breaks, maynard!
shadow = parser.unescape(proof)
for line in shadow.split("<br />")[1:-1]:
user, pw, uid = line.split(':')[0:3]
d['f_services_id'] = svc_id
d['f_username'] = user
d['f_hash1'] = pw
d['f_hash1_type'] = "crypt"
d['f_uid'] = uid
d['f_source'] = "shadow"
d['f_active'] = True
d['f_source'] = "rpc-solaris-kcms-readfile"
query = (db.t_accounts.f_services_id == svc_id) & (
db.t_accounts.f_username == user)
db.t_accounts.update_or_insert(query, **d)
db.commit()
db.t_service_vulns.update_or_insert(
f_services_id=svc_id,
f_status=test.attrib['status'],
f_proof=proof,
f_vulndata_id=vuln_id)
if "cisco-default-http-account" in vulnid.lower():
d['f_services_id'] = svc_id
d['f_username'] = vulnid.split('-')[4]
d['f_password'] = vulnid.split('-')[6]
d['f_source'] = "cisco-default-http-account"
query = (db.t_accounts.f_services_id == svc_id) & (
db.t_accounts.f_username == d['f_username'])
db.t_accounts.update_or_insert(query, **d)
db.commit()
# add services (ports) and resulting vulndata
for endpoint in node.findall("endpoints/endpoint"):
f_proto = endpoint.attrib['protocol']
f_number = endpoint.attrib['port']
f_status = endpoint.attrib['status']
query = (db.t_services.f_hosts_id == host_id) \
& (db.t_services.f_proto == f_proto) \
& (db.t_services.f_number == f_number)
svc_id = db.t_services.update_or_insert(query,
f_proto=f_proto,
f_number=f_number,
f_status=f_status,
f_hosts_id=host_id)
if not svc_id:
svc_id = db(query).select().first().id
for service in endpoint.findall("services/service"):
d = {}
if 'name' in service.attrib:
db.t_services[svc_id] = dict(f_name=service.attrib['name'])
for test in service.findall("tests/test"):
vulnid = test.get('id').lower()
if test.attrib['status'] == 'vulnerable-exploited' or \
test.attrib['status'] == 'potential' or \
test.attrib['status'] == 'exception-vulnerable-exploited' or \
test.attrib['status'] == 'exception-vulnerable-version' or \
test.attrib['status'] == 'exception-vulnerable-potential' or \
test.attrib['status'] == 'vulnerable-version':
if vulnid in existing_vulnids:
vuln_id = existing_vulnids[vulnid]['id']
else:
log(
" [!] Unknown vulnid, Skipping! (id: %s)" %
vulnid, logging.ERROR)
continue
test_str = etree.tostring(test,
xml_declaration=False,
encoding=str)
test_str = test_str.encode('ascii',
'xmlcharrefreplace')
proof = nx_xml_to_html(StringIO(test_str))
proof = html_to_markmin(proof)
# Check for SNMP strings
if "snmp-read-" in vulnid:
snmpstring = re.search("pw\[(?P<pw>.*?)\]",
proof).group('pw')
db.t_snmp.update_or_insert(f_hosts_id=host_id,
f_community=snmpstring,
f_access="READ",
f_version="v1")
db.commit()
if "snmp-write" in vulnid:
snmpstring = re.search("pw\[(?P<pw>.*?)\]",
proof).group('pw')
db.t_snmp.update_or_insert(f_hosts_id=host_id,
f_community=snmpstring,
f_access="WRITE",
f_version="v1")
db.commit()
# TODO: account names
# Dell DRAC root/calvin
if vulnid == "http-drac-default-login":
d['f_services_id'] = svc_id
d['f_username'] = '******'
d['f_password'] = '******'
d['f_active'] = True
d['f_compromised'] = True
d['f_source'] = vulnid
query = (db.t_accounts.f_services_id == svc_id) & (
db.t_accounts.f_username == 'root')
db.t_accounts.update_or_insert(query, **d)
db.commit()
# Check for uid/pw
if "ftp-iis-" in vulnid or \
"telnet-" in vulnid or \
"cifs-" in vulnid or \
"tds-" in vulnid or \
"oracle-" in vulnid or \
"-default-" in vulnid or \
"ftp-generic-" in vulnid:
try:
uid = re.search("uid\[(?P<u>.*?)\]",
proof).group('u')
pw = re.search("pw\[(?P<p>.*?)\]",
proof).group('p')
realm = re.search("realm\[(?P<r>.*?)\]",
proof).group('r')
d['f_services_id'] = svc_id
d['f_username'] = uid
d['f_password'] = pw
d['f_description'] = realm
d['f_active'] = True
d['f_compromised'] = True
d['f_source'] = vulnid
query = (db.t_accounts.f_services_id
== svc_id) & (db.t_accounts.f_username
== uid)
db.t_accounts.update_or_insert(query, **d)
db.commit()
except AttributeError:
db.commit()
except Exception as e:
log(
"Error inserting account (%s): %s" %
(uid, e), logging.ERROR)
db.commit()
# cisco default http login accounts
if "cisco-default-http-account" in vulnid.lower():
d['f_services_id'] = svc_id
d['f_username'] = vulnid.split('-')[4]
d['f_password'] = vulnid.split('-')[6]
d['f_source'] = "cisco-default-http-account"
query = (db.t_accounts.f_services_id == svc_id) \
& (db.t_accounts.f_username == d['f_username'])
db.t_accounts.update_or_insert(query, **d)
db.commit()
db.t_service_vulns.update_or_insert(
f_services_id=svc_id,
f_status=test.attrib['status'],
f_proof=proof,
f_vulndata_id=vuln_id)
db.commit()
for config in service.findall("configuration/config"):
db.t_service_info.update_or_insert(
f_services_id=svc_id,
f_name=config.attrib['name'],
f_text=config.text)
db.commit()
if re.match('\w+.banner$', config.attrib['name']):
db.t_services[svc_id] = dict(f_banner=config.text)
db.commit()
if config.attrib['name'] == 'mac-address':
# update the mac address of the host
db.t_hosts[host_id] = dict(f_macaddr=config.text)
db.commit()
if "advertised-name" in config.attrib['name']:
# netbios computer name
d = config.text.split(" ")[0]
if "Computer Name" in config.text:
data = {'f_netbios_name': d}
# if hostname isn't defined then lowercase netbios name and put it in
if db.t_hosts[host_id].f_hostname is None:
data['f_hostname'] = d.lower()
db(db.t_hosts.id == host_id).update(**data)
db.commit()
elif "Domain Name" in config.text:
query = (db.t_netbios.f_hosts_id == host_id)
db.t_netbios.update_or_insert(query,
f_hosts_id=host_id,
f_domain=d)
db.commit()
for os_rec in node.findall('fingerprints/os'):
"""
<os certainty="1.00" device-class="Workstation" vendor="Microsoft" family="Windows" product="Windows 2000 Professional" version="SP4" arch="x86"/>
if using SCAP output the os line looks like:
<os certainty="0.66" device-class="General" vendor="Microsoft" family="Windows" product="Windows XP" arch="x86" cpe="cpe:/o:microsoft:windows_xp::sp3"/>
"""
if 'cpe' in os_rec.attrib:
# we have a cpe entry from xml! hooray!
cpe_name = os_rec.attrib['cpe'].replace('cpe:/o:', '')
os_id = lookup_cpe(cpe_name)
else:
# no cpe attribute in xml, go through our messy lookup
os_id = guess_cpe_os(os_rec)
if os_id is not None:
db.t_host_os_refs.update_or_insert(
f_certainty=os_rec.attrib['certainty'],
f_family=os_rec.get('family', 'Unknown'),
f_class=os_rec.get('device-class', 'Other'),
f_hosts_id=host_id,
f_os_id=os_id)
db.commit()
else:
log(
" [!] os_rec could not be parsed: %s" %
etree.tostring(os_rec), logging.ERROR)
db.commit()
if msf_settings.get('workspace'):
try:
# check to see if we have a Metasploit RPC instance configured and talking
from MetasploitProAPI import MetasploitProAPI
msf_api = MetasploitProAPI(host=msf_settings.get('url'),
apikey=msf_settings.get('key'))
working_msf_api = msf_api.login()
except Exception as error:
log(" [!] Unable to authenticate to MSF API: %s" % str(error),
logging.ERROR)
working_msf_api = False
try:
scan_data = open(filename, "r+").readlines()
except Exception as error:
log(
" [!] Error loading scan data to send to Metasploit: %s" %
str(error), logging.ERROR)
scan_data = None
if scan_data and working_msf_api:
task = msf_api.pro_import_data(
msf_settings.get('workspace'),
"".join(scan_data),
{
#'preserve_hosts': form.vars.preserve_hosts,
'blacklist_hosts': "\n".join(ip_ignore_list)
},
)
msf_workspace_num = session.msf_workspace_num or 'unknown'
msfurl = os.path.join(msf_settings.get('url'), 'workspaces',
msf_workspace_num, 'tasks', task['task_id'])
log(" [*] Added file to MSF Pro: %s" % msfurl)
# any new nexpose vulns need to be checked against exploits table and connected
log(" [*] Connecting exploits to vulns and performing do_host_status")
connect_exploits()
do_host_status(asset_group=asset_group)
msg = " [*] Import complete: hosts: %s added, %s skipped, %s errors - vulns: %s added, %s skipped" % (
hoststats['added'], hoststats['skipped'], hoststats['errored'],
vulns_added, vulns_skipped)
log(msg)
return msg
unames = re.search("Found user\(s\): (?P<unames>.+?) </li>", infotext).group('unames')
except AttributeError, e:
# regex not found
continue
for uname in unames.split():
# add account
d['f_username'] = uname
d['f_services_id'] = svc_id
d['f_source'] = 'cifs-nt-0001'
db.t_accounts.update_or_insert(**d)
db.commit()
test_str = etree.tostring(test, xml_declaration=False, encoding=unicode)
test_str = test_str.encode('ascii', 'xmlcharrefreplace')
proof = nx_xml_to_html(StringIO(test_str))
proof = html_to_markmin(proof)
if vulnid == 'cifs-insecure-acct-lockout-limit':
d['f_hosts_id'] = host_id
try:
d['f_lockout_limit'] = re.search("contains: (?P<l>\d+)", proof).group('l')
except AttributeError:
d['f_lockout_limit'] = 0
query = (db.t_netbios.f_hosts_id == host_id)
db.t_netbios.update_or_insert(query, **d)
db.commit()
# Check for CIFS uid/pw
if "cifs-" in vulnid:
try:
uid = re.search("uid\[(?P<u>.*?)\]", proof).group('u')
def vuln_parse(vuln, fromapi=False):
"""Parses Nexpose vulnerability XML"""
if vuln is None:
return (False, False)
vulnfields = {}
vulnfields["f_vulnid"] = vuln.attrib["id"].lower()
vulnfields["f_title"] = vuln.attrib["title"]
vulnfields["f_severity"] = vuln.attrib["severity"]
vulnfields["f_pci_sev"] = vuln.attrib["pciSeverity"]
if "published" in vuln.keys():
vulnfields["f_dt_published"] = vuln_time_convert(vuln.attrib["published"])
vulnfields["f_dt_added"] = vuln_time_convert(vuln.attrib["added"])
vulnfields["f_dt_modified"] = vuln_time_convert(vuln.attrib["modified"])
if "cvssScore" in vuln.keys():
vulnfields["f_cvss_score"] = vuln.attrib["cvssScore"]
cvss_vectors = vuln.attrib["cvssVector"] # cvssVector="(AV:N/AC:M/Au:N/C:P/I:P/A:P)"
vulnfields["f_cvss_av"] = cvss_vectors[4]
vulnfields["f_cvss_ac"] = cvss_vectors[9]
vulnfields["f_cvss_au"] = cvss_vectors[14]
vulnfields["f_cvss_c"] = cvss_vectors[18]
vulnfields["f_cvss_i"] = cvss_vectors[22]
vulnfields["f_cvss_a"] = cvss_vectors[26]
# print("Processing %s :: %s" % (vulnfields['f_vulnid'], vulnfields['f_title']))
# parse the first description field, since there can only be one
d = vuln.find("description")
if d is not None:
if fromapi:
result = etree.tostring(d)
result = result.replace("<description>", "")
result = result.replace("</description>", "")
vulnfields["f_description"] = result
else:
d = StringIO(etree.tostring(d))
vulnfields["f_description"] = html_to_markmin(nx_xml_to_html(d))
references = []
for d in vuln.findall("references/reference"):
references.append([d.attrib["source"], d.text])
# right now we don't do anything with tags
# tags = []
# for d in vuln.findall("tags/tag"):
# tags.append(d.text)
# parse the first solution field, since there can only be one
d = vuln.find("solution")
if d is not None:
if fromapi:
result = etree.tostring(d)
result = result.replace("<solution>", "")
result = result.replace("</solution>", "")
vulnfields["f_solution"] = result
else:
d = StringIO(etree.tostring(d))
vulnfields["f_solution"] = html_to_markmin(nx_xml_to_html(d))
return (vulnfields, references)
