def import_all_vulndata(overwrite=False, nexpose_server={}):
"""
Uses the NexposeAPI and imports each and every vulnerability to Kvasir. Can take a looooong time.
Args:
overwrite: Whether or not to overwrite an existing t_vulndata record
Returns:
msg: A string message of status.
"""
from NexposeAPI import VulnData
db = current.globalenv['db']
vuln_class = VulnData()
vuln_class.host = nexpose_server.get('host', 'localhost')
vuln_class.port = nexpose_server.get('port', '3780')
if vuln_class.login(user_id=nexpose_server.get('user'), password=nexpose_server.get('pw')):
log(" [*] Populating list of Nexpose vulnerability ID summaries")
try:
vuln_class.populate_summary()
except Exception, e:
log(" [!] Error populating summaries: %s" % str(e), logging.ERROR)
return False
try:
vulnxml = etree.parse(StringIO(vuln_class.vulnxml))
except Exception, e:
log(" [!] Error parsing summary XML: %s" % str(e), logging.ERROR)
return False
def vulnlist():
"""
Produces a list of Nexpose vulnids for a select/search box
"""
try:
from lxml import etree
except ImportError:
try:
from xml.etree import cElementTree as etree
except ImportError:
from xml.etree import ElementTree as etree
from NexposeAPI import VulnData
import os
import time
nexpose_config = nexpose_get_config()
vuln_class = VulnData()
vuln_class.host = nexpose_config['host']
vuln_class.port = nexpose_config['port']
nx_vuln_fname = os.path.join(request.folder, 'data',
'nexpose_vuln_summary.xml')
if os.path.exists(nx_vuln_fname):
# check to see if we should refresh the nexpose_vuln_summary.xml file
ctime = os.stat(nx_vuln_fname).st_ctime
if (time.time() - ctime >= 7500):
update_summary = True
else:
update_summary = False
else:
update_summary = True
if update_summary:
if vuln_class.login(user_id=nexpose_config['user'],
password=nexpose_config['password']):
# pull the list out
vuln_class.populate_summary()
fout = open(nx_vuln_fname, "wb+")
fout.writelines(vuln_class.vulnxml)
fout.close()
vulnxml = etree.parse(nx_vuln_fname)
vdata = []
counter = 0
for vuln in vulnxml.iterfind('.//VulnerabilitySummary[@id]'):
vdata.append([counter, vuln.get('id')])
return dict(data=vdata)
def vulnlist():
"""
Produces a list of Nexpose vulnids for a select/search box
"""
try:
from lxml import etree
except ImportError:
try:
from xml.etree import cElementTree as etree
except ImportError:
from xml.etree import ElementTree as etree
from NexposeAPI import VulnData
import os
import time
nexpose_config = nexpose_get_config()
vuln_class = VulnData()
vuln_class.host = nexpose_config["host"]
vuln_class.port = nexpose_config["port"]
nx_vuln_fname = os.path.join(request.folder, "data", "nexpose_vuln_summary.xml")
if os.path.exists(nx_vuln_fname):
# check to see if we should refresh the nexpose_vuln_summary.xml file
ctime = os.stat(nx_vuln_fname).st_ctime
if time.time() - ctime >= 7500:
update_summary = True
else:
update_summary = False
else:
update_summary = True
if update_summary:
if vuln_class.login(user_id=nexpose_config["user"], password=nexpose_config["password"]):
# pull the list out
vuln_class.populate_summary()
fout = open(nx_vuln_fname, "wb+")
fout.writelines(vuln_class.vulnxml)
fout.close()
vulnxml = etree.parse(nx_vuln_fname)
vdata = []
counter = 0
for vuln in vulnxml.iterfind(".//VulnerabilitySummary[@id]"):
vdata.append([counter, vuln.get("id")])
return dict(data=vdata)
def vulnlist():
"""Produces a list of Nexpose vulnids for a select/search box"""
from lxml import etree
from StringIO import StringIO
from NexposeAPI import VulnData
import os, time
vuln_class = VulnData()
vuln_class.user_id = auth.user.f_nexpose_user or 'nxadmin'
vuln_class.password = auth.user.f_nexpose_pw or 'password'
vuln_class.host = auth.user.f_nexpose_host or 'localhost'
vuln_class.port = auth.user.f_nexpose_port or '3780'
nx_vuln_fname = os.path.join(request.folder, 'data', 'nexpose_vuln_summary.xml')
if os.path.exists(nx_vuln_fname):
# check to see if we should refresh the nexpose_vuln_summary.xml file
ctime = os.stat(nx_vuln_fname).st_ctime
if (time.time() - ctime >= 7500):
update_summary = True
else:
update_summary = False
else:
update_summary = True
if update_summary:
if vuln_class.login():
# pull the list out
vuln_class.populate_summary()
fout = open(nx_vuln_fname, "wb+")
fout.writelines(vuln_class.vulnxml)
fout.close()
vulnxml = etree.parse(nx_vuln_fname)
vdata = []
counter = 0
for vuln in vulnxml.iterfind('.//VulnerabilitySummary[@id]'):
vdata.append([counter, vuln.get('id')])
return dict(data=vdata)
def import_vulnid():
"""
Downloads the detailed vulnerability data from Nexpose based on
a vuln id passed to it
"""
form = SQLFORM.factory(
Field('nexid', 'string', label=T('Nexpose ID')),
Field('nexid_list', 'text', label=T('Nexpose ID List')))
response.title = "%s :: Import Nexpose VulnID" % settings.title
nexpose_config = nexpose_get_config()
if form.process().accepted:
from NexposeAPI import VulnData
from skaldship.nexpose import vuln_parse
nxvulns = VulnData()
nxvulns.host = nexpose_config['host']
nxvulns.port = nexpose_config['port']
nexpose_ids = []
if form.vars.nexid:
nexpose_ids.extend([form.vars.nexid])
if form.vars.nexid_list:
nexpose_ids.extend(form.vars.nexid_list.split('\r\n'))
res = nxvulns.login(user_id=nexpose_config['user'],
password=nexpose_config['password'])
if res:
stats = {'added': 0, 'invalid': 0}
for nexid in nexpose_ids:
vulndetails = nxvulns.detail(nexid)
if vulndetails is not None:
(vulnfields, references) = vuln_parse(
vulndetails.find('Vulnerability'), fromapi=True)
else:
stats['invalid'] += 1
continue
# add the vulnerability to t_vulndata
query = (db.t_vulndata.f_vulnid == nexid)
vulnid = db.t_vulndata.update_or_insert(query, **vulnfields)
if not vulnid:
row = db(query).select().first()
if row:
vulnid = row.id
else:
log(" [!] Could not find %s in database.." % nexid,
logging.WARN)
stats['invalid'] += 1
continue
db.commit()
# add the references
if vulnid is not None and references:
for reference in references:
# check to see if reference exists first
query = (db.t_vuln_refs.f_source == reference[0]) & (
db.t_vuln_refs.f_text == reference[1])
ref_id = db.t_vuln_refs.update_or_insert(
query, f_source=reference[0], f_text=reference[1])
if not ref_id:
ref_id = db(query).select().first().id
# make many-to-many relationship with t_vuln_data
db.t_vuln_references.update_or_insert(
f_vuln_ref_id=ref_id, f_vulndata_id=vulnid)
db.commit()
from skaldship.exploits import connect_exploits
connect_exploits()
log(" [-] Added Nexpose vulnerability: %s" % nexid)
stats['added'] += 1
response.flash = "%s added, %s skipped" % (stats['added'],
stats['invalid'])
return dict(form=form)
else:
response.flash = "Unable to login to Nexpose"
elif form.errors:
response.flash = "Error in form"
return dict(form=form)
def import_vulnid():
"""
Downloads the detailed vulnerability data from Nexpose based on
a vuln id passed to it
"""
form = SQLFORM.factory(
Field("nexid", "string", label=T("Nexpose ID")), Field("nexid_list", "text", label=T("Nexpose ID List"))
)
response.title = "%s :: Import Nexpose VulnID" % settings.title
nexpose_config = nexpose_get_config()
if form.process().accepted:
from NexposeAPI import VulnData
from skaldship.nexpose import vuln_parse
nxvulns = VulnData()
nxvulns.host = nexpose_config["host"]
nxvulns.port = nexpose_config["port"]
nexpose_ids = []
if form.vars.nexid:
nexpose_ids.extend([form.vars.nexid])
if form.vars.nexid_list:
nexpose_ids.extend(form.vars.nexid_list.split("\r\n"))
res = nxvulns.login(user_id=nexpose_config["user"], password=nexpose_config["password"])
if res:
stats = {"added": 0, "invalid": 0}
for nexid in nexpose_ids:
vulndetails = nxvulns.detail(nexid)
if vulndetails is not None:
(vulnfields, references) = vuln_parse(vulndetails.find("Vulnerability"), fromapi=True)
else:
stats["invalid"] += 1
continue
# add the vulnerability to t_vulndata
query = db.t_vulndata.f_vulnid == nexid
vulnid = db.t_vulndata.update_or_insert(query, **vulnfields)
if not vulnid:
row = db(query).select().first()
if row:
vulnid = row.id
else:
log(" [!] Could not find %s in database.." % nexid, logging.WARN)
stats["invalid"] += 1
continue
db.commit()
# add the references
if vulnid is not None and references:
for reference in references:
# check to see if reference exists first
query = (db.t_vuln_refs.f_source == reference[0]) & (db.t_vuln_refs.f_text == reference[1])
ref_id = db.t_vuln_refs.update_or_insert(query, f_source=reference[0], f_text=reference[1])
if not ref_id:
ref_id = db(query).select().first().id
# make many-to-many relationship with t_vuln_data
db.t_vuln_references.update_or_insert(f_vuln_ref_id=ref_id, f_vulndata_id=vulnid)
db.commit()
from skaldship.exploits import connect_exploits
connect_exploits()
log(" [-] Added Nexpose vulnerability: %s" % nexid)
stats["added"] += 1
response.flash = "%s added, %s skipped" % (stats["added"], stats["invalid"])
return dict(form=form)
else:
response.flash = "Unable to login to Nexpose"
elif form.errors:
response.flash = "Error in form"
return dict(form=form)
def get_nexpose_vulndata():
"""Downloads the detailed vulnerability data from Nexpose based on
a vulnid passed to it"""
form = SQLFORM.factory(
Field('nexid', 'string', label=T('Nexpose ID')),
Field('update', 'boolean', label=T('Update existing')),
)
if form.accepts(request, session):
nxvulns = VulnData()
nxvulns.user_id = auth.user.f_nexpose_user or 'nxadmin'
nxvulns.password = auth.user.f_nexpose_pw or 'password'
nxvulns.host = auth.user.f_nexpose_host or 'localhost'
nxvulns.port = auth.user.f_nexpose_port or '3780'
if nxvulns.login():
vulndetails = nxvulns.detail(form.vars.nexid)
(vulnfields, references) = vuln_parse(vulndetails.find('Vulnerability'), fromapi=True)
if not vulnfields:
response.flash = "Invalid Nexpose ID"
return dict(form=form)
# add the vulnerability to t_vulndata
try:
vulnid = db.t_vulndata.insert(**vulnfields)
response.flash("%s added to vulndb" % (form.vars.nexid))
db.commit()
except Exception, e:
if form.vars.update:
try:
row = db(db.t_vulndata.f_vulnid == vulnfields['f_vulnid']).select().first()
row.update_record(**vulnfields)
vuln_id = row.id
response.flash("%s updated in vulndb" % (form.vars.nexid))
db.commit()
except Exception, e:
msg = "Error inserting %s to vulndata: %s" % (form.vars.nexid, e)
response.flash(msg)
logger.info(msg)
vulnid = None
db.commit()
else:
msg = "Error inserting %s to vulndata: %s" % (form.vars.nexid, e)
response.flash(msg)
logger.info(msg)
vulnid = None
# add the references
if vulnid is not None and references:
for reference in references:
# check to see if reference exists first
ref_id = db(db.t_vuln_refs.f_text == reference[1])
if ref_id.count() == 0:
# add because it doesn't
ref_id = db.t_vuln_refs.insert(f_source=reference[0], f_text=reference[1])
else:
# pick the first reference as the ID
ref_id = ref_id.select().first().id
# make many-to-many relationship with t_vuln_data
res = db.t_vuln_references.insert(f_vuln_ref_id=ref_id, f_vulndata_id=vulnid)
db.commit()
def import_all_vulndata(overwrite=False, nexpose_server={}):
"""
Uses the NexposeAPI and imports each and every vulnerability to Kvasir. Can take a looooong time.
Args:
overwrite: Whether or not to overwrite an existing t_vulndata record
Returns:
msg: A string message of status.
"""
from NexposeAPI import VulnData
db = current.globalenv['db']
vuln_class = VulnData()
vuln_class.host = nexpose_server.get('host', 'localhost')
vuln_class.port = nexpose_server.get('port', '3780')
if vuln_class.login(user_id=nexpose_server.get('user'),
password=nexpose_server.get('pw')):
log(" [*] Populating list of Nexpose vulnerability ID summaries")
try:
vuln_class.populate_summary()
except Exception as e:
log(" [!] Error populating summaries: %s" % str(e), logging.ERROR)
return False
try:
vulnxml = etree.parse(StringIO(vuln_class.vulnxml))
except Exception as e:
log(" [!] Error parsing summary XML: %s" % str(e), logging.ERROR)
return False
vulns = vulnxml.findall('VulnerabilitySummary')
log(" [*] %s vulnerabilities to parse" % len(vulns))
if vuln_class.vulnerabilities > 0:
existing_vulnids = []
[
existing_vulnids.extend([x['f_vulnid']])
for x in db(db.t_vulndata.f_source == "Nexpose").select(
db.t_vulndata.f_vulnid).as_list()
]
log(" [*] Found %d vulnerabilities in the database already." %
(len(existing_vulnids)))
stats = {'added': 0, 'updated': 0, 'skipped': 0, 'errors': 0}
for vuln in vulns:
if vuln.attrib['id'] in existing_vulnids and not overwrite:
# skip over existing entries if we're not overwriting
stats['skipped'] += 1
continue
try:
vulndetails = vuln_class.detail(vuln.attrib['id'])
except Exception as e:
log(
" [!] Error retrieving details for %s: %s" %
(vuln.attrib['id'], str(e)), logging.ERROR)
stats['errors'] += 1
if stats['errors'] == 50:
log(" [!] Too many errors, aborting!", logging.ERROR)
return False
else:
continue
if vulndetails is not None:
(vulnfields, references) = vuln_parse(
vulndetails.find('Vulnerability'), fromapi=True)
else:
log(
" [!] Unable to find %s in Nexpose" %
vuln.attrib['id'], logging.WARN)
continue
# add the vulnerability to t_vulndata
vulnid = db.t_vulndata.update_or_insert(**vulnfields)
if not vulnid:
vulnid = db(db.t_vulndata.f_vulnid ==
vulnfields['f_vulnid']).select().first().id
stats['updated'] += 1
log(" [-] Updated %s" % vulnfields['f_vulnid'])
else:
stats['added'] += 1
log(" [-] Added %s" % vulnfields['f_vulnid'])
db.commit()
# add the references
if vulnid is not None and references:
for reference in references:
# check to see if reference exists first
query = (db.t_vuln_refs.f_source == reference[0]) & (
db.t_vuln_refs.f_text == reference[1])
ref_id = db.t_vuln_refs.update_or_insert(
query, f_source=reference[0], f_text=reference[1])
if not ref_id:
ref_id = db(query).select().first().id
# make many-to-many relationship with t_vuln_data
db.t_vuln_references.update_or_insert(
f_vuln_ref_id=ref_id, f_vulndata_id=vulnid)
db.commit()
from skaldship.exploits import connect_exploits
connect_exploits()
msg = "%s added, %s updated, %s skipped" % (
stats['added'], stats['updated'], stats['skipped'])
log(" [*] %s" % msg)
else:
msg = "No vulndata populated from Nexpose"
log(" [!] Error: %s" % msg, logging.ERROR)
else:
msg = "Unable to communicate with Nexpose"
log(" [!] Error: %s" % msg, logging.ERROR)
return msg
def vuln_update():
# Update t_vulndata with vulndata from Nexpose
# Requires username/password and hostname of a Nexpose
# https instance. User can permit overwrite (updating)
# the data if a Vulnerability ID exists in the db.
from lxml import etree
from StringIO import StringIO
from NexposeAPI import NexposeAPI, VulnData
response.title = "%s :: Nexpose Vulnerability Update" % (settings.title)
form = SQLFORM.factory(
Field('hostname', default=auth.user.f_nexpose_host or 'localhost', requires=IS_NOT_EMPTY()),
Field('port', default=auth.user.f_nexpose_port or '3780', requires=IS_NOT_EMPTY()),
Field('username', default=auth.user.f_nexpose_user or 'nxadmin', requires=IS_NOT_EMPTY()),
Field('password', 'password', default=auth.user.f_nexpose_pw, requires=IS_NOT_EMPTY()),
Field('overwrite', 'boolean', default=False, label=T('Overwrite existing entries')),
)
if form.accepts(request.vars):
napi = NexposeAPI()
napi.user_id = form.vars.username
napi.password = form.vars.password
napi.host = form.vars.hostname
napi.port = form.vars.port
if napi.login():
# print("Logged in to Nexpose API")
vuln_class = VulnData(napi.sessionid)
vuln_class.populate_summary()
if vuln_class.vulnerabilities > 0:
existing_vulnids = []
for r in db(db.t_vulndata()).select(db.t_vulndata.f_vulnid):
existing_vulnids.append(r.f_vulnid)
logger.info("Found %d vulnerabilities in the database already." % (len(existing_vulnids)))
vulnxml = etree.parse(StringIO(vuln_class.vulnxml))
vulns_added = 0
vulns_updated = 0
vulns_skipped = 0
for vuln in vulnxml.findall('VulnerabilitySummary'):
if vuln.attrib['id'] in existing_vulnids and not request.vars.overwrite:
# skip over existing entries if we're not overwriting
continue
try:
vulndetails = vuln_class.detail(vuln.attrib['id'])
except:
try:
vulndetails = vuln_class.detail(vuln.attrib['id'])
except:
try:
vulndetails = vuln_class.detail(vuln.attrib['id'])
except:
raise
(vulnfields, references) = vuln_parse(vulndetails.find('Vulnerability'), fromapi=True)
if not vulnfields:
continue
# add the vulnerability to t_vulndata
try:
vulnid = db.t_vulndata.insert(**vulnfields)
vulns_added += 1
db.commit()
except Exception, e:
if request.vars.overwrite:
try:
row = db(db.t_vulndata.f_vulnid == vulnfields['f_vulnid']).select().first()
row.update_record(**vulnfields)
vulnid = row.id
vulns_updated += 1
db.commit()
except Exception, e:
logger.info("Error inserting %s to vulndata: %s" % (vulnfields['f_vulnid'], e))
vulnid = None
vulns_skipped += 1
db.commit()
continue
else:
logger.info("Error inserting %s to vulndata: %s" % (vulnfields['f_vulnid'], e))
vulnid = None
vulns_skipped += 1
db.commit()
continue
# add the references
if vulnid is not None and references:
for reference in references:
# check to see if reference exists first
ref_id = db(db.t_vuln_refs.f_text == reference[1])
if ref_id.count() == 0:
# add because it doesn't
ref_id = db.t_vuln_refs.insert(f_source=reference[0], f_text=reference[1])
else:
# pick the first reference as the ID
ref_id = ref_id.select().first().id
# make many-to-many relationship with t_vuln_data
res = db.t_vuln_references.insert(f_vuln_ref_id=ref_id, f_vulndata_id=vulnid)
db.commit()
logger.info("%d vulns added, %d updated, %d skipped" % (vulns_added, vulns_updated, vulns_skipped))
response.flash = "Completed - (A:%s/U:%s/S:%s)" % (vulns_added, vulns_updated, vulns_skipped)
