def import_all_vulndata(overwrite=False, nexpose_server={}): """ Uses the NexposeAPI and imports each and every vulnerability to Kvasir. Can take a looooong time. Args: overwrite: Whether or not to overwrite an existing t_vulndata record Returns: msg: A string message of status. """ from NexposeAPI import VulnData db = current.globalenv['db'] vuln_class = VulnData() vuln_class.host = nexpose_server.get('host', 'localhost') vuln_class.port = nexpose_server.get('port', '3780') if vuln_class.login(user_id=nexpose_server.get('user'), password=nexpose_server.get('pw')): log(" [*] Populating list of Nexpose vulnerability ID summaries") try: vuln_class.populate_summary() except Exception, e: log(" [!] Error populating summaries: %s" % str(e), logging.ERROR) return False try: vulnxml = etree.parse(StringIO(vuln_class.vulnxml)) except Exception, e: log(" [!] Error parsing summary XML: %s" % str(e), logging.ERROR) return False
def vulnlist(): """ Produces a list of Nexpose vulnids for a select/search box """ try: from lxml import etree except ImportError: try: from xml.etree import cElementTree as etree except ImportError: from xml.etree import ElementTree as etree from NexposeAPI import VulnData import os import time nexpose_config = nexpose_get_config() vuln_class = VulnData() vuln_class.host = nexpose_config['host'] vuln_class.port = nexpose_config['port'] nx_vuln_fname = os.path.join(request.folder, 'data', 'nexpose_vuln_summary.xml') if os.path.exists(nx_vuln_fname): # check to see if we should refresh the nexpose_vuln_summary.xml file ctime = os.stat(nx_vuln_fname).st_ctime if (time.time() - ctime >= 7500): update_summary = True else: update_summary = False else: update_summary = True if update_summary: if vuln_class.login(user_id=nexpose_config['user'], password=nexpose_config['password']): # pull the list out vuln_class.populate_summary() fout = open(nx_vuln_fname, "wb+") fout.writelines(vuln_class.vulnxml) fout.close() vulnxml = etree.parse(nx_vuln_fname) vdata = [] counter = 0 for vuln in vulnxml.iterfind('.//VulnerabilitySummary[@id]'): vdata.append([counter, vuln.get('id')]) return dict(data=vdata)
def vulnlist(): """ Produces a list of Nexpose vulnids for a select/search box """ try: from lxml import etree except ImportError: try: from xml.etree import cElementTree as etree except ImportError: from xml.etree import ElementTree as etree from NexposeAPI import VulnData import os import time nexpose_config = nexpose_get_config() vuln_class = VulnData() vuln_class.host = nexpose_config["host"] vuln_class.port = nexpose_config["port"] nx_vuln_fname = os.path.join(request.folder, "data", "nexpose_vuln_summary.xml") if os.path.exists(nx_vuln_fname): # check to see if we should refresh the nexpose_vuln_summary.xml file ctime = os.stat(nx_vuln_fname).st_ctime if time.time() - ctime >= 7500: update_summary = True else: update_summary = False else: update_summary = True if update_summary: if vuln_class.login(user_id=nexpose_config["user"], password=nexpose_config["password"]): # pull the list out vuln_class.populate_summary() fout = open(nx_vuln_fname, "wb+") fout.writelines(vuln_class.vulnxml) fout.close() vulnxml = etree.parse(nx_vuln_fname) vdata = [] counter = 0 for vuln in vulnxml.iterfind(".//VulnerabilitySummary[@id]"): vdata.append([counter, vuln.get("id")]) return dict(data=vdata)
def vulnlist(): """ Produces a list of Nexpose vulnids for a select/search box """ try: from lxml import etree except ImportError: try: from xml.etree import cElementTree as etree except ImportError: from xml.etree import ElementTree as etree from NexposeAPI import VulnData import os import time vuln_class = VulnData() vuln_class.host = auth.user.f_nexpose_host or 'localhost' vuln_class.port = auth.user.f_nexpose_port or '3780' nx_vuln_fname = os.path.join(request.folder, 'data', 'nexpose_vuln_summary.xml') if os.path.exists(nx_vuln_fname): # check to see if we should refresh the nexpose_vuln_summary.xml file ctime = os.stat(nx_vuln_fname).st_ctime if (time.time() - ctime >= 7500): update_summary = True else: update_summary = False else: update_summary = True if update_summary: if vuln_class.login(user_id=auth.user.f_nexpose_user, password=auth.user.f_nexpose_pw): # pull the list out vuln_class.populate_summary() fout = open(nx_vuln_fname, "wb+") fout.writelines(vuln_class.vulnxml) fout.close() vulnxml = etree.parse(nx_vuln_fname) vdata = [] counter = 0 for vuln in vulnxml.iterfind('.//VulnerabilitySummary[@id]'): vdata.append([counter, vuln.get('id')]) return dict(data=vdata)
def import_all_vulndata(overwrite=False, nexpose_server={}): """ Uses the NexposeAPI and imports each and every vulnerability to Kvasir. Can take a looooong time. Args: overwrite: Whether or not to overwrite an existing t_vulndata record Returns: msg: A string message of status. """ from NexposeAPI import VulnData db = current.globalenv['db'] vuln_class = VulnData() vuln_class.host = nexpose_server.get('host', 'localhost') vuln_class.port = nexpose_server.get('port', '3780') if vuln_class.login(user_id=nexpose_server.get('user'), password=nexpose_server.get('pw')): log(" [*] Populating list of Nexpose vulnerability ID summaries") try: vuln_class.populate_summary() except Exception as e: log(" [!] Error populating summaries: %s" % str(e), logging.ERROR) return False try: vulnxml = etree.parse(StringIO(vuln_class.vulnxml)) except Exception as e: log(" [!] Error parsing summary XML: %s" % str(e), logging.ERROR) return False vulns = vulnxml.findall('VulnerabilitySummary') log(" [*] %s vulnerabilities to parse" % len(vulns)) if vuln_class.vulnerabilities > 0: existing_vulnids = [] [ existing_vulnids.extend([x['f_vulnid']]) for x in db(db.t_vulndata.f_source == "Nexpose").select( db.t_vulndata.f_vulnid).as_list() ] log(" [*] Found %d vulnerabilities in the database already." % (len(existing_vulnids))) stats = {'added': 0, 'updated': 0, 'skipped': 0, 'errors': 0} for vuln in vulns: if vuln.attrib['id'] in existing_vulnids and not overwrite: # skip over existing entries if we're not overwriting stats['skipped'] += 1 continue try: vulndetails = vuln_class.detail(vuln.attrib['id']) except Exception as e: log( " [!] Error retrieving details for %s: %s" % (vuln.attrib['id'], str(e)), logging.ERROR) stats['errors'] += 1 if stats['errors'] == 50: log(" [!] Too many errors, aborting!", logging.ERROR) return False else: continue if vulndetails is not None: (vulnfields, references) = vuln_parse( vulndetails.find('Vulnerability'), fromapi=True) else: log( " [!] Unable to find %s in Nexpose" % vuln.attrib['id'], logging.WARN) continue # add the vulnerability to t_vulndata vulnid = db.t_vulndata.update_or_insert(**vulnfields) if not vulnid: vulnid = db(db.t_vulndata.f_vulnid == vulnfields['f_vulnid']).select().first().id stats['updated'] += 1 log(" [-] Updated %s" % vulnfields['f_vulnid']) else: stats['added'] += 1 log(" [-] Added %s" % vulnfields['f_vulnid']) db.commit() # add the references if vulnid is not None and references: for reference in references: # check to see if reference exists first query = (db.t_vuln_refs.f_source == reference[0]) & ( db.t_vuln_refs.f_text == reference[1]) ref_id = db.t_vuln_refs.update_or_insert( query, f_source=reference[0], f_text=reference[1]) if not ref_id: ref_id = db(query).select().first().id # make many-to-many relationship with t_vuln_data db.t_vuln_references.update_or_insert( f_vuln_ref_id=ref_id, f_vulndata_id=vulnid) db.commit() from skaldship.exploits import connect_exploits connect_exploits() msg = "%s added, %s updated, %s skipped" % ( stats['added'], stats['updated'], stats['skipped']) log(" [*] %s" % msg) else: msg = "No vulndata populated from Nexpose" log(" [!] Error: %s" % msg, logging.ERROR) else: msg = "Unable to communicate with Nexpose" log(" [!] Error: %s" % msg, logging.ERROR) return msg
def vuln_update(): # Update t_vulndata with vulndata from Nexpose # Requires username/password and hostname of a Nexpose # https instance. User can permit overwrite (updating) # the data if a Vulnerability ID exists in the db. from lxml import etree from StringIO import StringIO from NexposeAPI import NexposeAPI, VulnData response.title = "%s :: Nexpose Vulnerability Update" % (settings.title) form = SQLFORM.factory( Field('hostname', default=auth.user.f_nexpose_host or 'localhost', requires=IS_NOT_EMPTY()), Field('port', default=auth.user.f_nexpose_port or '3780', requires=IS_NOT_EMPTY()), Field('username', default=auth.user.f_nexpose_user or 'nxadmin', requires=IS_NOT_EMPTY()), Field('password', 'password', default=auth.user.f_nexpose_pw, requires=IS_NOT_EMPTY()), Field('overwrite', 'boolean', default=False, label=T('Overwrite existing entries')), ) if form.accepts(request.vars): napi = NexposeAPI() napi.user_id = form.vars.username napi.password = form.vars.password napi.host = form.vars.hostname napi.port = form.vars.port if napi.login(): # print("Logged in to Nexpose API") vuln_class = VulnData(napi.sessionid) vuln_class.populate_summary() if vuln_class.vulnerabilities > 0: existing_vulnids = [] for r in db(db.t_vulndata()).select(db.t_vulndata.f_vulnid): existing_vulnids.append(r.f_vulnid) logger.info("Found %d vulnerabilities in the database already." % (len(existing_vulnids))) vulnxml = etree.parse(StringIO(vuln_class.vulnxml)) vulns_added = 0 vulns_updated = 0 vulns_skipped = 0 for vuln in vulnxml.findall('VulnerabilitySummary'): if vuln.attrib['id'] in existing_vulnids and not request.vars.overwrite: # skip over existing entries if we're not overwriting continue try: vulndetails = vuln_class.detail(vuln.attrib['id']) except: try: vulndetails = vuln_class.detail(vuln.attrib['id']) except: try: vulndetails = vuln_class.detail(vuln.attrib['id']) except: raise (vulnfields, references) = vuln_parse(vulndetails.find('Vulnerability'), fromapi=True) if not vulnfields: continue # add the vulnerability to t_vulndata try: vulnid = db.t_vulndata.insert(**vulnfields) vulns_added += 1 db.commit() except Exception, e: if request.vars.overwrite: try: row = db(db.t_vulndata.f_vulnid == vulnfields['f_vulnid']).select().first() row.update_record(**vulnfields) vulnid = row.id vulns_updated += 1 db.commit() except Exception, e: logger.info("Error inserting %s to vulndata: %s" % (vulnfields['f_vulnid'], e)) vulnid = None vulns_skipped += 1 db.commit() continue else: logger.info("Error inserting %s to vulndata: %s" % (vulnfields['f_vulnid'], e)) vulnid = None vulns_skipped += 1 db.commit() continue # add the references if vulnid is not None and references: for reference in references: # check to see if reference exists first ref_id = db(db.t_vuln_refs.f_text == reference[1]) if ref_id.count() == 0: # add because it doesn't ref_id = db.t_vuln_refs.insert(f_source=reference[0], f_text=reference[1]) else: # pick the first reference as the ID ref_id = ref_id.select().first().id # make many-to-many relationship with t_vuln_data res = db.t_vuln_references.insert(f_vuln_ref_id=ref_id, f_vulndata_id=vulnid) db.commit() logger.info("%d vulns added, %d updated, %d skipped" % (vulns_added, vulns_updated, vulns_skipped)) response.flash = "Completed - (A:%s/U:%s/S:%s)" % (vulns_added, vulns_updated, vulns_skipped)