def cveItemsProcess(type, url, args): if args.v: print("%s... downloading: %s" % (type, url)) file = Configuration.getFile(url) try: (f, r) = file except: sys.exit( "Cannot open url %s. Bad URL or not connected to the internet?" % _url) # get your parser on !! parser = make_parser() ch = CVEHandler() parser.setContentHandler(ch) parser.parse(f) if args.u: i = db.getInfo("cves") if args.u: last_modified = parse_datetime(r.headers['last-modified'], ignoretz=True) if i is not None: if last_modified == i['last-modified']: print("Not modified") sys.exit(0) db.setColUpdate("cves", last_modified) if args.v: u_counter = 0 n_counter = 0 for item in progressbar(ch.cves): if 'cvss' not in item: item['cvss'] = None else: item['cvss'] = float(item['cvss']) if 'cwe' not in item: item['cwe'] = defaultvalue['cwe'] # check if already exists x = db.getCVE(item['id']) # if so, update the entry. if x: if args.v: u_counter += 1 db.updateCVE(item) else: if args.v: n_counter += 1 db.insertCVE(item) if args.v: print("New: %s Updated: %s" % (n_counter, u_counter)) print("")
def endElement(self, name): if name == 'nvd:statement': self.statementtag = False self.statement = self.statement + self.statement.rstrip() self.vendor[-1]['statement'] = self.statement # dictionary vendordict = Configuration.getVendorDict() # make parser parser = make_parser() ch = VendorHandler() parser.setContentHandler(ch) # check modification date try: (f, r) = Configuration.getFile(vendordict, compressed = True) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(vendordict)) i = db.getLastModified('vendor') if i is not None: if r.headers['last-modified'] == i: print("Not modified") sys.exit(0) # parse xml and store in database parser.parse(f) statements=[] for statement in progressbar(ch.vendor): if args.v: print (statement) statements.append(statement) db.bulkUpdate('vendor', statements)
# dictionary vendordict = Configuration.getVendorDict() # connect to db db = Configuration.getMongoConnection() vendor = db.vendor info = db.info # make parser parser = make_parser() ch = VendorHandler() parser.setContentHandler(ch) # check modification date try: f = Configuration.getFile(vendordict) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(vendordict)) i = info.find_one({'db': 'vendor'}) if i is not None: if f.headers['last-modified'] == i['last-modified']: print("Not modified") sys.exit(0) # parse xml and store in database parser.parse(f) bulk = vendor.initialize_ordered_bulk_op() for statement in progressbar(ch.vendor): if args.v: print (statement) bulk.find({'id': statement['id']}).upsert().update({"$set": {'statement': statement['statement'], 'id': statement['id'], 'organization': statement['organization'], 'contributor': statement['contributor'], 'lastmodified': statement['lastmodified']}}) bulk.execute()
version_info += cpe["versionEndExcluding"] if "versionEndIncluding" in item: cpe["versionEndIncluding"] = item["versionEndIncluding"] version_info += cpe["versionEndIncluding"] sha1_hash = hashlib.sha1(cpe["cpe_2_2"].encode("utf-8") + version_info.encode("utf-8")).hexdigest() cpe["id"] = sha1_hash return cpe if __name__ == '__main__': if args.u: try: (f, r) = Configuration.getFile(Configuration.getFeedURL('cpe')) except: sys.exit( "Cannot open url %s. Bad URL or not connected to the internet?" % (Configuration.getFeedURL("cpe"))) # check modification date i = db.getLastModified('cpe') last_modified = parse_datetime(r.headers['last-modified'], ignoretz=True) if i is not None: if last_modified == i and not args.f: print("Not modified") sys.exit(0) cpej = json.loads(f.read())
if name == 'vuln:last-modified-datetime': self.inPUBElem = 0 self.cves[-1]['Modified'] = self.PUB if __name__ == '__main__': parser = make_parser() ch = CVEHandler() parser.setContentHandler(ch) # start here if it's an update. if args.u: # get the 'modified' file getfile = file_prefix + file_mod + file_suffix try: (f, r) = Configuration.getFile(Configuration.getCVEDict() + getfile, compressed=True) except: sys.exit( "Cannot open url %s. Bad URL or not connected to the internet?" % (Configuration.getCVEDict() + getfile)) i = db.getInfo("cve") if i is not None: if r.headers['last-modified'] == i['last-modified']: print("Not modified") sys.exit(0) db.setColUpdate("cve", r.headers['last-modified']) # get your parser on !! parser = make_parser() ch = CVEHandler() parser.setContentHandler(ch)
sys.exit("Redis server not running on %s:%s"%(Configuration.getRedisHost(),Configuration.getRedisPort())) RefUrl = Configuration.getRefURL() tmppath = Configuration.getTmpdir() argparser = argparse.ArgumentParser(description='Populate/update the NIST ref database') argparser.add_argument('-v', action='store_true', help='verbose output', default=False) args = argparser.parse_args() if args.v: verbose = True # check modification date try: u = Configuration.getFile(RefUrl) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(RefUrl)) i = db.getLastModified('ref') if i is not None: if u.headers['last-modified'] == i: print("Not modified") sys.exit(0) # Create temp file and download and unpack database if not os.path.exists(tmppath): os.mkdir(tmppath) with open(tmppath+'/allrefmaps.zip', 'wb') as fp: shutil.copyfileobj(u, fp)
# dictionary d2securl = Configuration.getd2secDict() # connect to db db = Configuration.getMongoConnection() d2sec = db.d2sec info = db.info # make parser parser = make_parser() ch = ExploitHandler() parser.setContentHandler(ch) # check modification date try: f = Configuration.getFile(d2securl) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(d2securl)) i = dbLayer.getLastModified("d2sec") if i is not None: if f.headers['last-modified'] == i: print("Not modified") sys.exit(0) # parse xml and store in database parser.parse(f) bulk = d2sec.initialize_ordered_bulk_op() for exploit in progressbar(ch.d2sec): print (exploit) if args.v: print (exploit) bulk.find({'id': exploit['id']}).upsert().update({"$set": {'id': exploit['id'], 'url': exploit['url'], 'name': exploit['name']}})
self.Attack_Pattern_Catalog_tag = False # dictionary capecurl = Configuration.getCAPECDict() # connect to db db = Configuration.getMongoConnection() capec = db.capec info = db.info # make parser parser = make_parser() ch = CapecHandler() parser.setContentHandler(ch) # check modification date try: f = Configuration.getFile(capecurl) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(capecurl)) i = info.find_one({'db': 'capec'}) if i is not None: if f.headers['last-modified'] == i['last-modified']: print("Not modified") sys.exit(0) # parse xml and store in database parser.parse(f) bulk = capec.initialize_ordered_bulk_op() for attack in progressbar(ch.capec): bulk.find({'id': attack['id']}).upsert().update({"$set": {'name': attack['name'], 'summary': attack['summary'], 'prerequisites': attack['prerequisites'], 'solutions': attack['solutions'], 'related_weakness': attack['related_weakness']}}) bulk.execute() #update database info after successful program-run
# dict cpedict = Configuration.getCPEDict() # connect to db db = Configuration.getMongoConnection() cpe = db.cpe info = db.info # make parser parser = make_parser() ch = CPEHandler() parser.setContentHandler(ch) # check modification date try: f = Configuration.getFile(cpedict) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" % (cpedict)) i = info.find_one({'db': 'cpe'}) if i is not None: if f.headers['last-modified'] == i['last-modified']: print("Not modified") sys.exit(0) # parse xml and store in database parser.parse(f) bulk = cpe.initialize_ordered_bulk_op() for x in progressbar(ch.cpe): name = toStringFormattedCPE(x['name']) oldCPE = x['name'] title = x['title'][0]
if name == 'nvd:statement': self.statementtag = False self.statement = self.statement + self.statement.rstrip() self.vendor[-1]['statement'] = self.statement # dictionary vendordict = Configuration.getVendorDict() # make parser parser = make_parser() ch = VendorHandler() parser.setContentHandler(ch) # check modification date try: (f, r) = Configuration.getFile(vendordict, compressed=True) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" % (vendordict)) i = db.getLastModified('vendor') if i is not None: if r.headers['last-modified'] == i: print("Not modified") sys.exit(0) # parse xml and store in database parser.parse(f) statements = [] for statement in progressbar(ch.vendor): if args.v: print(statement) statements.append(statement)
import lib.DatabaseLayer as db import csv import argparse # dictionary exploitdburl = Configuration.getexploitdbDict() tmppath = Configuration.getTmpdir() argparser = argparse.ArgumentParser(description='Populate/update the exploitdb ref database') argparser.add_argument('-v', action='store_true', help='verbose output', default=False) args = argparser.parse_args() try: f = Configuration.getFile(exploitdburl) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(exploitdburl)) i = db.getLastModified('exploitdb') if i is not None: if f.headers['last-modified'] == i: print("Not modified") sys.exit(0) if not os.path.exists(tmppath): os.mkdir(tmppath) csvfile = tmppath+'/exploitdb.csv' with open(csvfile, 'wb') as fp: shutil.copyfileobj(f, fp)
import argparse # dictionary exploitdburl = Configuration.getexploitdbDict() tmppath = Configuration.getTmpdir() argparser = argparse.ArgumentParser( description='Populate/update the exploitdb ref database') argparser.add_argument('-v', action='store_true', help='verbose output', default=False) args = argparser.parse_args() try: f = Configuration.getFile(exploitdburl) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" % (exploitdburl)) i = db.getLastModified('exploitdb') if i is not None: if f.headers['last-modified'] == i: print("Not modified") sys.exit(0) if not os.path.exists(tmppath): os.mkdir(tmppath) csvfile = tmppath + '/exploitdb.csv' with open(csvfile, 'wb') as fp:
self.cves[-1]['Modified'] = self.PUB if __name__ == '__main__': # connect to the DB. db = Configuration.getMongoConnection() collection = db.cves info = db.info parser = make_parser() ch = CVEHandler() parser.setContentHandler(ch) # start here if it's an update. if args.u: # get the 'modified' file getfile = file_prefix + file_mod + file_suffix try: f = Configuration.getFile(Configuration.getCVEDict() + getfile) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getCVEDict() + getfile)) i = dbLayer.getInfo("cve") if i is not None: if f.headers['last-modified'] == i['last-modified']: print("Not modified") sys.exit(0) dbLayer.setColUpdate("cve", f.headers['last-modified']) # get your parser on !! parser = make_parser() ch = CVEHandler() parser.setContentHandler(ch) parser.parse(f) for item in ch.cves:
self.cves[-1]['Published'] = parse_datetime(self.DT, ignoretz=True) if name == 'vuln:last-modified-datetime': self.inPUBElem = 0 self.cves[-1]['Modified'] = parse_datetime(self.PUB, ignoretz=True) if __name__ == '__main__': parser = make_parser() ch = CVEHandler() parser.setContentHandler(ch) # start here if it's an update. if args.u: # get the 'modified' file getfile = file_prefix + file_mod + file_suffix try: (f, r) = Configuration.getFile( Configuration.getFeedURL('cve') + getfile) except: sys.exit( "Cannot open url %s. Bad URL or not connected to the internet?" % (Configuration.getFeedURL("cve") + getfile)) i = db.getInfo("cve") last_modified = parse_datetime(r.headers['last-modified'], ignoretz=True) if i is not None: if last_modified == i['last-modified']: print("Not modified") sys.exit(0) db.setColUpdate("cve", last_modified) # get your parser on !! parser = make_parser()
# datemode: 0 for 1900-based, 1 for 1904-based return ( datetime.datetime(1899, 12, 30) + datetime.timedelta(days=xldate + 1462 * datemode) ) # dictionary msbulletinurl = Configuration.getMSBULLETINDict() # connect to db db = Configuration.getMongoConnection() msbulletin = db.ms info = db.info try: f = Configuration.getFile(msbulletinurl) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(msbulletinurl)) # check modification date i = info.find_one({'db': 'ms'}) if i is not None: if f.headers['last-modified'] == i['last-modified']: print("Not modified") sys.exit(0) if not os.path.exists('./tmp'): os.mkdir('./tmp') with open('./tmp/BulletinSearch.xls', 'wb') as fp: shutil.copyfileobj(f, fp)
tmppath = Configuration.getTmpdir() argparser = argparse.ArgumentParser( description='Populate/update the NIST ref database') argparser.add_argument('-v', action='store_true', help='verbose output', default=False) args = argparser.parse_args() if args.v: verbose = True # check modification date try: u = Configuration.getFile(RefUrl) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" % (RefUrl)) i = db.getLastModified('ref') if i is not None: if u.headers['last-modified'] == i: print("Not modified") sys.exit(0) # Create temp file and download and unpack database if not os.path.exists(tmppath): os.mkdir(tmppath) with open(tmppath + '/allrefmaps.zip', 'wb') as fp: shutil.copyfileobj(u, fp)
self.inDTElem = 0 self.cves[-1]['Published'] = parse_datetime(self.DT, ignoretz=True) if name == 'vuln:last-modified-datetime': self.inPUBElem = 0 self.cves[-1]['Modified'] = parse_datetime(self.PUB, ignoretz=True) if __name__ == '__main__': parser = make_parser() ch = CVEHandler() parser.setContentHandler(ch) # start here if it's an update. if args.u: # get the 'modified' file getfile = file_prefix + file_mod + file_suffix try: (f, r) = Configuration.getFile(Configuration.getFeedURL('cve') + getfile) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getFeedURL("cve") + getfile)) i = db.getInfo("cves") last_modified = parse_datetime(r.headers['last-modified'], ignoretz=True) if i is not None: if last_modified == i['last-modified']: print("Not modified") sys.exit(0) db.setColUpdate("cves", last_modified) # get your parser on !! parser = make_parser() ch = CVEHandler() parser.setContentHandler(ch) parser.parse(f)
self.inDTElem = 0 self.cves[-1]['Published'] = self.DT if name == 'vuln:last-modified-datetime': self.inPUBElem = 0 self.cves[-1]['Modified'] = self.PUB if __name__ == '__main__': parser = make_parser() ch = CVEHandler() parser.setContentHandler(ch) # start here if it's an update. if args.u: # get the 'modified' file getfile = file_prefix + file_mod + file_suffix try: (f, r) = Configuration.getFile(Configuration.getCVEDict() + getfile, compressed = True) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(Configuration.getCVEDict() + getfile)) i = db.getInfo("cve") if i is not None: if r.headers['last-modified'] == i['last-modified']: print("Not modified") sys.exit(0) db.setColUpdate("cve", r.headers['last-modified']) # get your parser on !! parser = make_parser() ch = CVEHandler() parser.setContentHandler(ch) parser.parse(f) for item in ch.cves:
self.exploittag = False self.refl = [] if name == 'elliot': self.elliottag = False # dictionary d2securl = Configuration.getd2secDict() # make parser parser = make_parser() ch = ExploitHandler() parser.setContentHandler(ch) # check modification date try: f = Configuration.getFile(d2securl) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" % (d2securl)) last_modified = parse_datetime(f.headers['last-modified'], ignoretz=True) i = db.getLastModified("d2sec") if i is not None: if last_modified == i: print("Not modified") sys.exit(0) # parse xml and store in database parser.parse(f) exploitList = [] for exploit in progressbar(ch.d2sec): print(exploit) if args.v:
elif name == 'references': self.referencestag = False elif name == 'reference': self.referencetag = False self.href = None # dict cpedict = Configuration.getCPEDict() # make parser parser = make_parser() ch = CPEHandler() parser.setContentHandler(ch) # check modification date try: f = Configuration.getFile(cpedict) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(cpedict)) i = db.getLastModified('cpe') if i is not None: if f.headers['last-modified'] == i: print("Not modified") sys.exit(0) # parse xml and store in database parser.parse(f) cpeList=[] for x in progressbar(ch.cpe): x['id']= toStringFormattedCPE(x['name']) x['title']=x['title'][0] x['cpe_2_2'] = x.pop('name') if not x['references']: x.pop('references')
# datemode: 0 for 1900-based, 1 for 1904-based return (datetime.datetime(1899, 12, 30) + datetime.timedelta(days=xldate + 1462 * datemode)) # dictionary msbulletinurl = Configuration.getMSBULLETINDict() tmppath = Configuration.getTmpdir() # connect to db db = Configuration.getMongoConnection() msbulletin = db.ms info = db.info try: f = Configuration.getFile(msbulletinurl) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" % (msbulletinurl)) # check modification date i = info.find_one({'db': 'ms'}) if i is not None: if f.headers['last-modified'] == i['last-modified']: print("Not modified") sys.exit(0) if not os.path.exists(tmppath): os.mkdir(tmppath) with open(tmppath + '/BulletinSearch.xlsx', 'wb') as fp: shutil.copyfileobj(f, fp)
import tarfile import shutil import sqlite3 from lib.ProgressBar import progressbar from lib.Config import Configuration import lib.DatabaseLayer as db vFeedurl = Configuration.getvFeedURL() vFeedstatus = Configuration.getvFeedStatus() tmppath = os.path.join(runPath, "..", Configuration.getTmpdir()) # check modification date try: u = Configuration.getFile(vFeedurl) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(vFeedurl)) i = db.getLastModified('vfeed') if i is not None: if u.headers['last-modified'] == i: print("Not modified") sys.exit(0) # create temp file and download and unpack database if not os.path.exists(tmppath): os.mkdir(tmppath) with open(tmppath+'/vfeed.db.tgz', 'wb') as fp: shutil.copyfileobj(u, fp) t = tarfile.open(name=tmppath+'/vfeed.db.tgz', mode='r') t.extract('vfeed.db', path=tmppath) t.close
if cwe['lang'] == 'en': cve['cwe'] = cwe['value'] if not ('cwe' in cve): cve['cwe'] = defaultvalue['cwe'] else: cve['cwe'] = defaultvalue['cwe'] cve['vulnerable_configuration_cpe_2_2'] = [] return cve if __name__ == '__main__': if args.u: # get the 'modified' file getfile = file_prefix + file_mod + file_suffix try: (f, r) = Configuration.getFile( Configuration.getFeedURL('cve') + getfile) except: sys.exit( "Cannot open url %s. Bad URL or not connected to the internet?" % (Configuration.getFeedURL("cve") + getfile)) i = db.getInfo("cves") last_modified = parse_datetime(r.headers['last-modified'], ignoretz=True) if i is not None: if last_modified == i['last-modified'] and not args.f: print("Not modified") sys.exit(0) db.setColUpdate("cves", last_modified) cvej = json.loads(f.read()) for cveitem in cvej['CVE_Items']: item = process_cve_item(item=cveitem)
from dateutil.parser import parse as parse_datetime import tarfile import shutil from lib.Config import Configuration import lib.DatabaseLayer as db vFeedurl = Configuration.getvFeedURL() vFeedstatus = Configuration.getvFeedStatus() tmppath = os.path.join(runPath, "..", Configuration.getTmpdir()) print(tmppath) # check modification date try: u = Configuration.getFile(vFeedurl) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?"%(vFeedurl)) last_modified = parse_datetime(u.headers['last-modified'], ignoretz=True) i = db.getLastModified('vfeed') if i is not None: if last_modified == i: print("Not modified") sys.exit(0) # create temp file and download and unpack database if not os.path.exists(tmppath): os.mkdir(tmppath) with open(tmppath+'/vfeed.db.tgz', 'wb') as fp: shutil.copyfileobj(u, fp) t = tarfile.open(name=tmppath+'/vfeed.db.tgz', mode='r:*') t.extract('vfeed.db', path=tmppath)
if name == 'capec:Attack_Patterns': self.Attack_Patterns_tag = False if name == 'capec:Attack_Pattern_Catalog': self.Attack_Pattern_Catalog_tag = False # dictionary capecurl = Configuration.getCAPECDict() # make parser parser = make_parser() ch = CapecHandler() parser.setContentHandler(ch) # check modification date try: f = Configuration.getFile(capecurl) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" % (capecurl)) i = db.getLastModified('capec') if i is not None: if f.headers['last-modified'] == i: print("Not modified") sys.exit(0) # parse xml and store in database parser.parse(f) attacks = [] for attack in progressbar(ch.capec): attacks.append(attack) db.bulkUpdate("capec", attacks)
# dictionary vendordict = Configuration.getVendorDict() # connect to db db = Configuration.getMongoConnection() vendor = db.vendor info = db.info # make parser parser = make_parser() ch = VendorHandler() parser.setContentHandler(ch) # check modification date try: f = Configuration.getFile(vendordict) except: sys.exit("Cannot open url %s. Bad URL or not connected to the internet?" % (vendordict)) i = info.find_one({'db': 'vendor'}) if i is not None: if f.headers['last-modified'] == i['last-modified']: print("Not modified") sys.exit(0) # parse xml and store in database parser.parse(f) bulk = vendor.initialize_ordered_bulk_op() for statement in progressbar(ch.vendor): if args.v: print(statement) bulk.find({