def __init__(self, config): self.config = config self.logger = logging.getLogger("NVD") self.database = MongoDB(self) self.xmpp = NVDXMPP(self) if self.config.importxml: self.import_xml() else: if not self.config.disablexmpp: self.xmpp.run() self.update()
def __init__(self, config): self.config = config self.logger = logging.getLogger("NVD") self.database = MongoDB(self) self.xmpp = NVDXMPP(self) if self.config.importxml: self.import_xml() else: self.update() self.xmpp.run()
class NVD(object): CHECK_INTERVAL = 1800 def __init__(self, config): self.config = config self.logger = logging.getLogger("NVD") self.database = MongoDB(self) self.xmpp = NVDXMPP(self) if self.config.importxml: self.import_xml() else: if not self.config.disablexmpp: self.xmpp.run() self.update() def import_xml(self): """ Import XML from file to Database. Will import new entries, or update old entries. """ self.parse_nvd(self.config.importxml) def parse_nvd(self, xmlfile=None): """ Parse the XML and insert the CVE IDs into the database """ if xmlfile and os.path.isfile(xmlfile): self.logger.info("Importing {} into database".format(xmlfile)) with open(xmlfile, 'r') as f: xml = f.read() else: self.logger.debug("Fetching XML from mongodb") mongo_xml = self.database.collection.find_one( {"xml": { '$exists': True }}) if not mongo_xml: self.logger.error("Could not find XML in MongoDB") return 0 xml = mongo_xml['xml']['data'] self.logger.info("Parsing XML data") nvd_data = _nvd.CreateFromDocument(xml) for entry in nvd_data.entry: vulnerability = Vulnerability() if "REJECT" in entry.summary: continue if entry.vulnerable_software_list: vulnerability.cpe = list( entry.vulnerable_software_list.product) for cpe in entry.vulnerable_software_list.product: product = Product(cpe) vulnerability.product.append(product) vulnerability.cve_id = entry.cve_id vulnerability.publish_date = str(entry.published_datetime) vulnerability.update_date = str(entry.last_modified_datetime) vulnerability.summary = entry.summary vulnerability.cvss = None vulnerability.vector = None if entry.cvss: vulnerability.cvss = str(entry.cvss.base_metrics[0].score) vulnerability.vector = entry.cvss.base_metrics[ 0].access_vector.value() try: vulnerability_raw = copy.deepcopy( vulnerability ) # The Transform in Product chagnes the vulnerability object except copy.error as e: self.logger.error(e) existing = Vulnerability(self.find_cve(entry.cve_id)) if existing: self.logger.info( "We have {} from before with CVSS={}, new is={}".format( existing.cve_id, existing.cvss, vulnerability.cvss)) write_result = self.database.collection.update( {"cve_id": entry.cve_id}, {"$set": { "vulnerability": vulnerability.__dict__ }}, upsert=True, manipulate=True) self.logger.debug(write_result) if write_result['nModified'] > 0: if not self.config.importxml: if existing.cvss == vulnerability.cvss: self.logger.debug( "Existing CVSS of {} equals this CVSS of {}, skipping print" .format(existing.cvss, vulnerability.cvss)) else: self.xmpp.updated(vulnerability_raw, vulnerability.publish_date) self.logger.info("{} has been updated".format(entry.cve_id)) elif existing is False or write_result['updatedExisting'] is False: if not self.config.importxml: self.xmpp.new(vulnerability_raw) self.logger.debug("Inserting {} into DB".format(entry.cve_id)) def download_if_needed(self, force=False): """ Downloads a new and updated NVD recent XML if needed """ url = 'http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-Recent.xml.gz' request = urllib.request.Request(url) request.get_method = lambda: 'HEAD' try: self.logger.debug("HEADing {}".format(url)) response = urllib.request.urlopen(request) except urllib.error.HTTPError as e: self.logger.error('HTTPError = ' + str(e.code)) sys.exit(1) except urllib.error.URLError as e: self.logger.error('URLError = ' + str(e.reason)) sys.exit(1) except http.client.HTTPException as e: self.logger.error('HTTPException') sys.exit(1) except Exception: import traceback self.logger.error('generic exception: ' + traceback.format_exc()) sys.exit(1) modified = dict(response.info())['Last-Modified'] modified_time = time.mktime( time.strptime(modified, '%a, %d %b %Y %H:%M:%S %Z')) self.logger.info( "Last-Modified of the remote XML: {}".format(modified)) self.logger.debug("Fetching XML from mongodb") mongo_xml = self.database.collection.find_one( {"xml": { '$exists': True }}) if mongo_xml: self.logger.info("Local XML age: {}".format( time.ctime(mongo_xml['xml']['mtime']))) if mongo_xml['xml']['mtime'] >= modified_time: if self.config.force or force: self.logger.info( "Local XML is unchanged from remote. But forcing download" ) else: self.logger.info( "Local XML is unchanged from remote. Skipping download" ) return 0 request.get_method = lambda: 'GET' try: self.logger.debug("GETing {}".format(url)) response = urllib.request.urlopen(request) except urllib2.error.HTTPError as e: self.logger.error('HTTPError = ' + str(e.code)) sys.exit(1) except urllib2.error.URLError as e: self.logger.error('URLError = ' + str(e.reason)) sys.exit(1) except http.client.HTTPException as e: self.logger.error('HTTPException') sys.exit(1) except Exception: import traceback self.logger.error('generic exception: ' + traceback.format_exc()) sys.exit(1) gzip = zlib.decompressobj(16 + zlib.MAX_WBITS) xml = { "xml": { "mtime": modified_time, "data": gzip.decompress(response.read()) } } self.database.collection.update({"xml": { '$exists': True }}, xml, upsert=True) return 1 def update(self, force=False): """ Update timer. Downloads the NVD XML every CHECK_INTERVAL """ self.logger.info( "Hitting update interval ({}), downloading new XML".format( self.CHECK_INTERVAL)) self.logger.debug("Active threads: {}".format(active_count())) Timer(self.CHECK_INTERVAL, self.update).start() with Lock(): if self.download_if_needed(force=force): self.parse_nvd() return 1 return 0 def find_cve(self, cve): """ Search for and return CVE entry """ return self.database.collection.find_one({"cve_id": cve}, manipulate=True)
class NVD(object): CHECK_INTERVAL = 1800 def __init__(self, config): self.config = config self.logger = logging.getLogger("NVD") self.database = MongoDB(self) self.xmpp = NVDXMPP(self) if self.config.importxml: self.import_xml() else: if not self.config.disablexmpp: self.xmpp.run() self.update() def import_xml(self): """ Import XML from file to Database. Will import new entries, or update old entries. """ self.parse_nvd(self.config.importxml) def parse_nvd(self, xmlfile=None): """ Parse the XML and insert the CVE IDs into the database """ if xmlfile and os.path.isfile(xmlfile): self.logger.info("Importing {} into database".format(xmlfile)) with open(xmlfile, 'r') as f: xml = f.read() else: self.logger.debug("Fetching XML from mongodb") mongo_xml = self.database.collection.find_one({"xml": {'$exists': True}}) if not mongo_xml: self.logger.error("Could not find XML in MongoDB") return 0 xml = mongo_xml['xml']['data'] self.logger.info("Parsing XML data") nvd_data = _nvd.CreateFromDocument(xml) for entry in nvd_data.entry: vulnerability = Vulnerability() if "REJECT" in entry.summary: continue if entry.vulnerable_software_list: vulnerability.cpe = list(entry.vulnerable_software_list.product) for cpe in entry.vulnerable_software_list.product: product = Product(cpe) vulnerability.product.append(product) vulnerability.cve_id = entry.cve_id vulnerability.publish_date = str(entry.published_datetime) vulnerability.update_date = str(entry.last_modified_datetime) vulnerability.summary = entry.summary vulnerability.cvss = None vulnerability.vector = None if entry.cvss: vulnerability.cvss = str(entry.cvss.base_metrics[0].score) vulnerability.vector = entry.cvss.base_metrics[0].access_vector.value() try: vulnerability_raw = copy.deepcopy(vulnerability) # The Transform in Product chagnes the vulnerability object except copy.error as e: self.logger.error(e) existing = Vulnerability(self.find_cve(entry.cve_id)) if existing: self.logger.info("We have {} from before with CVSS={}, new is={}".format(existing.cve_id, existing.cvss, vulnerability.cvss)) write_result = self.database.collection.update( {"cve_id": entry.cve_id}, {"$set": {"vulnerability": vulnerability.__dict__}}, upsert=True, manipulate=True ) self.logger.debug(write_result) if write_result['nModified'] > 0: if not self.config.importxml: if existing.cvss == vulnerability.cvss: self.logger.debug("Existing CVSS of {} equals this CVSS of {}, skipping print".format(existing.cvss, vulnerability.cvss)) else: self.xmpp.updated(vulnerability_raw, vulnerability.publish_date) self.logger.info("{} has been updated".format(entry.cve_id)) elif existing is False or write_result['updatedExisting'] is False: if not self.config.importxml: self.xmpp.new(vulnerability_raw) self.logger.debug("Inserting {} into DB".format(entry.cve_id)) def download_if_needed(self, force=False): """ Downloads a new and updated NVD recent XML if needed """ url = 'http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-Recent.xml.gz' request = urllib.request.Request(url) request.get_method = lambda: 'HEAD' try: self.logger.debug("HEADing {}".format(url)) response = urllib.request.urlopen(request) except urllib.error.HTTPError as e: self.logger.error('HTTPError = ' + str(e.code)) sys.exit(1) except urllib.error.URLError as e: self.logger.error('URLError = ' + str(e.reason)) sys.exit(1) except http.client.HTTPException as e: self.logger.error('HTTPException') sys.exit(1) except Exception: import traceback self.logger.error('generic exception: ' + traceback.format_exc()) sys.exit(1) modified = dict(response.info())['Last-Modified'] modified_time = time.mktime(time.strptime(modified, '%a, %d %b %Y %H:%M:%S %Z')) self.logger.info("Last-Modified of the remote XML: {}".format(modified)) self.logger.debug("Fetching XML from mongodb") mongo_xml = self.database.collection.find_one({"xml": {'$exists': True}}) if mongo_xml: self.logger.info("Local XML age: {}".format(time.ctime(mongo_xml['xml']['mtime']))) if mongo_xml['xml']['mtime'] >= modified_time: if self.config.force or force: self.logger.info("Local XML is unchanged from remote. But forcing download") else: self.logger.info("Local XML is unchanged from remote. Skipping download") return 0 request.get_method = lambda: 'GET' try: self.logger.debug("GETing {}".format(url)) response = urllib.request.urlopen(request) except urllib2.error.HTTPError as e: self.logger.error('HTTPError = ' + str(e.code)) sys.exit(1) except urllib2.error.URLError as e: self.logger.error('URLError = ' + str(e.reason)) sys.exit(1) except http.client.HTTPException as e: self.logger.error('HTTPException') sys.exit(1) except Exception: import traceback self.logger.error('generic exception: ' + traceback.format_exc()) sys.exit(1) gzip = zlib.decompressobj(16 + zlib.MAX_WBITS) xml = { "xml": { "mtime": modified_time, "data": gzip.decompress(response.read()) } } self.database.collection.update({"xml": {'$exists': True}}, xml, upsert=True) return 1 def update(self, force=False): """ Update timer. Downloads the NVD XML every CHECK_INTERVAL """ self.logger.info("Hitting update interval ({}), downloading new XML".format(self.CHECK_INTERVAL)) self.logger.debug("Active threads: {}".format(active_count())) Timer(self.CHECK_INTERVAL, self.update).start() with Lock(): if self.download_if_needed(force=force): self.parse_nvd() return 1 return 0 def find_cve(self, cve): """ Search for and return CVE entry """ return self.database.collection.find_one({"cve_id": cve}, manipulate=True)
class NVD(object): CHECK_INTERVAL = 1800 def __init__(self, config): self.config = config self.logger = logging.getLogger("NVD") self.database = MongoDB(self) self.xmpp = NVDXMPP(self) if self.config.importxml: self.import_xml() else: self.update() self.xmpp.run() def import_xml(self): """ Import XML from file to Database. Will import new entries, or update old entries. """ self.parse_nvd(self.config.importxml) def parse_nvd(self, xmlfile=None): """ Parse the XML and insert the CVE IDs into the database """ if xmlfile and os.path.isfile(xmlfile): self.logger.info("Importing {} into database".format(xmlfile)) with open(xmlfile, 'r') as f: xml = f.read() else: self.logger.debug("Fetching XML from mongodb") mongo_xml = self.database.collection.find_one({"xml": {'$exists': True}}) if not mongo_xml: self.logger.error("Could not find XML in MongoDB") return 0 xml = mongo_xml['xml']['data'] self.logger.info("Parsing XML data") nvd_data = _nvd.CreateFromDocument(xml) for entry in nvd_data.entry: vulnerability = {} if "REJECT" in entry.summary: continue vulnerability['product'] = [] if entry.vulnerable_software_list: product_list = entry.vulnerable_software_list.product for product in product_list: try: vulnerability['product'].append({ "vendor": product.split(':')[2], "product": product.split(':')[3], "version": product.split(':')[4] }) except IndexError as e: self.logger.error(e) vulnerability['cve_id'] = entry.cve_id vulnerability['publish_date'] = str(entry.published_datetime) vulnerability['update_date'] = str(entry.last_modified_datetime) vulnerability['summary'] = entry.summary vulnerability['cvss'] = None vulnerability['vector'] = None if entry.cvss: vulnerability['cvss'] = str(entry.cvss.base_metrics[0].score) vulnerability['vector'] = entry.cvss.base_metrics[0].access_vector.value() write_result = self.database.collection.update( {"cve_id": entry.cve_id}, {"$set": {"vulnerability": vulnerability}}, upsert=True ) self.logger.debug( write_result ) if write_result['nModified'] > 0: if not self.config.importxml: self.xmpp.updated(vulnerability) self.logger.info("{} has been updated".format(entry.cve_id)) elif write_result['updatedExisting'] == False: if not self.config.importxml: self.xmpp.new(vulnerability) self.logger.debug("Inserting {} into DB".format(entry.cve_id)) def download_if_needed(self): """ Downloads a new and updated NVD recent XML if needed """ url = 'http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-Recent.xml' # url = "https://blog.mrfjo.org/nvdcve-2.0-Recent.xml" request = urllib.request.Request(url) request.get_method = lambda: 'HEAD' try: self.logger.debug("HEADing {}".format(url)) response = urllib.request.urlopen(request) except urllib.error.HTTPError as e: self.logger.error('HTTPError = ' + str(e.code)) sys.exit(1) except urllib.error.URLError as e: self.logger.error('URLError = ' + str(e.reason)) sys.exit(1) except http.client.HTTPException as e: self.logger.error('HTTPException') sys.exit(1) except Exception: import traceback self.logger.error('generic exception: ' + traceback.format_exc()) sys.exit(1) modified = dict(response.info())['Last-Modified'] modified_time = time.mktime(time.strptime(modified, '%a, %d %b %Y %H:%M:%S %Z')) self.logger.info("Last-Modified of the remote XML: {}".format(modified)) self.logger.debug("Fetching XML from mongodb") mongo_xml = self.database.collection.find_one({"xml": {'$exists': True}}) if mongo_xml: self.logger.info("Local XML age: {}".format(time.ctime(mongo_xml['xml']['mtime']))) if mongo_xml['xml']['mtime'] >= modified_time: if self.config.force: self.logger.info("Local XML is unchanged from remote. But forcing download") else: self.logger.info("Local XML is unchanged from remote. Skipping download") return 0 request.get_method = lambda: 'GET' try: self.logger.debug("GETing {}".format(url)) response = urllib.request.urlopen(request) except urllib2.error.HTTPError as e: self.logger.error('HTTPError = ' + str(e.code)) sys.exit(1) except urllib2.error.URLError as e: self.logger.error('URLError = ' + str(e.reason)) sys.exit(1) except http.client.HTTPException as e: self.logger.error('HTTPException') sys.exit(1) except Exception: import traceback self.logger.error('generic exception: ' + traceback.format_exc()) sys.exit(1) xml = { "xml": { "mtime": modified_time, "data": response.read() } } self.database.collection.update({"xml": {'$exists': True}}, xml, upsert=True) return 1 def update(self): """ Update timer. Downloads the NVD XML every CHECK_INTERVAL """ self.logger.info("Hitting update interval ({}), downloading new XML".format(self.CHECK_INTERVAL)) self.logger.debug("Active threads: {}".format(active_count())) Timer(self.CHECK_INTERVAL, self.update).start() with Lock(): if self.download_if_needed(): self.parse_nvd() return 1 return 0 def find_cve(self, cve): """ Search for and return CVE entry """ return self.database.collection.find_one({"cve_id": cve})