def manage_vulner_script(self, test, dupes, script_element, endpoint, report_date=None): for component_element in script_element.findall('table'): component_cpe = CPE(component_element.attrib['key']) for vuln in component_element.findall('table'): # convert elements in dict vuln_attributes = dict() for elem in vuln.findall('elem'): vuln_attributes[elem.attrib['key'].lower()] = elem.text vuln_id = vuln_attributes['id'] description = "### Vulnerability\n\n" description += "**ID**: `" + str(vuln_id) + "`\n" description += "**CPE**: " + str(component_cpe) + "\n" for attribute in vuln_attributes: description += "**" + attribute + "**: `" + vuln_attributes[ attribute] + "`\n" severity = self.convert_cvss_score(vuln_attributes['cvss']) finding = Finding( title=vuln_id, test=test, description=description, severity=severity, component_name=component_cpe.get_product()[0] if len(component_cpe.get_product()) > 0 else '', component_version=component_cpe.get_version()[0] if len(component_cpe.get_version()) > 0 else '', vuln_id_from_tool=vuln_id, nb_occurences=1, ) finding.unsaved_endpoints = [endpoint] # manage if CVE is in metadata if "type" in vuln_attributes and "cve" == vuln_attributes[ "type"]: finding.cve = vuln_attributes["id"] if report_date: finding.date = report_date dupe_key = finding.vuln_id_from_tool if dupe_key in dupes: find = dupes[dupe_key] if description is not None: find.description += "\n-----\n\n" + finding.description # fives '-' produces an horizontal line find.unsaved_endpoints.extend(finding.unsaved_endpoints) find.nb_occurences += finding.nb_occurences else: dupes[dupe_key] = finding
def manage_vulner_script(self, test, dupes, script_element, endpoint): for component_element in script_element.findall('table'): component_cpe = CPE(component_element.attrib['key']) for vuln in component_element.findall('table'): description = "### Vulnerability\n\n" description += "**CPE**: " + str(component_cpe) + "\n" vuln_attributes = dict() for elem in vuln.findall('elem'): vuln_attributes[elem.attrib['key'].lower()] = elem.text description += "**" + elem.attrib['key'] + "**: " + elem.text + "\n" cve = vuln_attributes['id'] severity = self.convert_cvss_score(vuln_attributes['cvss']) dupe_key = cve if dupe_key in dupes: find = dupes[dupe_key] if description is not None: find.description += description else: find = Finding(title=cve, cve=cve, test=test, description=description, severity=severity, mitigation="N/A", impact="No impact provided", component_name=component_cpe.get_product()[0] if len(component_cpe.get_product()) > 0 else '', component_version=component_cpe.get_version()[0] if len(component_cpe.get_version()) > 0 else '', ) find.unsaved_endpoints = list() dupes[dupe_key] = find find.unsaved_endpoints.append(endpoint)
async def load(self) -> None: """Load HassOS data.""" try: if not self.sys_host.info.cpe: raise NotImplementedError() cpe = CPE(self.sys_host.info.cpe) os_name = cpe.get_product()[0] if os_name not in ("hassos", "haos"): raise NotImplementedError() except NotImplementedError: _LOGGER.info("No Home Assistant Operating System found") return # Store meta data self._available = True self.sys_host.supported_features.cache_clear() self._version = AwesomeVersion(cpe.get_version()[0]) self._board = cpe.get_target_hardware()[0] self._os_name = cpe.get_product()[0] await self.sys_dbus.rauc.update() await self.datadisk.load() _LOGGER.info( "Detect Home Assistant Operating System %s / BootSlot %s", self.version, self.sys_dbus.rauc.boot_slot, )
def config_cpe_match(self, cm): if all("$.vulnerable", cm)[0]: v = PLATFORM.VulnerableConfiguration else: v = PLATFORM.NotVulnerableConfiguration subject = BNode() cveStr = all("$.cpe23Uri", cm)[0] self.triples(subject, v, [(PLATFORM.hasPlatform, cpeURI(cveStr))] + \ self.versionStartExcluding(cm) + self.versionStartIncluding(cm) + self.versionEndExcluding(cm) + self.versionEndIncluding(cm)) #print(cveStr) c = CPE(cveStr) if c.is_hardware(): self.g.add((cpeURI(cveStr), RDF.type, PLATFORM.HardwarePlatform)) elif c.is_application(): self.g.add( (cpeURI(cveStr), RDF.type, PLATFORM.ApplicationPlatform)) elif c.is_operating_system(): self.g.add( (cpeURI(cveStr), RDF.type, PLATFORM.OperatingSystemPlatform)) vendor = "" for i in c.get_vendor(): self.g.add((cpeURI(cveStr), PLATFORM.vendor, self.plEnt(i, "Vendor_", cls=PLATFORM.Vendor))) vendor = i for i in c.get_product(): self.g.add((cpeURI(cveStr), PLATFORM.product, self.plEnt(i, "Product_" + vendor + "_", cls=PLATFORM.Product))) for i in c.get_edition(): self.g.add((cpeURI(cveStr), PLATFORM.edition, self.plEnt(i, "Edition_", cls=PLATFORM.Edition))) for i in c.get_language(): self.g.add((cpeURI(cveStr), PLATFORM.language, self.plEnt(i, "Language_", cls=PLATFORM.Language))) for i in c.get_other(): self.g.add((cpeURI(cveStr), PLATFORM.other, self.plEnt(i, "Other_", cls=PLATFORM.Other))) for i in c.get_software_edition(): self.g.add((cpeURI(cveStr), PLATFORM.softwareEdition, self.plEnt(i, "SoftwareEdition_", cls=PLATFORM.SoftwareEdition))) for i in c.get_target_hardware(): self.g.add((cpeURI(cveStr), PLATFORM.targetHardware, self.plEnt(i, "Hardware_", cls=CORE.Hardware))) for i in c.get_target_software(): self.g.add((cpeURI(cveStr), PLATFORM.targetSoftware, self.plEnt(i, "Software_", cls=CORE.Software))) for i in c.get_update(): if not i == "-": self.g.add((cpeURI(cveStr), PLATFORM.update, Literal(i))) for i in c.get_version(): if not i == "-": self.g.add((cpeURI(cveStr), PLATFORM.version, Literal(i))) return subject
def get_description(self, address, name, cpe): '''Define a description based on hostname and CPE''' if name: return name else: c = CPE(cpe[0], CPE.VERSION_2_3) vendor = c.get_vendor()[0].upper() if vendor in self.devs: try: client = SSHClient() client.set_missing_host_key_policy(AutoAddPolicy()) client.connect(address, username=self.devs[vendor]['USER'], password=self.devs[vendor]['PASSWORD']) stdin, stdout, stderr = client.exec_command( self.devs[vendor]['COMMAND']) return '{}:{}'.format( vendor.lower(), re.search(self.devs[vendor]['REGEX'], str(stdout.read().decode('utf-8'))).group( self.devs[vendor]['REGROUP'])) except (AuthenticationException, SSHException, NoValidConnectionsError, TimeoutError, ConnectionResetError): pass return '{}.{}.{}'.format(c.get_vendor()[0], c.get_product()[0], c.get_version()[0])
async def load(self) -> None: """Load HassOS data.""" try: if not self.sys_host.info.cpe: raise NotImplementedError() cpe = CPE(self.sys_host.info.cpe) if cpe.get_product()[0] != "hassos": raise NotImplementedError() except NotImplementedError: _LOGGER.debug("Found no HassOS") return else: self._available = True # Store meta data self._version = cpe.get_version()[0] self._board = cpe.get_target_hardware()[0] await self.sys_dbus.rauc.update() _LOGGER.info("Detect HassOS %s / BootSlot %s", self.version, self.sys_dbus.rauc.boot_slot) with suppress(DockerAPIError): await self.instance.attach(tag="latest")
def get_description(self, address, name, cpe): '''Define a description based on hostname and CPE''' if name: return name else: c = CPE(cpe[0], CPE.VERSION_2_3) vendor = c.get_vendor()[0] if self.tacacs and vendor == 'cisco': try: client = SSHClient() client.set_missing_host_key_policy(AutoAddPolicy()) client.connect(address, username=self.tacacs['user'], password=self.tacacs['password']) stdin, stdout, stderr = client.exec_command( self.tacacs['command']) return '{}:{}'.format( vendor.lower(), re.search(self.tacacs['regex'], str(stdout.read().decode('utf-8'))).group( self.tacacs['regroup'])) except (AuthenticationException, SSHException, NoValidConnectionsError, TimeoutError, ConnectionResetError): pass return '{}.{}.{}'.format(c.get_vendor()[0], c.get_product()[0], c.get_version()[0])
def getinfo(filename): if os.path.exists(filename + '.bin'): print(filename, "exists -- taking in") with open(filename + '.bin', 'rb') as reader: return eval(reader.read()) print(filename, "does not exist, parsing") f = open(filename) tree = etree.parse(f) f.close() vulns = [] entry_nodes = tree.xpath( '//prefix:entry', namespaces={ 'prefix': 'http://scap.nist.gov/schema/feed/vulnerability/2.0' }) for entry in entry_nodes: thisVuln = {} thisVuln['id'] = entry.find(prefixed('vuln', 'cve-id')).text vulnSoftware = entry.find(prefixed('vuln', 'vulnerable-software-list')) if vulnSoftware is not None: for v in vulnSoftware: try: myCPE = CPE(v.text) except NotImplementedError: print("Could not parse") #logging.warning("Unable to parse CPE '%s'" % v.text) else: thisVuln['part'] = myCPE.get_part()[0] thisVuln['vendor'] = myCPE.get_vendor()[0] if 'linux' in thisVuln['vendor']: thisVuln['vendor'] = 'linux' thisVuln['product'] = myCPE.get_product()[0] if 'linux' in thisVuln['product']: thisVuln['vendor'] = 'linux' thisVuln['version'] = myCPE.get_version()[0] thisVuln['update'] = myCPE.get_update()[0] thisVuln['edition'] = myCPE.get_edition()[0] thisVuln['language'] = myCPE.get_language()[0] cvss = entry.find(prefixed('vuln', 'cvss')) if cvss is not None: thisVuln['score'] = cvss.getchildren()[0].getchildren()[0].text thisVuln['accessVector'] = cvss.getchildren()[0].getchildren( )[1].text thisVuln['accessComplexity'] = cvss.getchildren()[0].getchildren( )[2].text thisVuln['auth'] = cvss.getchildren()[0].getchildren()[3].text thisVuln['impactConf'] = cvss.getchildren()[0].getchildren( )[4].text thisVuln['impactInt'] = cvss.getchildren()[0].getchildren()[5].text thisVuln['impactAvail'] = cvss.getchildren()[0].getchildren( )[6].text vulns.append(thisVuln) with open(filename + '.bin', 'wrb+') as myFile: myFile.write(bytes(vulns))
def owasp_dependency_checker_task(source_code_location: str) -> List[Dict]: """ Run OWASP dependency-check and storage all vulnerabilities in an unified format in Redis """ command = ['dependency-check --project "{loc}" --scan', ' "{loc}" -f "XML" -o {loc}', ' --enableExperimental'] # Install dependencies is VERY important os.system("npm install") # We use os.system instead of subprocess.call because because the OWASP # Dependency check tool can take te CVE wordlist downloaded from the OS # context. Using subprocess.call this is not possible. This implies # that without the context no vulnerabilities was detected os.system("".join(command).format(loc=source_code_location)) tree = ET.parse('{}/dependency-check-report.xml'. \ format(source_code_location)) root = tree.getroot() SCHEME = "{https://jeremylong.github.io/DependencyCheck/dependency" \ "-check.1.3.xsd}" results = [] for dependency in root.iterfind(".//{}dependency".format(SCHEME)): # Get dep info vulnerabilities = dependency.findall(".//{}vulnerability".format(SCHEME)) for vulnerability in vulnerabilities: advisory = getattr( vulnerability.find("{}name".format(SCHEME)), "text", "") severity = getattr( vulnerability.find("{}severity".format(SCHEME)), "text", "") summary = getattr( vulnerability.find("{}description".format( SCHEME)), "text", "") for vulnerable_version in vulnerability.findall( ".//{}vulnerableSoftware/{}software[" "@allPreviousVersion='true']".format( SCHEME, SCHEME)): cpe = CPE(vulnerable_version.text) library = cpe.get_product()[0] version = cpe.get_version()[0] results.append(dict(library=library, version=version, severity=severity, summary=summary, advisory=advisory)) return results
def __init__(self, cve_entry, thread: ThreadPool = None): self.id = cve_entry.find(prefixed('vuln', 'cve-id')).text self.vulnsoftware = cve_entry.find( prefixed('vuln', 'vulnerable-software-list')) self.part = [] self.vendor = [] self.version = [] self.update = [] self.edition = [] self.language = [] self.product = [] self.year_path = os.path.join("CVE_Detail", self.get_year()) self.cvssscore = None self.accessVector = None self.accessComplexity = None self.auth = None self.impactConf = None self.impactInt = None self.impactAvail = None if self.vulnsoftware is not None: for product in self.vulnsoftware: try: mycpe = CPE(product.text) except NotImplementedError as e: print(e) else: self.part.append(mycpe.get_part()[0]) self.vendor.append(mycpe.get_vendor()[0]) self.version.append(mycpe.get_version()[0]) self.update.append(mycpe.get_update()[0]) self.edition.append(mycpe.get_edition()[0]) self.language.append(mycpe.get_language()[0]) self.product.append(mycpe.get_product()[0]) cvss = cve_entry.find(prefixed('vuln', 'cvss')) if cvss is not None: self.cvssscore = cvss.getchildren()[0].getchildren()[0].text self.accessVector = cvss.getchildren()[0].getchildren()[1].text self.accessComplexity = cvss.getchildren()[0].getchildren()[2].text self.auth = cvss.getchildren()[0].getchildren()[3].text self.impactConf = cvss.getchildren()[0].getchildren()[4].text self.impactInt = cvss.getchildren()[0].getchildren()[5].text self.impactAvail = cvss.getchildren()[0].getchildren()[6].text self.summery = cve_entry.find(prefixed('vuln', 'summary')).text if thread != None: thread.apply_async(self.get_from_pycvesearch)
async def load(self): """Load HassOS data.""" try: assert self.sys_host.info.cpe is not None cpe = CPE(self.sys_host.info.cpe) assert cpe.get_product()[0] == 'hassos' except (NotImplementedError, IndexError, AssertionError): _LOGGER.info("Can't detect HassOS") return # Store meta data self._available = True self._version = cpe.get_version()[0] self._board = cpe.get_target_hardware()[0] _LOGGER.info("Detect HassOS %s on host system", self.version)
async def load(self): """Load HassOS data.""" try: # Check needed host functions assert self.sys_dbus.rauc.is_connected assert self.sys_dbus.systemd.is_connected assert self.sys_dbus.hostname.is_connected assert self.sys_host.info.cpe is not None cpe = CPE(self.sys_host.info.cpe) assert cpe.get_product()[0] == 'hassos' except (AssertionError, NotImplementedError): _LOGGER.debug("Ignore HassOS") return # Store meta data self._available = True self._version = cpe.get_version()[0] self._board = cpe.get_target_hardware()[0] _LOGGER.info("Detect HassOS %s on host system", self.version)
async def load(self) -> None: """Load HassOS data.""" try: # Check needed host functions assert self.sys_dbus.rauc.is_connected assert self.sys_dbus.systemd.is_connected assert self.sys_dbus.hostname.is_connected assert self.sys_host.info.cpe is not None cpe = CPE(self.sys_host.info.cpe) assert cpe.get_product()[0] == "hassos" except (AssertionError, NotImplementedError): _LOGGER.debug("Found no HassOS") return # Store meta data self._available = True self._version = cpe.get_version()[0] self._board = cpe.get_target_hardware()[0] _LOGGER.info("Detect HassOS %s on host system", self.version) with suppress(DockerAPIError): await self.instance.attach()
async def load(self) -> None: """Load HassOS data.""" try: if not self.sys_host.info.cpe: raise NotImplementedError() cpe = CPE(self.sys_host.info.cpe) if cpe.get_product()[0] != "hassos": raise NotImplementedError() except NotImplementedError: _LOGGER.info("No Home Assistant Operating System found") return else: self._available = True # Store meta data self._version = cpe.get_version()[0] self._board = cpe.get_target_hardware()[0] await self.sys_dbus.rauc.update() _LOGGER.info("Detect HassOS %s / BootSlot %s", self.version, self.sys_dbus.rauc.boot_slot)
def sync_es_index(cvesearch_url, esd_url, namelen): """ synchronize vulnsearch esd index with cvesearch data for all cpe notes in storage """ esclient = Elasticsearch([esd_url]) indexer = BulkIndexer(esclient) # create new index. there can be new or updated services or cpe notes of # existing services, also there might be new or updated cves for cpes the # easiest way to handle such complex situation is to create new index and # update alias used by the vulnsearch elk ui objects current_index = f'{ES_INDEX}-{time()}' for note in windowed_query(Note.query.filter(Note.xtype == 'cpe'), Note.id): for icpe in json.loads(note.data): try: parsed_cpe = CPE(icpe) except Exception: # pylint: disable=broad-except ; library does not provide own core exception class current_app.logger.warning(f'invalid cpe, note_id:{note.id} {icpe}') continue if not parsed_cpe.get_version()[0]: continue for cve in cvefor(icpe, cvesearch_url): data_id, data = vulndata(note, parsed_cpe, cve, namelen) indexer.index(current_index, data_id, data) indexer.flush() # update alias and prune old indexes update_managed_indices(esclient, current_index) # print cache stats current_app.logger.debug(f'cvefor cache: {cvefor.cache_info()}') # pylint: disable=no-value-for-parameter ; lru decorator side-effect
async def load(self) -> None: """Load OppOS data.""" try: if not self.sys_host.info.cpe: raise NotImplementedError() cpe = CPE(self.sys_host.info.cpe) if cpe.get_product()[0] != "oppos": raise NotImplementedError() except NotImplementedError: _LOGGER.info("No Open Peer Power Operating System found") return else: self._available = True self.sys_host.supported_features.cache_clear() # Store meta data self._version = AwesomeVersion(cpe.get_version()[0]) self._board = cpe.get_target_hardware()[0] await self.sys_dbus.rauc.update() _LOGGER.info("Detect OppOS %s / BootSlot %s", self.version, self.sys_dbus.rauc.boot_slot)
def populate_CVE(root): cve_data = [] vuln_data = [] for entry in root: cve_id = entry.find(prefixed('vuln', 'cve-id')).text cve_id = int(re.sub("[^0-9]", "", cve_id)) pubdate = entry.find(prefixed('vuln', 'published-datetime')).text moddate = entry.find(prefixed('vuln', 'last-modified-datetime')).text summary = entry.find(prefixed('vuln', 'summary')).text pubdate = parser.parse(pubdate) moddate = parser.parse(moddate) vulnSoftware = entry.find(prefixed('vuln', 'vulnerable-software-list')) vulnList = [] unableToParse=0 if vulnSoftware is not None: for v in vulnSoftware: try: myCPE = CPE(v.text) except NotImplementedError: unableToParse+=1 #logging.warning("Unable to parse CPE '%s'" % v.text) else: part = myCPE.get_part()[0] vendor = myCPE.get_vendor()[0] product = myCPE.get_product()[0] version = myCPE.get_version()[0] update = myCPE.get_update()[0] edition = myCPE.get_edition()[0] language = myCPE.get_language()[0] derpa = {"part" : part, "vendor":vendor, "product":product, "version":version, "update":update, "edition":edition, "language":language, "cve":cve_id} vuln_data.append(derpa) if unableToParse>0: logging.warning("Could not parse %d lines from file." % unableToParse) vuln = entry.find(prefixed('vuln','cvss')) #metrics = vuln.find(prefixed('cvss','base_metrics')) if vuln is not None: score = vuln.getchildren()[0].getchildren()[0].text accessVector = vuln.getchildren()[0].getchildren()[1].text accessComplexity = vuln.getchildren()[0].getchildren()[2].text auth = vuln.getchildren()[0].getchildren()[3].text impactConf = vuln.getchildren()[0].getchildren()[4].text impactInt = vuln.getchildren()[0].getchildren()[5].text impactAvail = vuln.getchildren()[0].getchildren()[6].text if "DO NOT USE THIS CANDIDATE NUMBER" not in summary: data = { "cve":cve_id, "pubdate":pubdate, "moddate":moddate, "summary":summary, "score":score, "accessVector":accessVector, "accessComp":accessComplexity, "auth":auth, "impactConf": impactConf, "impactInt": impactInt, "impactAvail": impactAvail } cve_data.append(data) tables['CVEs'].insert().execute(cve_data) tables['VulnSoftware'].insert().execute(vuln_data)
def get_findings(self, filename, test: Test): content = filename.read() if type(content) is bytes: content = content.decode('utf-8') csv.field_size_limit(int(sys.maxsize / 10)) # the request/resp are big reader = csv.DictReader(io.StringIO(content)) dupes = dict() for row in reader: # manage severity from two possible columns 'Severity' and 'Risk' severity = 'Info' if 'Severity' in row: severity = self._convert_severity(row.get('Severity')) elif 'Risk' in row: severity = self._convert_severity(row.get('Risk')) # manage title from two possible columns 'Nme' and 'Plugin Name' title = row.get('Name') if title is None and 'Plugin Name' in row: title = row.get('Plugin Name') # special case to skip empty titles if not title: continue description = row.get('Synopsis') mitigation = str(row.get('Solution')) impact = row.get('Description', 'N/A') references = row.get('See Also', 'N/A') dupe_key = severity + title + row.get('Host', 'No host') + str( row.get('Port', 'No port')) + row.get('Synopsis', 'No synopsis') detected_cve = self._format_cve(str(row.get('CVE'))) cve = None if detected_cve: # FIXME support more than one CVE in Nessus CSV parser cve = detected_cve[0] if len(detected_cve) > 1: LOGGER.warning( "more than one CVE for a finding. NOT supported by Nessus CSV parser" ) if dupe_key in dupes: find = dupes[dupe_key] if 'Plugin Output' in row: find.description += row.get('Plugin Output') else: if 'Plugin Output' in row: description = description + str(row.get('Plugin Output')) find = Finding(title=title, test=test, cve=cve, description=description, severity=severity, mitigation=mitigation, impact=impact, references=references) # manage CVSS vector (only v3.x for now) if 'CVSS V3 Vector' in row and '' != row.get('CVSS V3 Vector'): find.cvssv3 = CVSS3('CVSS:3.0/' + str(row.get('CVSS V3 Vector')) ).clean_vector(output_prefix=False) # manage CPE data detected_cpe = self._format_cpe(str(row.get('CPE'))) if detected_cpe: # FIXME support more than one CPE in Nessus CSV parser if len(detected_cpe) > 1: LOGGER.warning( "more than one CPE for a finding. NOT supported by Nessus CSV parser" ) cpe_decoded = CPE(detected_cpe[0]) find.component_name = cpe_decoded.get_product()[0] if len( cpe_decoded.get_product()) > 0 else None find.component_version = cpe_decoded.get_version( )[0] if len(cpe_decoded.get_version()) > 0 else None find.unsaved_endpoints = list() dupes[dupe_key] = find # manage endpoints endpoint = Endpoint(host='localhost') if 'Host' in row: endpoint.host = row.get('Host') elif 'IP Address' in row: endpoint.host = row.get('IP Address') endpoint.port = row.get('Port') if 'Protocol' in row: endpoint.protocol = row.get('Protocol').lower() find.unsaved_endpoints.append(endpoint) return list(dupes.values())
def get_component_name_and_version_from_dependency(self, dependency, related_dependency, namespace): component_name, component_version = None, None # big try catch to avoid crashint the parser on some unexpected stuff try: identifiers_node = dependency.find(namespace + 'identifiers') if identifiers_node: # <identifiers> # <identifier type="cpe" confidence="HIGHEST"> # <name>cpe:/a:apache:xalan-java:2.7.1</name> # <url>https://web.nvd.nist.gov/view/vuln/search-results?adv_search=true&cves=on&cpe_version=cpe%3A%2Fa%3Aapache%3Axalan-java%3A2.7.1</url> # </identifier> # <identifier type="maven" confidence="HIGHEST"> # <name>xalan:serializer:2.7.1</name> # <url>https://search.maven.org/remotecontent?filepath=xalan/serializer/2.7.1/serializer-2.7.1.jar</url> # </identifier> # </identifiers> # newly found in v6.0.0 # <identifiers> # <package confidence="HIGH"> # <id>pkg:maven/nl.isaac.client.offerservice/[email protected]</id> # <url>https://ossindex.sonatype.org/component/pkg:maven/nl.isaac.client.offerservice/[email protected]</url> # </package> # </identifiers> # <identifiers> # <package confidence="HIGHEST"> # <id>pkg:npm/[email protected]</id> # <url>https://ossindex.sonatype.org/component/pkg:npm/[email protected]</url> # </package> # </identifiers> package_node = identifiers_node.find('.//' + namespace + 'package') if package_node: logger.debug( 'package string: ' + self.get_field_value(package_node, 'id', namespace)) id = self.get_field_value(package_node, 'id', namespace) purl = PackageURL.from_string(id) purl_parts = purl.to_dict() component_name = purl_parts[ 'namespace'] + ':' if purl_parts['namespace'] and len( purl_parts['namespace']) > 0 else '' component_name += purl_parts['name'] if purl_parts[ 'name'] and len(purl_parts['name']) > 0 else '' component_name = component_name if component_name else None component_version = purl_parts['version'] if purl_parts[ 'version'] and len(purl_parts['version']) > 0 else '' return component_name, component_version cpe_node = identifiers_node.find('.//' + namespace + 'identifier[@type="cpe"]') if cpe_node: # logger.debug('cpe string: ' + self.get_field_value(cpe_node, 'name')) cpe = CPE(self.get_field_value(cpe_node, 'name')) component_name = cpe.get_vendor()[0] + ':' if len( cpe.get_vendor()) > 0 else '' component_name += cpe.get_product()[0] if len( cpe.get_product()) > 0 else '' component_name = component_name if component_name else None component_version = cpe.get_version()[0] if len( cpe.get_version()) > 0 else None # logger.debug('get_edition: ' + str(cpe.get_edition())) # logger.debug('get_language: ' + str(cpe.get_language())) # logger.debug('get_part: ' + str(cpe.get_part())) # logger.debug('get_software_edition: ' + str(cpe.get_software_edition())) # logger.debug('get_target_hardware: ' + str(cpe.get_target_hardware())) # logger.debug('get_target_software: ' + str(cpe.get_target_software())) # logger.debug('get_vendor: ' + str(cpe.get_vendor())) # logger.debug('get_update: ' + str(cpe.get_update())) return component_name, component_version maven_node = identifiers_node.find('.//' + namespace + 'identifier[@type="maven"]') if maven_node: # logger.debug('maven_string: ' + self.get_field_value(maven_node, 'name')) maven_parts = self.get_field_value(maven_node, 'name', namespace).split(':') # logger.debug('maven_parts:' + str(maven_parts)) if len(maven_parts) == 3: component_name = maven_parts[0] + ':' + maven_parts[1] component_version = maven_parts[2] return component_name, component_version # TODO # include identifiers in description? # <identifiers> # <package confidence="HIGH"> # <id>pkg:maven/org.dom4j/[email protected]</id> # <url>https://ossindex.sonatype.org/component/pkg:maven/org.dom4j/[email protected]</url> # </package> # <vulnerabilityIds confidence="HIGHEST"> # <id>cpe:2.3:a:dom4j_project:dom4j:2.1.1.hat-00001:*:*:*:*:*:*:*</id> # <url>https://nvd.nist.gov/vuln/search/results?form_type=Advanced&results_type=overview&search_type=all&cpe_vendor=cpe%3A%2F%3Adom4j_project&cpe_product=cpe%3A%2F%3Adom4j_project%3Adom4j&cpe_version=cpe%3A%2F%3Adom4j_project%3Adom4j%3A2.1.1.hat-00001</url> # </vulnerabilityIds> # TODO what happens when there multiple evidencecollectednodes with product or version as type? evidence_collected_node = dependency.find(namespace + 'evidenceCollected') if evidence_collected_node: # <evidenceCollected> # <evidence type="product" confidence="HIGH"> # <source>file</source> # <name>name</name> # <value>jquery</value> # </evidence> # <evidence type="version" confidence="HIGH"> # <source>file</source> # <name>version</name> # <value>3.1.1</value> # </evidence>' # will find the first product and version node. if there are multiple it may not pick the best # since 6.0.0 howoever it seems like there's always a packageurl above so not sure if we need the effort to # implement more logic here product_node = evidence_collected_node.find( './/' + namespace + 'evidence[@type="product"]') if product_node: component_name = self.get_field_value( product_node, 'value', namespace) version_node = evidence_collected_node.find( './/' + namespace + 'evidence[@type="version"]') if version_node: component_version = self.get_field_value( version_node, 'value', namespace) return component_name, component_version except: logger.exception( 'error parsing component_name and component_version') logger.debug( 'dependency: %s', ElementTree.tostring(dependency, encoding='utf8', method='xml')) return component_name, component_version
def get_component_name_and_version_from_dependency(self, dependency, related_dependency, namespace): identifiers_node = dependency.find(namespace + 'identifiers') if identifiers_node: # analyzing identifier from the more generic to package_node = identifiers_node.find('.//' + namespace + 'package') if package_node: id = package_node.findtext(f'{namespace}id') purl = PackageURL.from_string(id) purl_parts = purl.to_dict() component_name = purl_parts['namespace'] + ':' if purl_parts[ 'namespace'] and len(purl_parts['namespace']) > 0 else '' component_name += purl_parts['name'] if purl_parts[ 'name'] and len(purl_parts['name']) > 0 else '' component_name = component_name if component_name else None component_version = purl_parts['version'] if purl_parts[ 'version'] and len(purl_parts['version']) > 0 else '' return component_name, component_version # vulnerabilityIds_node = identifiers_node.find('.//' + namespace + 'vulnerabilityIds') # if vulnerabilityIds_node: # id = vulnerabilityIds_node.findtext(f'{namespace}id') # cpe = CPE(id) # component_name = cpe.get_vendor()[0] + ':' if len(cpe.get_vendor()) > 0 else '' # component_name += cpe.get_product()[0] if len(cpe.get_product()) > 0 else '' # component_name = component_name if component_name else None # component_version = cpe.get_version()[0] if len(cpe.get_version()) > 0 else None # return component_name, component_version cpe_node = identifiers_node.find('.//' + namespace + 'identifier[@type="cpe"]') if cpe_node: id = cpe_node.findtext(f'{namespace}name') cpe = CPE(id) component_name = cpe.get_vendor()[0] + ':' if len( cpe.get_vendor()) > 0 else '' component_name += cpe.get_product()[0] if len( cpe.get_product()) > 0 else '' component_name = component_name if component_name else None component_version = cpe.get_version()[0] if len( cpe.get_version()) > 0 else None return component_name, component_version maven_node = identifiers_node.find('.//' + namespace + 'identifier[@type="maven"]') if maven_node: maven_parts = maven_node.findtext(f'{namespace}name').split( ':') # logger.debug('maven_parts:' + str(maven_parts)) if len(maven_parts) == 3: component_name = maven_parts[0] + ':' + maven_parts[1] component_version = maven_parts[2] return component_name, component_version # TODO what happens when there multiple evidencecollectednodes with product or version as type? evidence_collected_node = dependency.find(namespace + 'evidenceCollected') if evidence_collected_node: # <evidenceCollected> # <evidence type="product" confidence="HIGH"> # <source>file</source> # <name>name</name> # <value>jquery</value> # </evidence> # <evidence type="version" confidence="HIGH"> # <source>file</source> # <name>version</name> # <value>3.1.1</value> # </evidence>' # will find the first product and version node. if there are multiple it may not pick the best # since 6.0.0 howoever it seems like there's always a packageurl above so not sure if we need the effort to # implement more logic here product_node = evidence_collected_node.find( './/' + namespace + 'evidence[@type="product"]') if product_node: component_name = product_node.findtext(f'{namespace}value') version_node = evidence_collected_node.find( './/' + namespace + 'evidence[@type="version"]') if version_node: component_version = version_node.findtext( f'{namespace}value') return component_name, component_version return None, None
def get_cpe_df(self, debug=False): """Get the list of CPE names for the vulnerability. """ type_list = [] part_list = [] vendor_list = [] product_list = [] version_list = [] update_list = [] edition_list = [] language_list = [] sw_edition_list = [] target_sw_list = [] target_hw_list = [] other_list = [] published_datetime_list = [] for cpe_entry in self.cpe_list: #if(debug): #print(cpe_entry) try: cp = CPE(cpe_entry) if(cp.is_hardware()): type_list.append("HW") elif(cp.is_operating_system()): type_list.append("OS") elif(cp.is_application()): type_list.append("APP") else: type_list.append("UNDEFINED") part_list.append(list_to_string(cp.get_part())) vendor_list.append(list_to_string(cp.get_vendor())) product_list.append(list_to_string(cp.get_product())) version_list.append(list_to_string(cp.get_version())) update_list.append(list_to_string(cp.get_update())) edition_list.append(list_to_string(cp.get_edition())) language_list.append(list_to_string(cp.get_language())) sw_edition_list.append(list_to_string(cp.get_software_edition())) target_sw_list.append(list_to_string(cp.get_target_software())) target_hw_list.append(list_to_string(cp.get_target_hardware())) other_list.append(list_to_string(cp.get_other())) published_datetime_list.append(self.published_datetime) except Exception as inst: print(inst) data = pd.DataFrame() data['type'] = type_list data['part'] = part_list data['vendor'] = vendor_list data['product'] = product_list data['version'] = version_list data['update'] = update_list data['edition'] = edition_list data['language'] = language_list data['sw_edition'] = sw_edition_list data['target_sw'] = target_sw_list data['target_hw'] = target_hw_list data['other'] = other_list data['published_datetime'] = published_datetime_list return data
class Service(object): """ Represents service/application/operating system. Contains basic information: name, version """ _CPE_SPECIAL = r"\!|\"|\;|\#|\$|\%|\&|\'|\(|\)|\+|\,|\/|\:|\<|\=|\>|\@|\[|\]|\^|\`|\{|\||\}|\~|\-" _ESCAPE_CPE = re.compile(_CPE_SPECIAL) _UNESCAPE_CPE = re.compile(r"(\\({0}))".format(_CPE_SPECIAL)) def __init__(self, name=None, version=None, cpe=None): self.name = name self.version = version self._cpe = None self.cpe = cpe @property def cpe(self) -> CPE: """ CPE representation of service """ return self._cpe @cpe.setter def cpe(self, value): if value: self._cpe = CPE(value) @property def cpe_vendor(self) -> str: """ Get vendor name based on CPE """ if isinstance(self._cpe, CPE): return self._unescape_cpe(" ".join(self._cpe.get_vendor())) @property def name_with_version(self) -> str: """ Service name with version included """ if self.version is None or self.name is None: return None return "{name} {version}".format(name=self.name, version=self.version) @property def cpe_product(self) -> str: """ Get product name based on CPE """ if isinstance(self._cpe, CPE): return self._unescape_cpe(" ".join(self._cpe.get_product())) @property def cpe_version(self) -> str: """ Get product name based on CPE """ if isinstance(self._cpe, CPE): return self._unescape_cpe(" ".join(self._cpe.get_version())) def __str__(self): return "{name} {version}".format(name=self.name or '', version=self.version or '').strip() def copy(self) -> 'Service': """ Make copy of service """ return_value = Service(name=self.name, version=self.version) return_value._cpe = self._cpe return return_value @classmethod def _escape_cpe(cls, text: str) -> str: """ Special characters should be escaped before building CPE string """ text = text.lower() def _replace(txt): return r"\{0}".format(txt.group()) if " " in text: raise ValueError("{0}: Space is not allowed in CPE string".format(text)) return cls._ESCAPE_CPE.sub(_replace, text) @classmethod def _unescape_cpe(cls, text: str) -> str: text = text.lower() def _replace(txt): return txt.group()[1] return cls._UNESCAPE_CPE.sub(_replace, text) @classmethod def validate_cpe_arguments(cls, vendor: str, product: str, version: str) -> (str, str, str): """ Validate cpe arguments, and fix as much as possible """ if " " in version: if product.lower() == "ios": version = version.split(" ")[0].strip(",") if vendor == "*": if product.lower() == "ios": vendor = "cisco" return cls._escape_cpe(vendor), cls._escape_cpe(product), cls._escape_cpe(version) @classmethod def build_cpe(cls, part: 'CPEType', vendor: str = '*', product: str = '*', version: str = '*') -> str: """ Build cpe 2.3 string base on vendor, product, version and part """ vendor, product, version = cls.validate_cpe_arguments(vendor=vendor, product=product, version=version) return "cpe:2.3:{part}:{vendor}:{product}:{version}:*:*:*:*:*:*:*".format(part=str(part.value), vendor=vendor, product=product, version=version)
def extract_vendor_product_version(cpe_str): """Extract vendor and product from NVD cve entry.""" cpe_ = CPE(cpe_str) return cpe_.get_vendor()[0], cpe_.get_product()[0], cpe_.get_version()[0]
def get_item(vuln, test): finding = Finding( test=test, unique_id_from_tool=vuln["id"], nb_occurences=1, ) # Defining variables location = vuln["location"] # Endpoint # using url if "url" in location and location["url"] and location["url"] != "None": endpoint = Endpoint.from_uri(location["url"]) # fallback to using old way of creating endpoints elif "domain" in location and location["domain"] and location["domain"] != "None": endpoint = Endpoint(host=str(location["domain"])) else: # no domain, use ip instead if "ip" in location and location["ip"] and location["ip"] != "None": endpoint = Endpoint(host=str(location["ip"])) # check for protocol if ( "applicationProtocol" in location and location["applicationProtocol"] and location["applicationProtocol"] != "None" ): endpoint.protocol = location["applicationProtocol"] # check for port if ( "port" in location and location["port"] in location and location["port"] != "None" ): endpoint.port = location["port"] finding.unsaved_endpoints = [endpoint] # assigning endpoint # Title finding.title = vuln["name"] # Description + CVEs description = vuln["classification"] cves = "no match" if "CVE-NO-MATCH" not in vuln["kb"]["cves"]: finding.cve = vuln["kb"]["cves"][0] cves = "" for cve in vuln["kb"]["cves"]: cves += f"{cve}, " cves = cves[: len(cves) - 2] # removing the comma and the blank space finding.description = description + "; CVEs: " + cves finding.severity = vuln["severity"].title() # Date date_str = vuln["createdOn"] date_str = date_str[: len(date_str) - 3] + date_str[-2:] finding.date = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%f%z") # Component Name and Version if ( "applicationCpe" in location and location["applicationCpe"] and location["applicationCpe"] != "None" ): cpe = CPE(location["applicationCpe"]) component_name = cpe.get_vendor()[0] + ":" if len( cpe.get_vendor()) > 0 else "" component_name += cpe.get_product()[0] if len( cpe.get_product()) > 0 else "" finding.component_name = component_name if component_name else None finding.component_version = ( cpe.get_version()[0] if len(cpe.get_version()) > 0 else None ) return finding
def populate_CVE(root): cve_data = [] vuln_data = [] for entry in root: cve_id = entry.find(prefixed("vuln", "cve-id")).text cve_id = int(re.sub("[^0-9]", "", cve_id)) pubdate = entry.find(prefixed("vuln", "published-datetime")).text moddate = entry.find(prefixed("vuln", "last-modified-datetime")).text summary = entry.find(prefixed("vuln", "summary")).text pubdate = parser.parse(pubdate) moddate = parser.parse(moddate) vulnSoftware = entry.find(prefixed("vuln", "vulnerable-software-list")) vulnList = [] unableToParse = 0 if vulnSoftware is not None: for v in vulnSoftware: try: myCPE = CPE(v.text) except NotImplementedError: unableToParse += 1 # logging.warning("Unable to parse CPE '%s'" % v.text) else: part = myCPE.get_part()[0] vendor = myCPE.get_vendor()[0] product = myCPE.get_product()[0] version = myCPE.get_version()[0] update = myCPE.get_update()[0] edition = myCPE.get_edition()[0] language = myCPE.get_language()[0] derpa = { "part": part, "vendor": vendor, "product": product, "version": version, "update": update, "edition": edition, "language": language, "cve": cve_id, } vuln_data.append(derpa) if unableToParse > 0: logging.warning("Could not parse %d lines from file." % unableToParse) vuln = entry.find(prefixed("vuln", "cvss")) # metrics = vuln.find(prefixed('cvss','base_metrics')) if vuln is not None: score = vuln.getchildren()[0].getchildren()[0].text accessVector = vuln.getchildren()[0].getchildren()[1].text accessComplexity = vuln.getchildren()[0].getchildren()[2].text auth = vuln.getchildren()[0].getchildren()[3].text impactConf = vuln.getchildren()[0].getchildren()[4].text impactInt = vuln.getchildren()[0].getchildren()[5].text impactAvail = vuln.getchildren()[0].getchildren()[6].text if "DO NOT USE THIS CANDIDATE NUMBER" not in summary: data = { "cve": cve_id, "pubdate": pubdate, "moddate": moddate, "summary": summary, "score": score, "accessVector": accessVector, "accessComp": accessComplexity, "auth": auth, "impactConf": impactConf, "impactInt": impactInt, "impactAvail": impactAvail, } cve_data.append(data) tables["CVEs"].insert().execute(cve_data) tables["VulnSoftware"].insert().execute(vuln_data)