def addGithubAdvisory(alert): vuln = alert['securityAdvisory'] ghsa = vuln['ghsaId'] vulnId = common.getVulnerabilityId(None, ghsa) if vulnId > 0: return vulnId description = vuln['description'] publishDate = dt.parse(vuln['publishedAt']) q = 'insert into vulnerability values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)' sql.execute(q, (None, 'GitHub', None, ghsa, publishDate, description, None, None, None, None)) return common.getVulnerabilityId(None, ghsa)
def addSteadyVulnerability(vuln): #process this vuln sourceId = vuln['bug']['id'] vulnId = common.getVulnerabilityId(None, sourceId) if vulnId > 0: return vulnId insertQ = 'insert into vulnerability values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)' sql.execute( insertQ, (None, 'Steady', None, sourceId, None, None, None, None, None, None)) return common.getVulnerabilityId(None, sourceId)
def getCVEids(cves, packageId): ids=[] for cve in cves: if not cve.startswith('CVE'): raise Exception('non cve vulnerability in victims report', cve) ids.append(common.getVulnerabilityId(cve,None)) return ids
def readAdvisories(advisories): ''' Get advisory list from npm audit retrieve corresponding vulnerabilityId and returns the mapping ''' hm = {} severity = {} for k in advisories.keys(): data = advisories[k] vulnIds = [] flag = True #True if no valid CVE associated if 'cves' in data.keys() and len(data['cves']) > 0: for cve in data['cves']: assert cve.startswith('CVE') vulnId = common.getVulnerabilityId(cve, None) if vulnId > 0: vulnIds.append(vulnId) severity[vulnId] = data['severity'] flag = False if flag: vulnId = getNPMVulnerability(data) vulnIds.append(vulnId) severity[vulnId] = data['severity'] hm[data['id']] = vulnIds return hm, severity
def processNpmAlert(repoId, alert, hm): artifact = alert['securityVulnerability']['package']['name'] group = 'npm' version = alert['vulnerableRequirements'][2:] packageId = common.getPackageId(group, artifact, version, ecosystem='npm', insertIfNotExists=True) dependencyId = common.getDependencyId(repoId, packageId, idtool=toolId, insertIfNotExists=True) cve = getCVE(alert) if cve: vulnId = common.getVulnerabilityId(cve, None) else: vulnId = addGithubAdvisory(alert) severity = alert['securityAdvisory']['severity'] if (dependencyId, vulnId, toolId) not in hm: hm[(dependencyId, vulnId, toolId)] = {'severity': severity, 'count': 0} else: hm[(dependencyId, vulnId, toolId)]['count'] += 1
def getNPMVulnerability(data): sourceId = 'NPM-' + str(data['id']) vulnId = common.getVulnerabilityId(None, sourceId) if vulnId > 0: return vulnId publishDate = dt.parse(data['created']) description = data['title'] + data['overview'] insertQ = 'insert into vulnerability values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)' try: sql.execute(insertQ, (None, 'NPM', None, sourceId, publishDate, description, None, None, None, None)) except sql.pymysql.IntegrityError as error: if error.args[0] == sql.PYMYSQL_DUPLICATE_ERROR: print(sourceId, ' already exists') else: raise Exception(str(error)) return common.getVulnerabilityId(None, sourceId)
def process_vuln(vuln, cvss): headers = vuln.find_all('p') publishDate = url = description = None for header in headers: if header.text.strip() == '': continue if 'Publish Date' in header.text: publishDate = header.text elif 'URL:' in header.text: url = header.text else: description = header.text assert publishDate and url and description url = url.strip() publishDate = publishDate.strip() publishDate = publishDate[:-len(url)] publishDate = publishDate[len('Publish Date:') + 1:].strip() publishDate = dt.parse(publishDate) id = url[len('URL:'):].strip() wsid = True if 'CVE' in id: vulnId = common.getVulnerabilityId(id, None) if vulnId > 0: wsid = False if wsid: vulnId = common.getVulnerabilityId(None, id) if vulnId <= 0: vulnId = addWhiteSourceVulnerability(id, publishDate, description, cvss) vulnId = common.getVulnerabilityId(None, id) return vulnId
def processMavenModules(repoId, mavenModules): d = {} for module in mavenModules: print(module['projectName']) for vuln in module['vulnerabilities']: group = vuln['mavenModuleName']['groupId'] artifact = vuln['mavenModuleName']['artifactId'] version = vuln['version'] packageId = common.getPackageId(group, artifact, version) dependencyId = common.getDependencyId(repoId, packageId) dependencyPath = constructDependencyPath(vuln['from']) dependencyPathId = common.getDependencyPathId(dependencyPath) ids = vuln['identifiers'] severity = vuln['severity'] vulnIds = [] snykFlag = True if ids['CVE']: snykFlag = False #CVE id present for cve in ids['CVE']: vulnId = common.getVulnerabilityId(cve, None) if vulnId == -1: if len(ids['CVE']) == 1: snykFlag = True continue vulnIds.append(vulnId) addVulnerabilityInfo(vulnId, vuln) if snykFlag: vulnId = addSnykVulenrability(vuln) vulnIds.append(vulnId) addVulnerabilityInfo(vulnId, vuln) for vulnId in vulnIds: if (vulnId, dependencyId) not in d: d[(vulnId, dependencyId)] = { 'count': 1, 'severity': severity } else: d[(vulnId, dependencyId)]['count'] += 1 addSnykInfo(vuln, dependencyPathId, repoId, 'maven') return d
def process_vulnerabilities(repoId, data, allLibraries): #get vuln id srcclrId = True assert 'cve' in data.keys() if data['cve'] is not None: cveId = 'CVE-' + data['cve'] vulnId = common.getVulnerabilityId(cveId, None) if vulnId > 0: srcclrId = False print(cveId) if srcclrId: vulnId = getSrcClrVulnerability(data) depIds = [] for library in data['libraries']: ref = library['_links']['ref'] assert ref.startswith('/records/0/libraries/') ref = ref[len('/records/0/libraries/'):] ref = ref.split('/') p, v = int(ref[0]), int(ref[-1]) package = allLibraries[p] group = package['coordinate1'] artifact = package['coordinate2'] version = package['versions'][v]['version'] packageId = common.getPackageId(group, artifact, version, 'maven', True) dependencyId = common.getDependencyId(repoId, packageId, toolId, True) depIds.append(dependencyId) for dependencyId in depIds: insertQ = 'insert into mavenAlert values(%s,%s,%s,%s,%s,%s,%s,%s)' try: sql.execute(insertQ, (None, None, dependencyId, vulnId, toolId, None, None, 1 / len(depIds))) except sql.pymysql.IntegrityError as error: if error.args[0] == sql.PYMYSQL_DUPLICATE_ERROR: #TODO update scandate print('maven alert exists already in db') else: raise Exception(str(error))
def owaspVulnerabiltyId(dependency, source, cve, cwe, description, vulnerability, CVSS2_severity, CVSS2_score, CVSS3_severity, CVSS3_score): if cve.startswith('CVE'): vulnId = common.getVulnerabilityId(cve, None) if vulnId > 0: return vulnId #non CVEs for OWASP can vary based on both dependency and description in cve columns sourceId = '-'.join(['OWASP', cve, dependency]) vulnId = common.getVulnerabilityId(None, sourceId) if vulnId > 0: return vulnId #nan values are creating issue for pymysql args = (dependency, source, cve, cwe, description, vulnerability, CVSS2_severity, CVSS2_score, CVSS3_severity, CVSS3_score) dependency, source, cve, cwe, description, vulnerability, \ CVSS2_severity, CVSS2_score, \ CVSS3_severity, CVSS3_score = map(common.changeNaNToNone, args) if description: description = description.replace('"', ' ').replace('\\', '') if vulnerability: vulnerability = vulnerability.replace('"', ' ').replace('\\', '') if description or vulnerability: if description: description += ' : ' else: description = '' if vulnerability: description += vulnerability insertQ = 'insert into vulnerability values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)' try: sql.execute(insertQ, (None, source, None, sourceId, None, description, CVSS2_score, CVSS2_severity, CVSS3_score, CVSS3_severity)) except sql.pymysql.IntegrityError as error: if error.args[0] == sql.PYMYSQL_DUPLICATE_ERROR: print(sourceId, ' already exists') else: raise Exception(str(error)) vulnId = common.getVulnerabilityId(None, sourceId) if not cwe or 'NVD-CWE' in cwe: cwes = [-1] else: cwes = [] cweTexts = cwe.split(',') for text in cweTexts: text = text.strip() text = text.split(' ')[0] assert text.startswith('CWE-') id = text[len('CWE-'):] cwes.append(id) common.addCWEs(vulnId, cwes) return vulnId
if not component['Vulnerabilities']: #no vulnerability present continue #get package id artifact=component['Name'] version=component['Version'] if not version: version='%' #any version else: version+='%' #Some suffixes like RELEASE could be ignored in seeker selectQ='''select id from package where artifact=%s and version like %s''' results=sql.execute(selectQ,(artifact,version)) if not results: insertPackage(artifact,version[:-1]) results=sql.execute(selectQ,(artifact,version)) packageId= results[0]['id'] labels = ','.join(component['Labels']) for vuln in component['Vulnerabilities']: if 'CVE' not in vuln.keys(): raise Exception('checl this', vuln) cve=vuln['CVE'] vulnId=common.getVulnerabilityId(cve,None) q='insert into seeker values(%s,%s,%s)' sql.execute(q,(packageId, vulnId, labels))
def processReport(repoName, repoId): filename = repoName + '-vulas.json' with open(filename, 'r') as file: data = json.loads(file.read())['vulasReport'] if 'vulnerabilities' not in data.keys() or len( data['vulnerabilities']) == 0: return scandate = data['generatedAt'] vulnerabilities = data['vulnerabilities'] for vuln in vulnerabilities: #get package id package = vuln['filename'] if not package.endswith('.jar') or 'jar' in package[:-4]: #second condition ensures single package raise Exception('package not jar', package) package = package[:-4] q = '''select id from package where concat(artifact,'-',version) = %s;''' packageId = sql.execute(q, (package, ))[0]['id'] dependencyId = common.getDependencyId(repoId, packageId) steadyId = True #get cve id if vuln['bug']['id'].startswith( 'CVE') and not 'CVE' in vuln['bug']['id'][3:]: #second condition ensures single CVE cve = vuln['bug']['id'] if len(cve.split('-')) > 3: #some in vulas has extra addendums print("check", cve) cve = '-'.join(x for x in cve.split('-')[:3]) vulnerabilityId = common.getVulnerabilityId(cve, None) if vulnerabilityId > 0: steadyId = False if steadyId: vulnerabilityId = addSteadyVulnerability(vuln) integrationTest = [] #DONE: write in a way so that first it can be run for only unit test #then again for integration test as well #check if already alert is pushed for the first three alertId = getAlertId(scandate, dependencyId, vulnerabilityId) vulnerableVersion = [] staticAnalysis = [] unitTest = [] for module in vuln['modules']: #get the static and dynamic analysis results into a separate table vulnerableVersion.append(hm[module['containsVulnerableCode']]) staticAnalysis.append( hm[module['potentiallyExecutesVulnerableCode']]) unitTest.append(hm[module['actuallyExecutesVulnerableCode']]) vv = max(vulnerableVersion) sa = max(staticAnalysis) ut = max(unitTest) q = 'insert into steady values(%s,%s,%s,%s,%s)' try: sql.execute( q, (alertId, inv_hm[vv], inv_hm[sa], inv_hm[ut], None)) except sql.pymysql.IntegrityError as error: if error.args[0] == sql.PYMYSQL_DUPLICATE_ERROR: #TODO update scandate print('maven alert exists already in steady db') else: raise Exception(str(error))