def addGithubAdvisory(alert):
vuln = alert['securityAdvisory']
ghsa = vuln['ghsaId']
vulnId = common.getVulnerabilityId(None, ghsa)
if vulnId > 0:
return vulnId
description = vuln['description']
publishDate = dt.parse(vuln['publishedAt'])
q = 'insert into vulnerability values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
sql.execute(q, (None, 'GitHub', None, ghsa, publishDate, description, None,
None, None, None))
return common.getVulnerabilityId(None, ghsa)
def addSteadyVulnerability(vuln):
#process this vuln
sourceId = vuln['bug']['id']
vulnId = common.getVulnerabilityId(None, sourceId)
if vulnId > 0:
return vulnId
insertQ = 'insert into vulnerability values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
sql.execute(
insertQ,
(None, 'Steady', None, sourceId, None, None, None, None, None, None))
return common.getVulnerabilityId(None, sourceId)
def getCVEids(cves, packageId):
ids=[]
for cve in cves:
if not cve.startswith('CVE'):
raise Exception('non cve vulnerability in victims report', cve)
ids.append(common.getVulnerabilityId(cve,None))
return ids
def readAdvisories(advisories):
'''
Get advisory list from npm audit
retrieve corresponding vulnerabilityId
and returns the mapping
'''
hm = {}
severity = {}
for k in advisories.keys():
data = advisories[k]
vulnIds = []
flag = True #True if no valid CVE associated
if 'cves' in data.keys() and len(data['cves']) > 0:
for cve in data['cves']:
assert cve.startswith('CVE')
vulnId = common.getVulnerabilityId(cve, None)
if vulnId > 0:
vulnIds.append(vulnId)
severity[vulnId] = data['severity']
flag = False
if flag:
vulnId = getNPMVulnerability(data)
vulnIds.append(vulnId)
severity[vulnId] = data['severity']
hm[data['id']] = vulnIds
return hm, severity
def processNpmAlert(repoId, alert, hm):
artifact = alert['securityVulnerability']['package']['name']
group = 'npm'
version = alert['vulnerableRequirements'][2:]
packageId = common.getPackageId(group,
artifact,
version,
ecosystem='npm',
insertIfNotExists=True)
dependencyId = common.getDependencyId(repoId,
packageId,
idtool=toolId,
insertIfNotExists=True)
cve = getCVE(alert)
if cve:
vulnId = common.getVulnerabilityId(cve, None)
else:
vulnId = addGithubAdvisory(alert)
severity = alert['securityAdvisory']['severity']
if (dependencyId, vulnId, toolId) not in hm:
hm[(dependencyId, vulnId, toolId)] = {'severity': severity, 'count': 0}
else:
hm[(dependencyId, vulnId, toolId)]['count'] += 1
def getNPMVulnerability(data):
sourceId = 'NPM-' + str(data['id'])
vulnId = common.getVulnerabilityId(None, sourceId)
if vulnId > 0:
return vulnId
publishDate = dt.parse(data['created'])
description = data['title'] + data['overview']
insertQ = 'insert into vulnerability values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
try:
sql.execute(insertQ, (None, 'NPM', None, sourceId, publishDate,
description, None, None, None, None))
except sql.pymysql.IntegrityError as error:
if error.args[0] == sql.PYMYSQL_DUPLICATE_ERROR:
print(sourceId, ' already exists')
else:
raise Exception(str(error))
return common.getVulnerabilityId(None, sourceId)
def process_vuln(vuln, cvss):
headers = vuln.find_all('p')
publishDate = url = description = None
for header in headers:
if header.text.strip() == '':
continue
if 'Publish Date' in header.text:
publishDate = header.text
elif 'URL:' in header.text:
url = header.text
else:
description = header.text
assert publishDate and url and description
url = url.strip()
publishDate = publishDate.strip()
publishDate = publishDate[:-len(url)]
publishDate = publishDate[len('Publish Date:') + 1:].strip()
publishDate = dt.parse(publishDate)
id = url[len('URL:'):].strip()
wsid = True
if 'CVE' in id:
vulnId = common.getVulnerabilityId(id, None)
if vulnId > 0:
wsid = False
if wsid:
vulnId = common.getVulnerabilityId(None, id)
if vulnId <= 0:
vulnId = addWhiteSourceVulnerability(id, publishDate, description,
cvss)
vulnId = common.getVulnerabilityId(None, id)
return vulnId
def processMavenModules(repoId, mavenModules):
d = {}
for module in mavenModules:
print(module['projectName'])
for vuln in module['vulnerabilities']:
group = vuln['mavenModuleName']['groupId']
artifact = vuln['mavenModuleName']['artifactId']
version = vuln['version']
packageId = common.getPackageId(group, artifact, version)
dependencyId = common.getDependencyId(repoId, packageId)
dependencyPath = constructDependencyPath(vuln['from'])
dependencyPathId = common.getDependencyPathId(dependencyPath)
ids = vuln['identifiers']
severity = vuln['severity']
vulnIds = []
snykFlag = True
if ids['CVE']:
snykFlag = False
#CVE id present
for cve in ids['CVE']:
vulnId = common.getVulnerabilityId(cve, None)
if vulnId == -1:
if len(ids['CVE']) == 1:
snykFlag = True
continue
vulnIds.append(vulnId)
addVulnerabilityInfo(vulnId, vuln)
if snykFlag:
vulnId = addSnykVulenrability(vuln)
vulnIds.append(vulnId)
addVulnerabilityInfo(vulnId, vuln)
for vulnId in vulnIds:
if (vulnId, dependencyId) not in d:
d[(vulnId, dependencyId)] = {
'count': 1,
'severity': severity
}
else:
d[(vulnId, dependencyId)]['count'] += 1
addSnykInfo(vuln, dependencyPathId, repoId, 'maven')
return d
def process_vulnerabilities(repoId, data, allLibraries):
#get vuln id
srcclrId = True
assert 'cve' in data.keys()
if data['cve'] is not None:
cveId = 'CVE-' + data['cve']
vulnId = common.getVulnerabilityId(cveId, None)
if vulnId > 0:
srcclrId = False
print(cveId)
if srcclrId:
vulnId = getSrcClrVulnerability(data)
depIds = []
for library in data['libraries']:
ref = library['_links']['ref']
assert ref.startswith('/records/0/libraries/')
ref = ref[len('/records/0/libraries/'):]
ref = ref.split('/')
p, v = int(ref[0]), int(ref[-1])
package = allLibraries[p]
group = package['coordinate1']
artifact = package['coordinate2']
version = package['versions'][v]['version']
packageId = common.getPackageId(group, artifact, version, 'maven',
True)
dependencyId = common.getDependencyId(repoId, packageId, toolId, True)
depIds.append(dependencyId)
for dependencyId in depIds:
insertQ = 'insert into mavenAlert values(%s,%s,%s,%s,%s,%s,%s,%s)'
try:
sql.execute(insertQ, (None, None, dependencyId, vulnId, toolId,
None, None, 1 / len(depIds)))
except sql.pymysql.IntegrityError as error:
if error.args[0] == sql.PYMYSQL_DUPLICATE_ERROR:
#TODO update scandate
print('maven alert exists already in db')
else:
raise Exception(str(error))
def owaspVulnerabiltyId(dependency, source, cve, cwe, description,
vulnerability, CVSS2_severity, CVSS2_score,
CVSS3_severity, CVSS3_score):
if cve.startswith('CVE'):
vulnId = common.getVulnerabilityId(cve, None)
if vulnId > 0:
return vulnId
#non CVEs for OWASP can vary based on both dependency and description in cve columns
sourceId = '-'.join(['OWASP', cve, dependency])
vulnId = common.getVulnerabilityId(None, sourceId)
if vulnId > 0:
return vulnId
#nan values are creating issue for pymysql
args = (dependency, source, cve, cwe, description, vulnerability,
CVSS2_severity, CVSS2_score, CVSS3_severity, CVSS3_score)
dependency, source, cve, cwe, description, vulnerability, \
CVSS2_severity, CVSS2_score, \
CVSS3_severity, CVSS3_score = map(common.changeNaNToNone, args)
if description:
description = description.replace('"', ' ').replace('\\', '')
if vulnerability:
vulnerability = vulnerability.replace('"', ' ').replace('\\', '')
if description or vulnerability:
if description:
description += ' : '
else:
description = ''
if vulnerability:
description += vulnerability
insertQ = 'insert into vulnerability values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
try:
sql.execute(insertQ,
(None, source, None, sourceId, None, description,
CVSS2_score, CVSS2_severity, CVSS3_score, CVSS3_severity))
except sql.pymysql.IntegrityError as error:
if error.args[0] == sql.PYMYSQL_DUPLICATE_ERROR:
print(sourceId, ' already exists')
else:
raise Exception(str(error))
vulnId = common.getVulnerabilityId(None, sourceId)
if not cwe or 'NVD-CWE' in cwe:
cwes = [-1]
else:
cwes = []
cweTexts = cwe.split(',')
for text in cweTexts:
text = text.strip()
text = text.split(' ')[0]
assert text.startswith('CWE-')
id = text[len('CWE-'):]
cwes.append(id)
common.addCWEs(vulnId, cwes)
return vulnId
if not component['Vulnerabilities']:
#no vulnerability present
continue
#get package id
artifact=component['Name']
version=component['Version']
if not version:
version='%' #any version
else:
version+='%' #Some suffixes like RELEASE could be ignored in seeker
selectQ='''select id from package
where artifact=%s and version like %s'''
results=sql.execute(selectQ,(artifact,version))
if not results:
insertPackage(artifact,version[:-1])
results=sql.execute(selectQ,(artifact,version))
packageId= results[0]['id']
labels = ','.join(component['Labels'])
for vuln in component['Vulnerabilities']:
if 'CVE' not in vuln.keys():
raise Exception('checl this', vuln)
cve=vuln['CVE']
vulnId=common.getVulnerabilityId(cve,None)
q='insert into seeker values(%s,%s,%s)'
sql.execute(q,(packageId, vulnId, labels))
def processReport(repoName, repoId):
filename = repoName + '-vulas.json'
with open(filename, 'r') as file:
data = json.loads(file.read())['vulasReport']
if 'vulnerabilities' not in data.keys() or len(
data['vulnerabilities']) == 0:
return
scandate = data['generatedAt']
vulnerabilities = data['vulnerabilities']
for vuln in vulnerabilities:
#get package id
package = vuln['filename']
if not package.endswith('.jar') or 'jar' in package[:-4]:
#second condition ensures single package
raise Exception('package not jar', package)
package = package[:-4]
q = '''select id
from package
where concat(artifact,'-',version) = %s;'''
packageId = sql.execute(q, (package, ))[0]['id']
dependencyId = common.getDependencyId(repoId, packageId)
steadyId = True
#get cve id
if vuln['bug']['id'].startswith(
'CVE') and not 'CVE' in vuln['bug']['id'][3:]:
#second condition ensures single CVE
cve = vuln['bug']['id']
if len(cve.split('-')) > 3:
#some in vulas has extra addendums
print("check", cve)
cve = '-'.join(x for x in cve.split('-')[:3])
vulnerabilityId = common.getVulnerabilityId(cve, None)
if vulnerabilityId > 0:
steadyId = False
if steadyId:
vulnerabilityId = addSteadyVulnerability(vuln)
integrationTest = []
#DONE: write in a way so that first it can be run for only unit test
#then again for integration test as well
#check if already alert is pushed for the first three
alertId = getAlertId(scandate, dependencyId, vulnerabilityId)
vulnerableVersion = []
staticAnalysis = []
unitTest = []
for module in vuln['modules']:
#get the static and dynamic analysis results into a separate table
vulnerableVersion.append(hm[module['containsVulnerableCode']])
staticAnalysis.append(
hm[module['potentiallyExecutesVulnerableCode']])
unitTest.append(hm[module['actuallyExecutesVulnerableCode']])
vv = max(vulnerableVersion)
sa = max(staticAnalysis)
ut = max(unitTest)
q = 'insert into steady values(%s,%s,%s,%s,%s)'
try:
sql.execute(
q, (alertId, inv_hm[vv], inv_hm[sa], inv_hm[ut], None))
except sql.pymysql.IntegrityError as error:
if error.args[0] == sql.PYMYSQL_DUPLICATE_ERROR:
#TODO update scandate
print('maven alert exists already in steady db')
else:
raise Exception(str(error))
