def processNpmAlert(repoId, alert, hm):
artifact = alert['securityVulnerability']['package']['name']
group = 'npm'
version = alert['vulnerableRequirements'][2:]
packageId = common.getPackageId(group,
artifact,
version,
ecosystem='npm',
insertIfNotExists=True)
dependencyId = common.getDependencyId(repoId,
packageId,
idtool=toolId,
insertIfNotExists=True)
cve = getCVE(alert)
if cve:
vulnId = common.getVulnerabilityId(cve, None)
else:
vulnId = addGithubAdvisory(alert)
severity = alert['securityAdvisory']['severity']
if (dependencyId, vulnId, toolId) not in hm:
hm[(dependencyId, vulnId, toolId)] = {'severity': severity, 'count': 0}
else:
hm[(dependencyId, vulnId, toolId)]['count'] += 1
def getVulns(repoId, table) -> dict:
def getCVEids(cves, packageId):
ids=[]
for cve in cves:
if not cve.startswith('CVE'):
raise Exception('non cve vulnerability in victims report', cve)
ids.append(common.getVulnerabilityId(cve,None))
return ids
rows=table.find_all('tr')
d={}
cur=None
for row in rows:
if row.find_all('th'):
#new module found
cur=row.getText().replace('\n','').replace(' ','')
d[cur]={}
else:
#new alert found
cols=row.find_all('td')
package=cols[0].getText()
group, artifact, version = package.split(':')
packageId=common.getPackageId(group, artifact, version)
dependencyId = common.getDependencyId(repoId, packageId)
cves=(cols[1].getText()).replace('\n','').replace(' ','').split(',')
d[cur][dependencyId] = getCVEids(cves,packageId)
return d
def processMavenModules(repoId, mavenModules):
d = {}
for module in mavenModules:
print(module['projectName'])
for vuln in module['vulnerabilities']:
group = vuln['mavenModuleName']['groupId']
artifact = vuln['mavenModuleName']['artifactId']
version = vuln['version']
packageId = common.getPackageId(group, artifact, version)
dependencyId = common.getDependencyId(repoId, packageId)
dependencyPath = constructDependencyPath(vuln['from'])
dependencyPathId = common.getDependencyPathId(dependencyPath)
ids = vuln['identifiers']
severity = vuln['severity']
vulnIds = []
snykFlag = True
if ids['CVE']:
snykFlag = False
#CVE id present
for cve in ids['CVE']:
vulnId = common.getVulnerabilityId(cve, None)
if vulnId == -1:
if len(ids['CVE']) == 1:
snykFlag = True
continue
vulnIds.append(vulnId)
addVulnerabilityInfo(vulnId, vuln)
if snykFlag:
vulnId = addSnykVulenrability(vuln)
vulnIds.append(vulnId)
addVulnerabilityInfo(vulnId, vuln)
for vulnId in vulnIds:
if (vulnId, dependencyId) not in d:
d[(vulnId, dependencyId)] = {
'count': 1,
'severity': severity
}
else:
d[(vulnId, dependencyId)]['count'] += 1
addSnykInfo(vuln, dependencyPathId, repoId, 'maven')
return d
def processMavenAlerts(mavenDf):
df = mavenDf
if len(df) == 0:
return
df['packageId'] = df.apply(lambda row: getMavenPackageId(
row.dependency, row.identifier, insertIfNotExists=True),
axis=1)
df['dependencyId'] = df.apply(lambda row: common.getDependencyId(
row.repoId, row.packageId, row.toolId, insertIfNotExists=True),
axis=1)
df = df[[
'scandate', 'dependencyId', 'vulnerabilityId', 'toolId', 'confidence'
]]
df['id'] = [np.nan] * len(df)
df['severity'] = [np.nan] * len(df)
df['count'] = [1] * len(df)
df = df[[
'scandate', 'dependencyId', 'vulnerabilityId', 'toolId', 'confidence'
]]
assert len(df[df.duplicated()]) == 0
alerts = df.values
insertQ = 'insert into mavenAlert values(%s,%s,%s,%s,%s,%s,%s,%s)'
for alert in alerts:
for i in range(len(alert)):
if pd.isna(alert[i]):
alert[i] = None
scandate, dependencyId, vulnerabilityId, toolId, confidence = alert
try:
sql.execute(insertQ,
(None, scandate, dependencyId, vulnerabilityId, toolId,
confidence, None, 1))
except sql.pymysql.IntegrityError as error:
if error.args[0] == sql.PYMYSQL_DUPLICATE_ERROR:
#TODO update scandate
print('maven alert exists already in db')
else:
raise Exception(str(error))
def process_vulnerabilities(repoId, data, allLibraries):
#get vuln id
srcclrId = True
assert 'cve' in data.keys()
if data['cve'] is not None:
cveId = 'CVE-' + data['cve']
vulnId = common.getVulnerabilityId(cveId, None)
if vulnId > 0:
srcclrId = False
print(cveId)
if srcclrId:
vulnId = getSrcClrVulnerability(data)
depIds = []
for library in data['libraries']:
ref = library['_links']['ref']
assert ref.startswith('/records/0/libraries/')
ref = ref[len('/records/0/libraries/'):]
ref = ref.split('/')
p, v = int(ref[0]), int(ref[-1])
package = allLibraries[p]
group = package['coordinate1']
artifact = package['coordinate2']
version = package['versions'][v]['version']
packageId = common.getPackageId(group, artifact, version, 'maven',
True)
dependencyId = common.getDependencyId(repoId, packageId, toolId, True)
depIds.append(dependencyId)
for dependencyId in depIds:
insertQ = 'insert into mavenAlert values(%s,%s,%s,%s,%s,%s,%s,%s)'
try:
sql.execute(insertQ, (None, None, dependencyId, vulnId, toolId,
None, None, 1 / len(depIds)))
except sql.pymysql.IntegrityError as error:
if error.args[0] == sql.PYMYSQL_DUPLICATE_ERROR:
#TODO update scandate
print('maven alert exists already in db')
else:
raise Exception(str(error))
def process_alert(s, repoId):
html = md.markdown(s)
soup = bs(html, 'html.parser')
headers = soup.find_all('details')
libraries, vuln, cvss, fix = [None] * 4
for header in headers:
if 'Vulnerable Librar' in header.text:
libraries = header
elif 'Vulnerability Details' in header.text:
vuln = header
elif 'CVSS' in header.text:
cvss = header
elif 'Suggested Fix' in header.text:
fix = header
assert libraries and vuln and cvss
vulnId = process_vuln(vuln, cvss)
libraryIds = process_libraries(libraries)
for id, eco in libraryIds:
dependencyId = common.getDependencyId(repoId, id, toolId, True)
if eco == 'maven':
try:
insertQ = 'insert into mavenAlert values(%s,%s,%s,%s,%s,%s,%s,%s)'
sql.execute(
insertQ,
(None, None, dependencyId, vulnId, toolId, None, None, 1))
except:
continue
elif eco == 'npm':
try:
insertQ = 'insert into npmAlert values(%s,%s,%s,%s,%s,%s,%s,%s,%s)'
sql.execute(insertQ, (None, None, dependencyId, vulnId, None,
toolId, None, None, 1))
except:
continue
def processReport(repoName, repoId):
filename = repoName + '-vulas.json'
with open(filename, 'r') as file:
data = json.loads(file.read())['vulasReport']
if 'vulnerabilities' not in data.keys() or len(
data['vulnerabilities']) == 0:
return
scandate = data['generatedAt']
vulnerabilities = data['vulnerabilities']
for vuln in vulnerabilities:
#get package id
package = vuln['filename']
if not package.endswith('.jar') or 'jar' in package[:-4]:
#second condition ensures single package
raise Exception('package not jar', package)
package = package[:-4]
q = '''select id
from package
where concat(artifact,'-',version) = %s;'''
packageId = sql.execute(q, (package, ))[0]['id']
dependencyId = common.getDependencyId(repoId, packageId)
steadyId = True
#get cve id
if vuln['bug']['id'].startswith(
'CVE') and not 'CVE' in vuln['bug']['id'][3:]:
#second condition ensures single CVE
cve = vuln['bug']['id']
if len(cve.split('-')) > 3:
#some in vulas has extra addendums
print("check", cve)
cve = '-'.join(x for x in cve.split('-')[:3])
vulnerabilityId = common.getVulnerabilityId(cve, None)
if vulnerabilityId > 0:
steadyId = False
if steadyId:
vulnerabilityId = addSteadyVulnerability(vuln)
integrationTest = []
#DONE: write in a way so that first it can be run for only unit test
#then again for integration test as well
#check if already alert is pushed for the first three
alertId = getAlertId(scandate, dependencyId, vulnerabilityId)
vulnerableVersion = []
staticAnalysis = []
unitTest = []
for module in vuln['modules']:
#get the static and dynamic analysis results into a separate table
vulnerableVersion.append(hm[module['containsVulnerableCode']])
staticAnalysis.append(
hm[module['potentiallyExecutesVulnerableCode']])
unitTest.append(hm[module['actuallyExecutesVulnerableCode']])
vv = max(vulnerableVersion)
sa = max(staticAnalysis)
ut = max(unitTest)
q = 'insert into steady values(%s,%s,%s,%s,%s)'
try:
sql.execute(
q, (alertId, inv_hm[vv], inv_hm[sa], inv_hm[ut], None))
except sql.pymysql.IntegrityError as error:
if error.args[0] == sql.PYMYSQL_DUPLICATE_ERROR:
#TODO update scandate
print('maven alert exists already in steady db')
else:
raise Exception(str(error))
