import sys, os
sys.path.append('..')
import sql, common
import csv
import pandas as pd
from bs4 import BeautifulSoup
import numpy as np
from datetime import datetime
toolId= common.getToolId('Maven Security Versions')
def getVulns(repoId, table) -> dict:
def getCVEids(cves, packageId):
ids=[]
for cve in cves:
if not cve.startswith('CVE'):
raise Exception('non cve vulnerability in victims report', cve)
ids.append(common.getVulnerabilityId(cve,None))
return ids
rows=table.find_all('tr')
d={}
cur=None
for row in rows:
if row.find_all('th'):
#new module found
cur=row.getText().replace('\n','').replace(' ','')
d[cur]={}
else:
import sys, os
sys.path.append('..')
import common, sql
import csv
import pandas as pd
import numpy as np
from datetime import datetime
from dateutil import parser as dt
import json
toolId = common.getToolId('NPM Audit')
def getNPMVulnerability(data):
sourceId = 'NPM-' + str(data['id'])
vulnId = common.getVulnerabilityId(None, sourceId)
if vulnId > 0:
return vulnId
publishDate = dt.parse(data['created'])
description = data['title'] + data['overview']
insertQ = 'insert into vulnerability values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
try:
sql.execute(insertQ, (None, 'NPM', None, sourceId, publishDate,
description, None, None, None, None))
except sql.pymysql.IntegrityError as error:
if error.args[0] == sql.PYMYSQL_DUPLICATE_ERROR:
print(sourceId, ' already exists')
else:
raise Exception(str(error))
import csv
import subprocess, shlex
import os, sys
sys.path.append('..')
import common, sql
toolId=common.getToolId('Contrast')
os.chdir('/Users/nasifimtiaz/Desktop/contrastReports')
files= subprocess.check_output(shlex.split('ls'), encoding='437').split('\n')[:-1]
for file in files:
with open(file) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
line_count = 0
for row in csv_reader:
if line_count == 0:
print(f'Column names are {", ".join(row)}')
else:
library = row[0]
assert library.endswith('.jar')
library=library[:-len('.jar')]
version=row[2]
print(library,version)
assert version.lower() in library.lower()
library=library[:-len('-'+version)]
q='select id from package where artifact=%s and version=%s'
packageId=sql.execute(q,(library,version))[0]['id']
language=row[1]
import os, sys, json
sys.path.append('..')
import common, sql
import dateutil.parser as dt
from datetime import datetime
toolId = common.getToolId('Snyk')
def addSnykVulenrability(vuln):
id = vuln['id']
#first check if the sourceId was already inserted
selectQ = 'select id from vulnerability where cveId is null and sourceId=%s'
results = sql.execute(selectQ, (id, ))
if not results:
cvssScore = vuln['cvssScore']
severity = vuln['severity']
title = vuln['title']
publishDate = dt.parse(vuln['publicationTime'])
insertQ = 'insert into vulnerability values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
sql.execute(insertQ, (None, 'Snyk', None, id, publishDate, title, None,
None, cvssScore, severity))
results = sql.execute(selectQ, (id, ))
cwes = vuln['identifiers']['CWE']
common.addCWEs(results[0]['id'], cwes)
return results[0]['id']
import os, sys
sys.path.append('..')
import sql, common
import subprocess, shlex
import json
from dateutil import parser as dt
toolId = common.getToolId('SourceClear')
path = '/Users/nasifimtiaz/Downloads/scans'
os.chdir(path)
lines = subprocess.check_output(shlex.split('ls'),
encoding='437').split('\n')[:-1]
failures = [
'openmrs-module-adminui-1.3.0', 'openmrs-module-attachments-2.2.0',
'openmrs-module-coreapps-1.28.0', 'openmrs-module-metadatasharing-1.6.0',
'openmrs-module-reportingcompatibility-2.0.6',
'openmrs-module-uicommons-2.12.0', 'openmrs-owa-sysadmin-1.2'
]
def getSrcClrVulnerability(data):
publishDate = dt.parse(data['disclosureDate'])
description = data['title'] + ' ' + data['overview']
cvssScore = data['cvssScore']
source = 'SourceClear'
selectQ = 'select * from vulnerability where source=%s and description=%s'
results = sql.execute(selectQ, (source, description))
import os, sys
sys.path.append('../..')
from gh_graphql import getDependencyAlerts
import distro_information.prepareDistro as distro
import common, sql
import time, dateutil.parser as dt
from datetime import datetime
toolId = common.getToolId('Github Dependabot')
token = os.environ['github_token']
def addGithubAdvisory(alert):
vuln = alert['securityAdvisory']
ghsa = vuln['ghsaId']
vulnId = common.getVulnerabilityId(None, ghsa)
if vulnId > 0:
return vulnId
description = vuln['description']
publishDate = dt.parse(vuln['publishedAt'])
q = 'insert into vulnerability values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
sql.execute(q, (None, 'GitHub', None, ghsa, publishDate, description, None,
None, None, None))
return common.getVulnerabilityId(None, ghsa)
def getCVE(alert):
identifiers = alert['securityAdvisory']['identifiers']
count = 0
cve = None
for id in identifiers:
import sys, os
sys.path.append('..')
import common, sql
import csv
import pandas as pd
import numpy as np
from datetime import datetime
toolId = common.getToolId('OWASP Dependency-Check')
file = open('owasplog.txt', 'w')
def redesignColumns(df):
keep = [
'ScanDate', 'DependencyName', 'DependencyPath', 'Description',
'Identifiers', 'CPE', 'CVE', 'CWE', 'Vulnerability', 'Source',
'CVSSv2_Severity', 'CVSSv2_Score', 'CVSSv3_BaseSeverity',
'CVSSv3_BaseScore', 'CPE Confidence', 'Evidence Count'
]
df = df[keep]
new_names = [
'scandate', 'dependency', 'dependencyPath', 'description',
'identifier', 'CPE', 'CVE', 'CWE', 'vulnerability', 'source',
'CVSS2_severity', 'CVSS2_score', 'CVSS3_severity', 'CVSS3_score',
'confidence', 'evidenceCount'
]
df.columns = new_names
return df
def parseMavenIdentifier(dependency, identifier):
import os, sys
sys.path.append('..')
import common, sql
import json
toolId=common.getToolId('Seeker')
data=json.loads(open('seeker2-10-0.json','r').read())
def insertPackage(artifact,version):
assert '%' not in version
q='select * from package where artifact = %s'
results=sql.execute(q,(artifact,))
if results:
group= results[0]['group']
eco=results[0]['ecosystem']
else:
group = 'seeker'
eco='seeker'
q='insert into package values(%s,%s,%s,%s,%s)'
sql.execute(q,(None,group,artifact,version,eco))
for component in data:
if not component['Vulnerabilities']:
#no vulnerability present
continue
#get package id
artifact=component['Name']
import os, sys
sys.path.append('..')
import sql, common
import json
import distro_information.prepareDistro as distro
from dateutil import parser as dt
from datetime import datetime
toolId = common.getToolId('Steady')
hm = {
'vulnerableVersion': 1,
'unknown': 2,
'nonVulnerableVersion': 3,
'noLibraryCodeAtAll': 4,
'nonVulnerableLibraryCode': 5,
'vulnerableLibraryCode': 6
}
inv_hm = {v: k for k, v in hm.items()}
def addSteadyVulnerability(vuln):
#process this vuln
sourceId = vuln['bug']['id']
vulnId = common.getVulnerabilityId(None, sourceId)
if vulnId > 0:
return vulnId
insertQ = 'insert into vulnerability values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
sql.execute(
insertQ,
import os, sys
sys.path.append('../..')
from gh_graphql import getDependencyAlerts
import distro_information.prepareDistro as distro
import common, sql
import time, dateutil.parser as dt
from datetime import datetime
toolId = common.getToolId('WhiteSource')
token = os.environ['github_token']
from github import Github
import json
import markdown as md
from bs4 import BeautifulSoup as bs
def getPackageId(library, eco):
q = '''select * from package
where concat(artifact,'-',version) = %s'''
results = sql.execute(q, (library, ))
if not results:
print("not in db", library, eco)
temp = library.split('-')
version = temp[-1]
temp = temp[:-1]
artifact = '-'.join(temp)
return common.getPackageId(eco, artifact, version, eco, True)
else:
return results[0]['id']
def acceptWhiteSourcePR(name):
