def load(self):
"""
Grabs all of the vulnerabilities from the specified files, parses them,
cleans them up, and saves them. Returns the total number of loaded
vulnerabilities.
"""
count = 0
with transaction.atomic():
f = files.Files(self.settings)
for (source, cve, bug_id) in f.get_vulnerabilities():
b = helpers.get_row(Bug, id=bug_id)
if b is None:
b = Bug(id=bug_id, type='Bug-Security', status='Redacted')
b.save()
v = helpers.get_row(Vulnerability, id=cve)
if v is None: # pragma: no cover
v = Vulnerability(id=cve, source=source)
v.save()
#print("VULNERABILITY: " + str(v.id))
vb = VulnerabilityBug(vulnerability=v,
bug=b) # pragma: no cover
vb.save()
count += 1
return count
def stream(iqueue, settings, num_doers):
f = files.Files(settings)
for year in settings.YEARS:
for review in f.get_reviews(year):
iqueue.put(f.transform_review(review))
for i in range(num_doers):
iqueue.put(parallel.EOI)
def stream(iqueue, settings, num_doers):
f = files.Files(settings)
for year in settings.YEARS:
for bug in f.get_bugs(year):
iqueue.put(f.transform_bug(bug))
for i in range(num_doers):
iqueue.put(parallel.EOI)
def test_save_reviews(self):
data = [{'issue': id} for id in range(100001, 100010)]
expected = data
with tempfile.TemporaryDirectory() as tempdir:
with self.settings(REVIEWS_PATH=os.path.join(tempdir, '{year}')):
f = files.Files(settings)
f.save_reviews(year=9999, chunk=1, reviews=data, errors=None)
path = os.path.join(tempdir, '9999')
self.assertTrue(os.path.exists(path))
path = os.path.join(tempdir, '9999', 'reviews.1.json')
self.assertTrue(os.path.exists(path))
actual = f.get_reviews(year=9999)
self.assertCountEqual(expected, actual)
def test_save_ids(self):
data = list(range(100001, 100010))
expected = [str(item) for item in data]
with tempfile.TemporaryDirectory() as tempdir:
with self.settings(IDS_PATH=tempdir):
f = files.Files(settings)
f.save_ids(year=9999, ids=data, switch='reviews')
path = os.path.join(tempdir, '9999.csv')
self.assertTrue(os.path.exists(path))
actual = None
with open(path, 'r') as file:
reader = csv.reader(file)
actual = [row[0] for row in reader]
self.assertCountEqual(expected, actual)
def setUp(self):
self.files = files.Files(settings)