def load(self): """ Grabs all of the vulnerabilities from the specified files, parses them, cleans them up, and saves them. Returns the total number of loaded vulnerabilities. """ count = 0 with transaction.atomic(): f = files.Files(self.settings) for (source, cve, bug_id) in f.get_vulnerabilities(): b = helpers.get_row(Bug, id=bug_id) if b is None: b = Bug(id=bug_id, type='Bug-Security', status='Redacted') b.save() v = helpers.get_row(Vulnerability, id=cve) if v is None: # pragma: no cover v = Vulnerability(id=cve, source=source) v.save() #print("VULNERABILITY: " + str(v.id)) vb = VulnerabilityBug(vulnerability=v, bug=b) # pragma: no cover vb.save() count += 1 return count
def stream(iqueue, settings, num_doers): f = files.Files(settings) for year in settings.YEARS: for review in f.get_reviews(year): iqueue.put(f.transform_review(review)) for i in range(num_doers): iqueue.put(parallel.EOI)
def stream(iqueue, settings, num_doers): f = files.Files(settings) for year in settings.YEARS: for bug in f.get_bugs(year): iqueue.put(f.transform_bug(bug)) for i in range(num_doers): iqueue.put(parallel.EOI)
def test_save_reviews(self): data = [{'issue': id} for id in range(100001, 100010)] expected = data with tempfile.TemporaryDirectory() as tempdir: with self.settings(REVIEWS_PATH=os.path.join(tempdir, '{year}')): f = files.Files(settings) f.save_reviews(year=9999, chunk=1, reviews=data, errors=None) path = os.path.join(tempdir, '9999') self.assertTrue(os.path.exists(path)) path = os.path.join(tempdir, '9999', 'reviews.1.json') self.assertTrue(os.path.exists(path)) actual = f.get_reviews(year=9999) self.assertCountEqual(expected, actual)
def test_save_ids(self): data = list(range(100001, 100010)) expected = [str(item) for item in data] with tempfile.TemporaryDirectory() as tempdir: with self.settings(IDS_PATH=tempdir): f = files.Files(settings) f.save_ids(year=9999, ids=data, switch='reviews') path = os.path.join(tempdir, '9999.csv') self.assertTrue(os.path.exists(path)) actual = None with open(path, 'r') as file: reader = csv.reader(file) actual = [row[0] for row in reader] self.assertCountEqual(expected, actual)
def setUp(self): self.files = files.Files(settings)