def load(self):
"""
Grabs all of the vulnerabilities from the specified files, parses them,
cleans them up, and saves them. Returns the total number of loaded
vulnerabilities.
"""
count = 0
with transaction.atomic():
f = files.Files(self.settings)
for (source, cve, bug_id) in f.get_vulnerabilities():
b = helpers.get_row(Bug, id=bug_id)
if b is None:
b = Bug(id=bug_id, type='Bug-Security', status='Redacted')
b.save()
v = helpers.get_row(Vulnerability, id=cve)
if v is None: # pragma: no cover
v = Vulnerability(id=cve, source=source)
v.save()
#print("VULNERABILITY: " + str(v.id))
vb = VulnerabilityBug(vulnerability=v,
bug=b) # pragma: no cover
vb.save()
count += 1
return count
def aggregate(oqueue, cqueue, num_doers):
count, done = 0, 0
while True:
item = cqueue.get()
if item == parallel.DD:
done += 1
if done == num_doers:
break # All doers are done
continue # pragma: no cover
(review, bug_ids) = item
try:
with transaction.atomic():
review.save()
review_bugs = list()
for id in bug_ids:
bug = helpers.get_row(Bug, id=id)
if bug is not None:
review_bugs.append(ReviewBug(review=review, bug=bug))
if review_bugs:
ReviewBug.objects.bulk_create(review_bugs)
count += 1
except Error as err: # pragma: no cover
sys.stderr.write('Exception\n')
sys.stderr.write(' Review {}\n'.format(review.id))
extype, exvalue, extrace = sys.exc_info()
traceback.print_exception(extype, exvalue, extrace)
oqueue.put(count)
def handle(self, *args, **options):
"""
"""
id = options['id']
clean = options.get('clean', False)
try:
review = helpers.get_row(Review, id=id)
if review is None:
raise CommandError('No review identified by {}'.format(id))
review = review.document
print('#' * 72)
print('# Code Review {}'.format(id))
print('# Description\n')
print(format(review['description']))
print('#' * 72)
print('# Conversation\n')
messages = self.get_messages(review, clean=clean)
for (i, (sender, message)) in enumerate(messages):
print('-' * 72)
print('[{}/{}] From {}'.format((i + 1), len(messages), sender))
print('-' * 72)
print(message)
print('#' * 72)
except KeyboardInterrupt:
warning('Attempting to abort.')
def aggregate(oqueue, cqueue, num_doers):
count, done = 0, 0
while True:
item = cqueue.get()
if item == parallel.DD:
done += 1
if done == num_doers:
break # All doers are done
continue # pragma: no cover
(bug, cves) = item
try:
with transaction.atomic():
bug.save()
objects = list()
for cve in cves:
vulnerability = helpers.get_row(Vulnerability, id=cve)
if vulnerability is None:
vulnerability = Vulnerability(id=cve)
vulnerability.save()
objects.append(
VulnerabilityBug(vulnerability=vulnerability, bug=bug))
if objects:
VulnerabilityBug.objects.bulk_create(objects)
count += 1
except Error as err: # pragma: no cover
sys.stderr.write('Exception\n')
sys.stderr.write(' Bug {}\n'.format(bug.id))
extype, exvalue, extrace = sys.exc_info()
traceback.print_exception(extype, exvalue, extrace)
oqueue.put(count)
def stream(review_ids, settings, iqueue, num_doers):
for review_id in review_ids:
review = helpers.get_row(Review, id=review_id)
messages = Message.objects.filter(review_id=review_id) \
.values_list('posted', 'sender', 'text', 'id')
iqueue.put((review_id, list(messages)))
for i in range(num_doers):
iqueue.put(parallel.EOI)
def stream(review_ids, settings, iqueue, num_doers):
for review_id in review_ids:
review = helpers.get_row(Review, id=review_id)
comments = Comment.objects.filter(patch__patchset__review_id=review_id) \
.values_list('posted', 'author', 'text', 'id')
iqueue.put((review_id, list(comments)))
for i in range(num_doers):
iqueue.put(parallel.EOI)
def stream(review_ids, settings, iqueue, num_doers):
for review_id in review_ids:
review = helpers.get_row(Review, id=review_id)
if review is not None:
author = review.document['owner_email']
patchsets = list()
for (psid, ps) in review.document['patchsets'].items():
patchsets.append((author, psid, ps))
iqueue.put((review, patchsets))
for i in range(num_doers):
iqueue.put(parallel.EOI)
def stream(review_ids, settings, iqueue, num_doers):
for review_id in review_ids:
review = helpers.get_row(Review, id=review_id)
messages = list()
for message in review.document['messages']:
(posted, sender, auto) = (message['date'], message['sender'],
message['auto_generated'])
if auto or sender in settings.BOTS:
continue
text = _strip_comments(message['text'])
messages.append((posted, sender, text))
iqueue.put((review_id, messages))
for i in range(num_doers):
iqueue.put(parallel.EOI)
