class Troll(object):
def __init__(self, url, args):
self.url = url
self.args = args
self.gerrit = Gerrit(url)
self.tag = 'autogenerated:review-o-matic'
self.blacklist = {}
self.stats = {
str(ReviewType.SUCCESS): 0,
str(ReviewType.BACKPORT): 0,
str(ReviewType.ALTERED_UPSTREAM): 0,
str(ReviewType.MISSING_FIELDS): 0,
str(ReviewType.MISSING_HASH): 0,
str(ReviewType.INVALID_HASH): 0,
str(ReviewType.MISSING_AM): 0,
str(ReviewType.INCORRECT_PREFIX): 0,
str(ReviewType.FIXES_REF): 0
}
def inc_stat(self, review_type):
if self.args.dry_run:
return
key = str(review_type)
if not self.stats.get(key):
self.stats[key] = 1
else:
self.stats[key] += 1
def do_review(self, change, review):
print('Review for change: {}'.format(change.url()))
print(' Issues: {}, Feedback: {}, Vote:{}, Notify:{}'.format(
review.issues.keys(), review.feedback.keys(), review.vote,
review.notify))
if review.dry_run:
print(review.generate_review_message())
print('------')
return
for i in review.issues:
self.inc_stat(i)
for f in review.feedback:
self.inc_stat(f)
self.gerrit.review(change,
self.tag,
review.generate_review_message(),
review.notify,
vote_code_review=review.vote)
def get_changes(self, prefix):
message = '{}:'.format(prefix)
after = datetime.date.today() - datetime.timedelta(days=5)
changes = self.gerrit.query_changes(
status='open',
message=message,
after=after,
project='chromiumos/third_party/kernel')
return changes
def add_change_to_blacklist(self, change):
self.blacklist[change.number] = change.current_revision.number
def is_change_in_blacklist(self, change):
return self.blacklist.get(
change.number) == change.current_revision.number
def process_changes(self, changes):
rev = Reviewer(git_dir=self.args.git_dir,
verbose=self.args.verbose,
chatty=self.args.chatty)
ret = 0
for c in changes:
if self.args.verbose:
print('Processing change {}'.format(c.url()))
# Blacklist if we've already reviewed this revision
for m in c.messages:
if m.tag == self.tag and m.revision_num == c.current_revision.number:
self.add_change_to_blacklist(c)
# Find a reviewer and blacklist if not found
reviewer = None
if FromlistChangeReviewer.can_review_change(c):
reviewer = FromlistChangeReviewer(rev, c, self.args.dry_run)
elif FromgitChangeReviewer.can_review_change(c):
reviewer = FromgitChangeReviewer(rev, c, self.args.dry_run)
elif UpstreamChangeReviewer.can_review_change(c):
reviewer = UpstreamChangeReviewer(rev, c, self.args.dry_run)
if not reviewer:
self.add_change_to_blacklist(c)
continue
force_review = self.args.force_cl or self.args.force_all
if not force_review and self.is_change_in_blacklist(c):
continue
result = reviewer.review_patch()
if result:
self.do_review(c, result)
ret += 1
self.add_change_to_blacklist(c)
return ret
def update_stats(self):
if not self.args.dry_run and self.args.stats_file:
with open(self.args.stats_file, 'wt') as f:
json.dump(self.stats, f)
print('--')
summary = ' Summary: '
total = 0
for k, v in self.stats.items():
summary += '{}={} '.format(k, v)
total += v
summary += 'total={}'.format(total)
print(summary)
print('')
def run(self):
if self.args.force_cl:
c = self.gerrit.get_change(self.args.force_cl)
print('Force reviewing change {}'.format(c))
self.process_changes([c])
return
if self.args.stats_file:
try:
with open(self.args.stats_file, 'rt') as f:
self.stats = json.load(f)
except FileNotFoundError:
self.update_stats()
prefixes = ['UPSTREAM', 'BACKPORT', 'FROMGIT', 'FROMLIST']
while True:
try:
did_review = 0
for p in prefixes:
changes = self.get_changes(p)
if self.args.verbose:
print('{} changes for prefix {}'.format(
len(changes), p))
did_review += self.process_changes(changes)
if did_review > 0:
self.update_stats()
if not self.args.daemon:
break
if self.args.verbose:
print('Finished! Going to sleep until next run')
except (requests.exceptions.HTTPError, OSError) as e:
sys.stderr.write('Error getting changes: ({})\n'.format(
str(e)))
time.sleep(60)
time.sleep(120)
class Troll(object):
STRING_HEADER = '''
-- Automated message --
'''
STRING_SUCCESS = '''
This change does not differ from its upstream source. It is certified {}
by review-o-matic!
'''
STRING_INCORRECT_PREFIX = '''
This change has a BACKPORT prefix, however it does not differ from its upstream
source. The BACKPORT prefix should be primarily used for patches which were
altered during the cherry-pick (due to conflicts or downstream inconsistencies).
Consider changing your subject prefix to UPSTREAM (or FROMGIT/FROMLIST as
appropriate) to better reflect the contents of this patch.
'''
STRING_MISSING_FIELDS = '''
Your commit message is missing the following required field(s):
{}
'''
STRING_MISSING_FIELDS_SUCCESS = '''
Don't worry, there is good news! Your patch does not differ from its upstream
source. Once the missing fields are present, it will be certified {} (or some
other similarly official-sounding certification) by review-o-matic!
'''
STRING_MISSING_FIELDS_DIFF = '''
In addition to the missing fields, this patch differs from its upstream source.
This may be expected, this message is posted to make reviewing backports easier.
'''
STRING_MISSING_HASH_HEADER = '''
Your commit message is missing the upstream commit hash. It should be in the
form:
'''
STRING_MISSING_HASH_FMT_FROMGIT = '''
(cherry picked from commit <commit SHA>
<remote git url> <remote git tree>)
'''
STRING_MISSING_HASH_FMT_UPSTREAM = '''
(cherry picked from commit <commit SHA>)
'''
STRING_MISSING_HASH_FOOTER = '''
Hint: Use the '-x' argument of git cherry-pick to add this automagically
'''
STRING_MISSING_AM = '''
Your commit message is missing the patchwork URL. It should be in the
form:
(am from https://patchwork.kernel.org/.../)
'''
STRING_UNSUCCESSFUL_HEADER = '''
This patch differs from the source commit.
'''
STRING_UPSTREAM_DIFF = '''
Since this is an UPSTREAM labeled patch, it shouldn't. Either this reviewing
script is incorrect (totally possible, pls send patches!), or something changed
when this was backported. If the backport required changes, please consider
using the BACKPORT label with a description of you downstream changes in your
commit message.
'''
STRING_BACKPORT_DIFF = '''
This is expected, and this message is posted to make reviewing backports easier.
'''
STRING_FROMGIT_DIFF = '''
This may be expected, this message is posted to make reviewing backports easier.
'''
STRING_UNSUCCESSFUL_FOOTER = '''
Below is a diff of the upstream patch referenced in this commit message, vs this
patch.
'''
STRING_FOUND_FIXES_REF = '''
!! NOTE: This patch has been referenced in the Fixes: tag of another commit. If
!! you haven't already, consider backporting the following patch:
!! {}
'''
STRING_FOOTER = '''
---
To learn more about backporting kernel patches to Chromium OS, check out:
https://chromium.googlesource.com/chromiumos/docs/+/master/kernel_faq.md#UPSTREAM_BACKPORT_FROMLIST_and-you
If you're curious about how this message was generated, head over to:
https://github.com/atseanpaul/review-o-matic
This link is not useful:
https://thats.poorly.run/
'''
SWAG = [
'Frrrresh', 'Crisper Than Cabbage', 'Awesome', 'Ahhhmazing',
'Cool As A Cucumber', 'Most Excellent', 'Eximious', 'Prestantious',
'Supernacular', 'Bodacious', 'Blue Chip', 'Blue Ribbon', 'Cracking',
'Dandy', 'Dynamite', 'Fab', 'Fabulous', 'Fantabulous',
'Scrumtrulescent', 'First Class', 'First Rate', 'First String',
'Five Star', 'Gangbusters', 'Grand', 'Groovy', 'HYPE', 'Jim-Dandy',
'Snazzy', 'Marvelous', 'Nifty', 'Par Excellence', 'Peachy Keen',
'PHAT', 'Prime', 'Prizewinning', 'Quality', 'Radical', 'Righteous',
'Sensational', 'Slick', 'Splendid', 'Lovely', 'Stellar', 'Sterling',
'Superb', 'Superior', 'Superlative', 'Supernal', 'Swell', 'Terrific',
'Tip-Top', 'Top Notch', 'Top Shelf', 'Unsurpassed', 'Wonderful'
]
def __init__(self, url, args):
self.url = url
self.args = args
self.gerrit = Gerrit(url)
self.tag = 'autogenerated:review-o-matic'
self.blacklist = []
self.stats = {
ReviewType.SUCCESS: 0,
ReviewType.BACKPORT: 0,
ReviewType.ALTERED_UPSTREAM: 0,
ReviewType.MISSING_FIELDS: 0,
ReviewType.MISSING_HASH: 0,
ReviewType.INCORRECT_PREFIX: 0,
ReviewType.FIXES_REF: 0
}
def do_review(self, review_type, change, fixes_ref, msg, notify, vote):
final_msg = self.STRING_HEADER
if fixes_ref:
print('Adding fixes ref for change {}'.format(change.url()))
self.stats[ReviewType.FIXES_REF] += 1
final_msg += self.STRING_FOUND_FIXES_REF.format(fixes_ref)
final_msg += msg
final_msg += self.STRING_FOOTER
self.stats[review_type] += 1
if not self.args.dry_run:
self.gerrit.review(change,
self.tag,
final_msg,
notify,
vote_code_review=vote)
else:
print('Review for change: {}'.format(change.url()))
print(' Type:{}, Vote:{}, Notify:{}'.format(
review_type, vote, notify))
print(final_msg)
print('------')
def handle_successful_review(self, change, prefix, fixes_ref):
# TODO: We should tag FROMLIST: BACKPORT: patches as incorrect, if needed
if prefix == 'BACKPORT':
print('Adding incorrect prefix review for change {}'.format(
change.url()))
msg = self.STRING_INCORRECT_PREFIX
self.do_review(ReviewType.INCORRECT_PREFIX, change, fixes_ref, msg,
True, 0)
else:
print('Adding successful review for change {}'.format(
change.url()))
msg = self.STRING_SUCCESS.format(random.choice(self.SWAG))
self.do_review(ReviewType.SUCCESS, change, fixes_ref, msg, True, 1)
def handle_missing_fields_review(self, change, fields, result, fixes_ref):
print('Adding missing fields review for change {}'.format(
change.url()))
missing = []
if not fields['bug']:
missing.append('BUG=')
if not fields['test']:
missing.append('TEST=')
if not fields['sob']:
cur_rev = change.current_revision
missing.append('Signed-off-by: {} <{}>'.format(
cur_rev.uploader_name, cur_rev.uploader_email))
msg = self.STRING_MISSING_FIELDS.format(', '.join(missing))
if len(result) == 0:
msg += self.STRING_MISSING_FIELDS_SUCCESS.format(
random.choice(self.SWAG))
else:
msg += self.STRING_MISSING_FIELDS_DIFF
msg += self.STRING_UNSUCCESSFUL_FOOTER
for l in result:
msg += '{}\n'.format(l)
self.do_review(ReviewType.MISSING_FIELDS, change, fixes_ref, msg, True,
-1)
def handle_missing_hash_review(self, change, prefix):
print('Adding missing hash review for change {}'.format(change.url()))
msg = self.STRING_MISSING_HASH_HEADER
if prefix == 'FROMGIT':
msg += self.STRING_MISSING_HASH_FMT_FROMGIT
else:
msg += self.STRING_MISSING_HASH_FMT_UPSTREAM
msg += self.STRING_MISSING_HASH_FOOTER
self.do_review(ReviewType.MISSING_HASH, change, None, msg, True, -1)
def handle_missing_am_review(self, change, prefix):
print('Adding missing am URL for change {}'.format(change.url()))
self.do_review(ReviewType.MISSING_HASH, change, None,
self.STRING_MISSING_AM, True, -1)
def handle_unsuccessful_review(self, change, prefix, result, fixes_ref):
vote = 0
notify = False
review_type = ReviewType.BACKPORT
msg = self.STRING_UNSUCCESSFUL_HEADER
if prefix == 'UPSTREAM':
review_type = ReviewType.ALTERED_UPSTREAM
vote = -1
notify = True
msg += self.STRING_UPSTREAM_DIFF
elif prefix == 'BACKPORT':
msg += self.STRING_BACKPORT_DIFF
elif prefix == 'FROMGIT' or prefix == 'FROMLIST':
msg += self.STRING_FROMGIT_DIFF
msg += self.STRING_UNSUCCESSFUL_FOOTER
for l in result:
msg += '{}\n'.format(l)
print('Adding unsuccessful review (vote={}) for change {}'.format(
vote, change.url()))
self.do_review(review_type, change, fixes_ref, msg, notify, vote)
def get_changes(self, prefix):
message = '{}:'.format(prefix)
after = datetime.date.today() - datetime.timedelta(days=5)
changes = self.gerrit.query_changes(
status='open',
message=message,
after=after,
project='chromiumos/third_party/kernel')
return changes
def print_error(self, error):
if self.args.verbose:
sys.stderr.write('\n')
sys.stderr.write(error)
def process_changes(self, prefix, changes):
rev = Reviewer(git_dir=self.args.git_dir,
verbose=self.args.verbose,
chatty=self.args.chatty)
num_changes = len(changes)
cur_change = 1
line_feed = False
ret = False
for c in changes:
cur_rev = c.current_revision
if self.args.chatty:
print('Processing change {}'.format(c.url()))
elif self.args.verbose:
sys.stdout.write('{}Processing change {}/{}'.format(
'\r' if line_feed else '', cur_change, num_changes))
cur_change += 1
line_feed = True
if c in self.blacklist:
continue
if not c.subject.startswith(prefix):
continue
skip = False
for m in c.messages:
if m.tag == self.tag and m.revision_num == cur_rev.number:
skip = True
if skip and not self.args.force_cl:
continue
ret = True
line_feed = False
if self.args.verbose:
print('')
gerrit_patch = rev.get_commit_from_remote('cros', cur_rev.ref)
if prefix == 'FROMLIST':
upstream_patchworks = rev.get_am_from_from_patch(gerrit_patch)
if not upstream_patchworks:
self.handle_missing_am_review(c, prefix)
continue
upstream_patch = None
for u in reversed(upstream_patchworks):
try:
upstream_patch = rev.get_commit_from_patchwork(u)
break
except:
continue
if not upstream_patch:
self.print_error(
'ERROR: patch missing from patchwork, or patchwork host '
'not whitelisted for {} ({})\n'.format(
c, upstream_patchworks))
self.blacklist.append(c)
continue
else:
upstream_shas = rev.get_cherry_pick_shas_from_patch(
gerrit_patch)
if not upstream_shas:
self.handle_missing_hash_review(c, prefix)
continue
upstream_patch = None
upstream_sha = None
for s in reversed(upstream_shas):
try:
upstream_patch = rev.get_commit_from_sha(s)
upstream_sha = s
break
except:
continue
if not upstream_patch:
self.print_error(
'ERROR: SHA missing from git for {} ({})\n'.format(
c, upstream_shas))
self.blacklist.append(c)
continue
if prefix != 'FROMLIST':
fixes_ref = rev.find_fixes_reference(upstream_sha)
else:
fixes_ref = None
result = rev.compare_diffs(upstream_patch, gerrit_patch)
fields = {'sob': False, 'bug': False, 'test': False}
sob_re = re.compile('Signed-off-by:\s+{}'.format(
cur_rev.uploader_name))
for l in cur_rev.commit_message.splitlines():
if l.startswith('BUG='):
fields['bug'] = True
continue
if l.startswith('TEST='):
fields['test'] = True
continue
if sob_re.match(l):
fields['sob'] = True
continue
if not fields['bug'] or not fields['test'] or not fields['sob']:
self.handle_missing_fields_review(c, fields, result, fixes_ref)
continue
if len(result) == 0:
self.handle_successful_review(c, prefix, fixes_ref)
continue
self.handle_unsuccessful_review(c, prefix, result, fixes_ref)
if self.args.verbose:
print('')
return ret
def run(self):
if self.args.force_cl != None:
c = self.gerrit.get_change(self.args.force_cl)
prefix = c.subject.split(':')[0]
print('Force reviewing change {}'.format(c))
self.process_changes(prefix, [c])
return
while True:
try:
prefixes = ['UPSTREAM', 'BACKPORT', 'FROMGIT']
did_review = False
for p in prefixes:
changes = self.get_changes(p)
if self.args.verbose:
print('{} changes for prefix {}'.format(
len(changes), p))
did_review |= self.process_changes(p, changes)
if did_review:
print('--')
summary = ' Summary: '
for k, v in self.stats.items():
summary += '{}={} '.format(k, v)
print(summary)
print('')
if not self.args.daemon:
break
if self.args.verbose:
print('Finished! Going to sleep until next run')
except requests.exceptions.HTTPError as e:
self.print_error('HTTPError ({})\n'.format(
e.response.status_code))
time.sleep(60)
time.sleep(120)
class Troll(object):
RETRY_REVIEW_KEY='retry-bot-review'
def __init__(self, config):
self.config = config
self.gerrit = Gerrit(config.gerrit_url, netrc=config.netrc)
self.gerrit_admin = Gerrit(config.gerrit_url, netrc=config.netrc_admin)
self.tag = 'autogenerated:review-o-matic'
self.ignore_list = {}
self.stats = TrollStats('{}'.format(self.config.stats_file))
def do_review(self, project, change, review):
logger.info('Review for change: {}'.format(change.url()))
logger.info(' Issues: {}, Feedback: {}, Vote:{}, Notify:{}'.format(
review.issues.keys(), review.feedback.keys(), review.vote,
review.notify))
if review.dry_run:
print(review.generate_review_message(self.RETRY_REVIEW_KEY))
if review.inline_comments:
print('')
print('-- Inline comments:')
for f,comments in review.inline_comments.items():
for c in comments:
print('{}:{}'.format(f, c['line']))
print(c['message'])
print('------')
return
self.stats.update_for_review(project, review)
self.gerrit.review(change, self.tag,
review.generate_review_message(self.RETRY_REVIEW_KEY),
review.notify, vote_code_review=review.vote,
inline_comments=review.inline_comments)
if self.config.results_file:
with open(self.config.results_file, 'a+') as f:
f.write('{}: Issues: {}, Feedback: {}, Vote:{}, Notify:{}\n'.format(
change.url(), review.issues.keys(), review.feedback.keys(),
review.vote, review.notify))
def get_changes(self, project, prefix):
message = '{}:'.format(prefix)
after = datetime.date.today() - datetime.timedelta(days=5)
changes = self.gerrit.query_changes(status='open', message=message,
after=after, project=project.gerrit_project,
branches=project.monitor_branches)
return changes
def add_change_to_ignore_list(self, change):
self.ignore_list[change.number] = change.current_revision.number
def is_change_in_ignore_list(self, change):
return self.ignore_list.get(change.number) == change.current_revision.number
def process_change(self, project, rev, c):
if self.config.chatty:
logger.debug('Processing change {}'.format(c.url()))
force_review = self.config.force_cl or self.config.force_all
# Look for a retry request in the topic
retry_request = False
topic_list = c.topic.split() if c.topic else None
if topic_list and self.RETRY_REVIEW_KEY in topic_list:
retry_request = True
force_review = True
logger.error('Received retry request on change {} (topic={})'.format(
c.url(), c.topic))
# Look for prior reviews and retry requests
last_review = None
for m in c.get_messages():
if not m.revision_num == c.current_revision.number:
continue
if m.tag == self.tag:
last_review = m
age_days = None
if not force_review and last_review:
age_days = (datetime.datetime.utcnow() - last_review.date).days
if age_days != None and self.config.chatty:
logger.debug(' Reviewed {} days ago'.format(age_days))
# Find a reviewer or ignore if not found
reviewer = None
if not ChangeReviewer.can_review_change(project, c, age_days):
# Some patches are blanket unreviewable, check these first
reviewer = None
elif FromlistChangeReviewer.can_review_change(project, c, age_days):
reviewer = FromlistChangeReviewer(project, rev, c,
self.config.gerrit_msg_limit,
self.config.dry_run)
elif FromgitChangeReviewer.can_review_change(project, c, age_days):
reviewer = FromgitChangeReviewer(project, rev, c,
self.config.gerrit_msg_limit,
self.config.dry_run, age_days)
elif UpstreamChangeReviewer.can_review_change(project, c, age_days):
reviewer = UpstreamChangeReviewer(project, rev, c,
self.config.gerrit_msg_limit,
self.config.dry_run)
elif ChromiumChangeReviewer.can_review_change(project, c, age_days):
reviewer = ChromiumChangeReviewer(project, rev, c,
self.config.gerrit_msg_limit,
self.config.dry_run,
self.config.verbose)
# Clear the retry request from the topic
if retry_request:
topic_list.remove(self.RETRY_REVIEW_KEY)
c.topic = ' '.join(topic_list)
if not self.gerrit_admin.set_topic(c):
logger.error('ERROR: Failed to clear retry request from change')
return None
if not reviewer:
self.add_change_to_ignore_list(c)
return None
if not force_review and self.is_change_in_ignore_list(c):
return None
return reviewer.review_patch()
def process_changes(self, project, changes):
rev = Reviewer(git_dir=project.local_repo, verbose=self.config.verbose,
chatty=self.config.chatty)
ret = 0
for c in changes:
ignore = False
for b in project.ignore_branches:
if re.match(b, c.branch):
ignore = True
break
if ignore:
if self.config.chatty:
logger.debug('Ignoring change {}'.format(c))
self.add_change_to_ignore_list(c)
continue
try:
result = self.process_change(project, rev, c)
if result:
self.do_review(project, c, result)
ret += 1
self.add_change_to_ignore_list(c)
except GerritFetchError as e:
logger.error('Gerrit fetch failed, will retry, {}'.format(c.url()))
logger.exception('Exception: {}'.format(e))
# Don't add change to ignore list, we want to retry next time
except Exception as e:
logger.error('Exception processing change {}'.format(c.url()))
logger.exception('Exception: {}'.format(e))
self.add_change_to_ignore_list(c)
return ret
def run(self):
if self.config.force_cl:
c = self.gerrit.get_change(self.config.force_cl, self.config.force_rev)
logger.info('Force reviewing change {}'.format(c))
project = self.config.get_project(c.project)
if not project:
raise ValueError('Could not find project!')
self.process_changes(project, [c])
return
while True:
try:
did_review = 0
for project in self.config.projects.values():
if (self.config.force_project and
project.name != self.config.force_project):
continue
if self.config.chatty:
logger.debug('Running for project {}'.format(project.name))
for p in project.prefixes:
changes = self.get_changes(project, p)
if self.config.chatty:
logger.debug('{} changes for prefix {}'.format(len(changes), p))
did_review += self.process_changes(project, changes)
if did_review > 0:
self.stats.summarize(logging.INFO)
if not self.config.dry_run:
self.stats.save()
if not self.config.daemon:
return
if self.config.chatty:
logger.debug('Finished! Going to sleep until next run')
except (requests.exceptions.HTTPError, OSError) as e:
logger.error('Error getting changes: ({})'.format(str(e)))
logger.exception('Exception getting changes: {}'.format(e))
time.sleep(60)
time.sleep(120)