def __init__(self, config):
self.config = config
self.gerrit = Gerrit(config.gerrit_url, netrc=config.netrc)
self.gerrit_admin = Gerrit(config.gerrit_url, netrc=config.netrc_admin)
self.tag = 'autogenerated:review-o-matic'
self.ignore_list = {}
self.stats = TrollStats('{}'.format(self.config.stats_file))
def main(self):
"""
Send emails to requested users filed in the users folder (use '-a'
for all users, or '-u user.address' for specific user) -- OR --
create a dump for specified users for debugging purposes.
"""
conf = Config()
if conf.smtp.authentication and not self._gpg_recipient:
logging.error("Cannot send email. SMTP authentication missing.")
return
cache = Cache(conf.cache_filename)
cache.read()
gerrit = Gerrit(cache, conf.gerrit_url)
gerrit.get_cached_today()
for user in self._users:
try:
user.add_content(gerrit.cache, conf.project, conf.gerrit_url)
user.send_email(
conf.project, conf.smtp, self._gpg_recipient, self._dry_run
)
except IOError as err:
logging.error("Could not send email for user %s: %s", user, str(err))
def main():
"""
See module docstring.
"""
conf = Config()
gerrit = Gerrit(Cache(conf.cache_filename), conf.gerrit_url)
gerrit.update()
def main():
logging.info('Launching gerrit-stats')
args = parse_commandline()
settings = load_settings(args)
gerrit = Gerrit(args, settings)
cur = init_db(gerrit.my_cnf)
gerrit.fetch_repos()
start_date = settings.get('creation_date')
yesterday = determine_yesterday()
changesets = {}
logging.info(
'Queries will span timeframe: %s - %s.' % (start_date, yesterday))
logging.info('Queries will always run up to \'yesterday\', so that we always have counts for full days.')
changesets = load_commit_data(cur, changesets)
changesets = load_patch_set_data(cur, changesets)
changesets = load_review_data(cur, changesets)
for changeset in changesets.itervalues():
changeset.is_all_positive_reviews()
changeset.calculate_wait_first_review()
changeset.calculate_wait_plus2()
changeset.is_self_reviewed()
repo = gerrit.repos.get(changeset.dest_project_name)
if repo:
repo.increment(changeset)
else:
logging.info('Repo %s does not exist, ignored repos are: %s' % (changeset.dest_project_name, ','.join(gerrit.ignore_repos)))
logging.info('Successfully parsed changesets data.')
# create datasets that are collections of repositories
create_aggregate_dataset(gerrit)
for repo in gerrit.repos.itervalues():
# if repo.name == 'mediawiki':
# change_ids = list(repo.observations[yesterday.date()].changeset_ids)
# change_ids.sort()
# fh = open('/Users/diederik/Development/gerrit-stats/gerritstats/tests/backlog_gerrit-stats.txt','w')
# for change_id in change_ids:
# print change_id
# fh.write('%s\n' % change_id)
# fh.close()
repo.fill_in_missing_days()
repo.create_headings()
repo.prune_observations()
repo.write_dataset(gerrit)
# save results for future use.
successful_exit()
def __init__(self, url, args):
self.url = url
self.args = args
self.gerrit = Gerrit(url)
self.tag = 'autogenerated:review-o-matic'
self.blacklist = []
self.stats = {
ReviewType.SUCCESS: 0,
ReviewType.BACKPORT: 0,
ReviewType.ALTERED_UPSTREAM: 0,
ReviewType.MISSING_FIELDS: 0,
ReviewType.MISSING_HASH: 0,
ReviewType.INCORRECT_PREFIX: 0,
ReviewType.FIXES_REF: 0
}
def start(self):
self.gerrit = Gerrit(['zuul'])
self.omqtt = OpenstackMqtt()
self.omqtt.on_connect = self.on_connect
self.omqtt.on_message = self.on_message
self.omqtt.connect()
self.omqtt.client.loop_forever()
def add_gerrit_remote(self,
name,
location,
project_name,
fetch=True,
fetch_changes=True):
repo = Gerrit(name, location, project_name)
self.addremote(repo, fetch=fetch)
repo.local_track = TrackedRepo(name, self.directory, project_name)
if fetch_changes:
shell('git fetch %s +refs/changes/*:refs/remotes/%s/changes/*' %
(name, name))
try:
os.stat(".git/hooks/commit-msg")
except OSError:
shell('scp -p %s:hooks/commit-msg .git/hooks/' % location)
def __init__(self, scheduler: JobScheduler, config=Dict[str, str]):
"""
Initialize a Gerrit based event source
Args:
@scheduler: scheduler to which job must be added
@config: dictionary configuring the connection to gerrit. Allowed
keys are 'hostname', 'username', 'port', 'keyfile' and
'clone_url'
"""
# Create connection to gerrit command line
cfg = subdict(config, ['hostname', 'username', 'port', 'keyfile'])
gerrit = Gerrit(**cfg)
# determine settings for git clone when a project must be build
clone_opts = {}
clone_url = config.get('clone_url')
if not clone_url:
clone_url = 'ssh://{}@{}:{:d}'.format(gerrit.username,
gerrit.hostname,
int(gerrit.port))
if gerrit.keyfile:
clone_opts['git_ssh_cmd'] = 'ssh -i ' + gerrit.keyfile
super().__init__(scheduler)
self.gerrit_instance = gerrit
self.clone_url = clone_url
self.clone_opts = clone_opts
def __init__(self, url, args):
self.url = url
self.args = args
self.gerrit = Gerrit(url)
self.tag = 'autogenerated:review-o-matic'
self.blacklist = {}
self.stats = {
str(ReviewType.SUCCESS): 0,
str(ReviewType.BACKPORT): 0,
str(ReviewType.ALTERED_UPSTREAM): 0,
str(ReviewType.MISSING_FIELDS): 0,
str(ReviewType.MISSING_HASH): 0,
str(ReviewType.INVALID_HASH): 0,
str(ReviewType.MISSING_AM): 0,
str(ReviewType.INCORRECT_PREFIX): 0,
str(ReviewType.FIXES_REF): 0
}
def test_add_project(self):
"""
Test that a project can be successfully managed
"""
# Felix wants to add a project, he uses the gerrit module to do this
gerrit = Gerrit(
url=self._url,
auth_type='http',
auth_id=self._admin_username,
auth_pw=self._admin_password,
)
created_project = gerrit.create_project(
self._project,
options={
'description': 'my description',
'branches': ['master'],
'create_empty_commit': True,
},
)
gotten_project = gerrit.get_project(self._project)
# Using the same module he can get his project
self.assertEqual(created_project, gotten_project)
self.assertEqual(gotten_project.description, 'my description')
# Felix uploads a new change
change = gerrit.create_change(created_project, 'My change')
# Wanting Mary to review the change, he adds her as a reviewer
self.assertTrue(change.add_reviewer('mary'))
# He can now see that Mary is a reviewer
reviewers = change.list_reviewers()
self.assertEqual(reviewers, [{
'username': '******',
'approvals': {
'Code-Review': ' 0'
},
'name': 'Mary',
'_account_id': 1000001
}])
# Felix made a mistake, Mary shouldn't be a reviewer.
# He removes her.
self.assertTrue(change.delete_reviewer('mary'))
# He can now see that Mary is no longer a reviewer
reviewers = change.list_reviewers()
self.assertEqual(reviewers, [])
# Happy with the change, Felix reviews and submits it
change.set_review(labels={'Code-Review': '+2'})
change.submit_change()
self.assertEqual(change.status, 'MERGED')
# Not needing the repo anymore Felix removes it
self.assertTrue(created_project.delete({'force': True}))
# Felix can no longer get the project
with self.assertRaises(ValueError):
gerrit.get_project(self._project)
def __init__(self, last_cid, review, verify, ready):
self.vote_review = 2 if review else None
self.vote_verify = 1 if verify else None
self.vote_trybot_ready = 1 if ready else None
self.vote_cq_ready = 1 if ready else None
self.tag = 'autogenerated:submit-o-matic'
self.max_in_flight = 100 # 50 for the cq, 50 for the pre-cq
self.in_flight = []
self.changes = []
self.gerrit = Gerrit('https://chromium-review.googlesource.com')
last_change = self.gerrit.get_change(last_cid)
self.changes = self.gerrit.get_related_changes(last_change)
if not self.changes:
self.changes = [last_change]
else:
self.changes.reverse()
def test_add_project(self):
# Felix wants to add a project, he uses the gerrit module to do this
gerrit = Gerrit(
url=self._url,
auth_type='http',
auth_id=self._admin_username,
auth_pw=self._admin_password,
)
created_project = gerrit.create_project(
self._project,
options={
'description': 'my description',
'branches': ['master'],
'create_empty_commit': True,
},
)
gotten_project = gerrit.get_project(self._project)
# Using the same module he can get his project
self.assertEqual(created_project, gotten_project)
self.assertEqual(gotten_project.description, 'my description')
# Felix uploads a new change
change = gerrit.create_change(created_project, 'My change')
# Wanting Mary to review the change, he adds her as a reviewer
self.assertTrue(change.add_reviewer('mary'))
# He can now see that Mary is a reviewer
reviewers = change.list_reviewers()
self.assertEqual(
reviewers,
[{'username': '******',
'approvals': {'Code-Review': ' 0'},
'name': 'Mary',
'_account_id': 1000001
}]
)
# Felix made a mistake, Mary shouldn't be a reviewer.
# He removes her.
self.assertTrue(change.delete_reviewer('mary'))
# He can now see that Mary is no longer a reviewer
reviewers = change.list_reviewers()
self.assertEqual(reviewers,[])
# Happy with the change, Felix reviews and submits it
change.set_review(labels={'Code-Review': '+2'})
change.submit_change()
self.assertEqual(change.status, 'MERGED')
# Not needing the repo anymore Felix removes it
self.assertTrue(created_project.delete({'force': True}))
# Felix can no longer get the project
with self.assertRaises(ValueError):
gerrit.get_project(self._project)
def __init__(self, last_cid, review, verify, ready, abandon, force_review,
dry_run):
self.abandon = abandon
self.vote_review = 2 if review else None
self.vote_verify = 1 if verify else None
self.vote_cq_ready = ready
self.force_review = force_review
self.dry_run = dry_run
self.tag = 'autogenerated:submit-o-matic'
self.max_in_flight = 100 # 50 for the cq, 50 for the pre-cq
self.in_flight = []
self.changes = []
self.gerrit = Gerrit('https://chromium-review.googlesource.com',
use_internal=False)
last_change = self.gerrit.get_change(last_cid)
ancestor_changes = self.gerrit.get_ancestor_changes(last_change)
for c in reversed(ancestor_changes):
if c.status == 'NEW':
self.changes.append(c)
self.changes.append(last_change)
class ChangesFetcher(object):
def __init__(self, host, port, username, pkey):
self._gerrit = Gerrit(host, port, username, pkey)
self._flow = _IDENTITY_FLOW
def set_flow(self, flow):
self._flow = flow
def get_changes(self, query):
changes = self._gerrit.query(query,
options=[
QueryOptions.Comments,
QueryOptions.CurrentPatchSet,
QueryOptions.CommitMessage
])
return self._flow(changes)
def set_objects(config):
"""
Sets bugzilla and gerrit objects
:param config: dict of configuration keys and values
:return: bugzilla and gerrit objects
"""
# set bugzilla object
bz_obj = Bugzilla(
user=config['BZ_USER'],
passwd=config['BZ_PASS'],
url=config['BZ_URL'],
)
# set gerrit object
gerrit_obj = Gerrit(config['GERRIT_SRV'])
return bz_obj, gerrit_obj
current_path_set != last_patch_set)
if last_patch_set != current_path_set:
change["diff_url"] = "http://%s/#/c/%s/%d..%d//COMMIT_MSG" % (
host, change["number"], last_patch_set, current_path_set
)
else:
change["change_since_last_comment"] = True
change["last_checked_patch_set"] = -1
return change
def fit_width(s, n):
if len(s) > n:
return s[:n-3] + "..."
else:
return s + " " * (n - len(s))
pkey = get_private_key()
g = Gerrit(host, port, username, pkey)
changes = g.query(query,
options=[QueryOptions.Comments,
QueryOptions.CurrentPatchSet])
changes = imap(remove_jenkins_comments, changes)
changes = imap(add_last_checked_information, changes)
changes = ifilter(not_mine, changes)
changes = ifilter(has_changed_since_comment, changes)
sys.stdout.write(str(Template(file=template_file,
searchList=[{"changes": changes,
"fit_width": fit_width}])))
class Submitter(object):
def __init__(self, last_cid, review, verify, ready):
self.vote_review = 2 if review else None
self.vote_verify = 1 if verify else None
self.vote_trybot_ready = 1 if ready else None
self.vote_cq_ready = 1 if ready else None
self.tag = 'autogenerated:submit-o-matic'
self.max_in_flight = 100 # 50 for the cq, 50 for the pre-cq
self.in_flight = []
self.changes = []
self.gerrit = Gerrit('https://chromium-review.googlesource.com')
last_change = self.gerrit.get_change(last_cid)
self.changes = self.gerrit.get_related_changes(last_change)
if not self.changes:
self.changes = [last_change]
else:
self.changes.reverse()
def change_needs_action(self, change):
return change.is_merged() or \
(self.vote_review and not change.is_reviewed()) or \
(self.vote_verify and not change.is_verified()) or \
(self.vote_cq_ready and not change.is_cq_ready())
def num_changes(self):
return len(self.changes)
def num_in_flight(self):
return len(self.in_flight)
def review_changes(self):
for i, c in enumerate(self.changes):
sys.stdout.write('\rRunning reviewer (%d/%d)' %
(i, self.num_changes()))
c = self.gerrit.get_change(c.change_id)
if c.is_merged() or not self.change_needs_action(c):
continue
self.gerrit.review(c, self.tag, '', False, self.vote_review,
self.vote_verify, None, self.vote_trybot_ready)
def submit_changes(self):
self.in_flight = []
merged = 0
for i, c in enumerate(self.changes):
if self.num_in_flight() >= self.max_in_flight:
break
sys.stdout.write('\rRunning submitter (%d/%d)' %
(i, self.num_changes()))
c = self.gerrit.get_change(c.change_id)
if c.is_merged():
merged += 1
continue
if self.change_needs_action(c):
self.gerrit.review(c, self.tag, '', False, self.vote_review,
self.vote_verify, self.vote_cq_ready,
self.vote_trybot_ready)
self.in_flight.append(c)
sys.stdout.write(
'\r%d Changes: \n' %
self.num_changes())
sys.stdout.write('-- %d merged\n' % merged)
sys.stdout.write('-- %d in flight\n' % self.num_in_flight())
def detect_change(self):
if self.num_in_flight(
) == 0: # everything is merged, so no detection needed
return True
c = self.in_flight[0]
sys.stdout.write('\rDetecting: %s' % c.url())
c = self.gerrit.get_change(c.change_id)
if self.change_needs_action(c):
return True
return False
def __init__(self, host, port, username, pkey):
self._gerrit = Gerrit(host, port, username, pkey)
self._flow = _IDENTITY_FLOW
class Troll(object):
RETRY_REVIEW_KEY='retry-bot-review'
def __init__(self, config):
self.config = config
self.gerrit = Gerrit(config.gerrit_url, netrc=config.netrc)
self.gerrit_admin = Gerrit(config.gerrit_url, netrc=config.netrc_admin)
self.tag = 'autogenerated:review-o-matic'
self.ignore_list = {}
self.stats = TrollStats('{}'.format(self.config.stats_file))
def do_review(self, project, change, review):
logger.info('Review for change: {}'.format(change.url()))
logger.info(' Issues: {}, Feedback: {}, Vote:{}, Notify:{}'.format(
review.issues.keys(), review.feedback.keys(), review.vote,
review.notify))
if review.dry_run:
print(review.generate_review_message(self.RETRY_REVIEW_KEY))
if review.inline_comments:
print('')
print('-- Inline comments:')
for f,comments in review.inline_comments.items():
for c in comments:
print('{}:{}'.format(f, c['line']))
print(c['message'])
print('------')
return
self.stats.update_for_review(project, review)
self.gerrit.review(change, self.tag,
review.generate_review_message(self.RETRY_REVIEW_KEY),
review.notify, vote_code_review=review.vote,
inline_comments=review.inline_comments)
if self.config.results_file:
with open(self.config.results_file, 'a+') as f:
f.write('{}: Issues: {}, Feedback: {}, Vote:{}, Notify:{}\n'.format(
change.url(), review.issues.keys(), review.feedback.keys(),
review.vote, review.notify))
def get_changes(self, project, prefix):
message = '{}:'.format(prefix)
after = datetime.date.today() - datetime.timedelta(days=5)
changes = self.gerrit.query_changes(status='open', message=message,
after=after, project=project.gerrit_project,
branches=project.monitor_branches)
return changes
def add_change_to_ignore_list(self, change):
self.ignore_list[change.number] = change.current_revision.number
def is_change_in_ignore_list(self, change):
return self.ignore_list.get(change.number) == change.current_revision.number
def process_change(self, project, rev, c):
if self.config.chatty:
logger.debug('Processing change {}'.format(c.url()))
force_review = self.config.force_cl or self.config.force_all
# Look for a retry request in the topic
retry_request = False
topic_list = c.topic.split() if c.topic else None
if topic_list and self.RETRY_REVIEW_KEY in topic_list:
retry_request = True
force_review = True
logger.error('Received retry request on change {} (topic={})'.format(
c.url(), c.topic))
# Look for prior reviews and retry requests
last_review = None
for m in c.get_messages():
if not m.revision_num == c.current_revision.number:
continue
if m.tag == self.tag:
last_review = m
age_days = None
if not force_review and last_review:
age_days = (datetime.datetime.utcnow() - last_review.date).days
if age_days != None and self.config.chatty:
logger.debug(' Reviewed {} days ago'.format(age_days))
# Find a reviewer or ignore if not found
reviewer = None
if not ChangeReviewer.can_review_change(project, c, age_days):
# Some patches are blanket unreviewable, check these first
reviewer = None
elif FromlistChangeReviewer.can_review_change(project, c, age_days):
reviewer = FromlistChangeReviewer(project, rev, c,
self.config.gerrit_msg_limit,
self.config.dry_run)
elif FromgitChangeReviewer.can_review_change(project, c, age_days):
reviewer = FromgitChangeReviewer(project, rev, c,
self.config.gerrit_msg_limit,
self.config.dry_run, age_days)
elif UpstreamChangeReviewer.can_review_change(project, c, age_days):
reviewer = UpstreamChangeReviewer(project, rev, c,
self.config.gerrit_msg_limit,
self.config.dry_run)
elif ChromiumChangeReviewer.can_review_change(project, c, age_days):
reviewer = ChromiumChangeReviewer(project, rev, c,
self.config.gerrit_msg_limit,
self.config.dry_run,
self.config.verbose)
# Clear the retry request from the topic
if retry_request:
topic_list.remove(self.RETRY_REVIEW_KEY)
c.topic = ' '.join(topic_list)
if not self.gerrit_admin.set_topic(c):
logger.error('ERROR: Failed to clear retry request from change')
return None
if not reviewer:
self.add_change_to_ignore_list(c)
return None
if not force_review and self.is_change_in_ignore_list(c):
return None
return reviewer.review_patch()
def process_changes(self, project, changes):
rev = Reviewer(git_dir=project.local_repo, verbose=self.config.verbose,
chatty=self.config.chatty)
ret = 0
for c in changes:
ignore = False
for b in project.ignore_branches:
if re.match(b, c.branch):
ignore = True
break
if ignore:
if self.config.chatty:
logger.debug('Ignoring change {}'.format(c))
self.add_change_to_ignore_list(c)
continue
try:
result = self.process_change(project, rev, c)
if result:
self.do_review(project, c, result)
ret += 1
self.add_change_to_ignore_list(c)
except GerritFetchError as e:
logger.error('Gerrit fetch failed, will retry, {}'.format(c.url()))
logger.exception('Exception: {}'.format(e))
# Don't add change to ignore list, we want to retry next time
except Exception as e:
logger.error('Exception processing change {}'.format(c.url()))
logger.exception('Exception: {}'.format(e))
self.add_change_to_ignore_list(c)
return ret
def run(self):
if self.config.force_cl:
c = self.gerrit.get_change(self.config.force_cl, self.config.force_rev)
logger.info('Force reviewing change {}'.format(c))
project = self.config.get_project(c.project)
if not project:
raise ValueError('Could not find project!')
self.process_changes(project, [c])
return
while True:
try:
did_review = 0
for project in self.config.projects.values():
if (self.config.force_project and
project.name != self.config.force_project):
continue
if self.config.chatty:
logger.debug('Running for project {}'.format(project.name))
for p in project.prefixes:
changes = self.get_changes(project, p)
if self.config.chatty:
logger.debug('{} changes for prefix {}'.format(len(changes), p))
did_review += self.process_changes(project, changes)
if did_review > 0:
self.stats.summarize(logging.INFO)
if not self.config.dry_run:
self.stats.save()
if not self.config.daemon:
return
if self.config.chatty:
logger.debug('Finished! Going to sleep until next run')
except (requests.exceptions.HTTPError, OSError) as e:
logger.error('Error getting changes: ({})'.format(str(e)))
logger.exception('Exception getting changes: {}'.format(e))
time.sleep(60)
time.sleep(120)
else:
change["change_since_last_comment"] = True
change["last_checked_patch_set"] = -1
return change
def fit_width(s, n):
if len(s) > n:
return s[: n - 3] + "..."
else:
return s + " " * (n - len(s))
pkey = get_private_key()
g = Gerrit(host, port, username, pkey)
changes = g.query(query, options=[QueryOptions.Comments, QueryOptions.CurrentPatchSet, QueryOptions.CommitMessage])
changes = imap(remove_jenkins_comments, changes)
changes = imap(add_last_checked_information, changes)
changes = imap(extract_headers, changes)
changes = imap(does_relate_to_bug, changes)
changes = imap(is_spec, changes)
# changes = ifilter(not_mine, changes)
changes = ifilter(has_changed_since_comment, changes)
sys.stdout.write(
str(
Template(
file=template_path,
searchList=[{"changes": changes, "fit_width": fit_width, "terminal_size": get_terminal_size()}],
)
class Troll(object):
def __init__(self, url, args):
self.url = url
self.args = args
self.gerrit = Gerrit(url)
self.tag = 'autogenerated:review-o-matic'
self.blacklist = {}
self.stats = {
str(ReviewType.SUCCESS): 0,
str(ReviewType.BACKPORT): 0,
str(ReviewType.ALTERED_UPSTREAM): 0,
str(ReviewType.MISSING_FIELDS): 0,
str(ReviewType.MISSING_HASH): 0,
str(ReviewType.INVALID_HASH): 0,
str(ReviewType.MISSING_AM): 0,
str(ReviewType.INCORRECT_PREFIX): 0,
str(ReviewType.FIXES_REF): 0
}
def inc_stat(self, review_type):
if self.args.dry_run:
return
key = str(review_type)
if not self.stats.get(key):
self.stats[key] = 1
else:
self.stats[key] += 1
def do_review(self, change, review):
print('Review for change: {}'.format(change.url()))
print(' Issues: {}, Feedback: {}, Vote:{}, Notify:{}'.format(
review.issues.keys(), review.feedback.keys(), review.vote,
review.notify))
if review.dry_run:
print(review.generate_review_message())
print('------')
return
for i in review.issues:
self.inc_stat(i)
for f in review.feedback:
self.inc_stat(f)
self.gerrit.review(change,
self.tag,
review.generate_review_message(),
review.notify,
vote_code_review=review.vote)
def get_changes(self, prefix):
message = '{}:'.format(prefix)
after = datetime.date.today() - datetime.timedelta(days=5)
changes = self.gerrit.query_changes(
status='open',
message=message,
after=after,
project='chromiumos/third_party/kernel')
return changes
def add_change_to_blacklist(self, change):
self.blacklist[change.number] = change.current_revision.number
def is_change_in_blacklist(self, change):
return self.blacklist.get(
change.number) == change.current_revision.number
def process_changes(self, changes):
rev = Reviewer(git_dir=self.args.git_dir,
verbose=self.args.verbose,
chatty=self.args.chatty)
ret = 0
for c in changes:
if self.args.verbose:
print('Processing change {}'.format(c.url()))
# Blacklist if we've already reviewed this revision
for m in c.messages:
if m.tag == self.tag and m.revision_num == c.current_revision.number:
self.add_change_to_blacklist(c)
# Find a reviewer and blacklist if not found
reviewer = None
if FromlistChangeReviewer.can_review_change(c):
reviewer = FromlistChangeReviewer(rev, c, self.args.dry_run)
elif FromgitChangeReviewer.can_review_change(c):
reviewer = FromgitChangeReviewer(rev, c, self.args.dry_run)
elif UpstreamChangeReviewer.can_review_change(c):
reviewer = UpstreamChangeReviewer(rev, c, self.args.dry_run)
if not reviewer:
self.add_change_to_blacklist(c)
continue
force_review = self.args.force_cl or self.args.force_all
if not force_review and self.is_change_in_blacklist(c):
continue
result = reviewer.review_patch()
if result:
self.do_review(c, result)
ret += 1
self.add_change_to_blacklist(c)
return ret
def update_stats(self):
if not self.args.dry_run and self.args.stats_file:
with open(self.args.stats_file, 'wt') as f:
json.dump(self.stats, f)
print('--')
summary = ' Summary: '
total = 0
for k, v in self.stats.items():
summary += '{}={} '.format(k, v)
total += v
summary += 'total={}'.format(total)
print(summary)
print('')
def run(self):
if self.args.force_cl:
c = self.gerrit.get_change(self.args.force_cl)
print('Force reviewing change {}'.format(c))
self.process_changes([c])
return
if self.args.stats_file:
try:
with open(self.args.stats_file, 'rt') as f:
self.stats = json.load(f)
except FileNotFoundError:
self.update_stats()
prefixes = ['UPSTREAM', 'BACKPORT', 'FROMGIT', 'FROMLIST']
while True:
try:
did_review = 0
for p in prefixes:
changes = self.get_changes(p)
if self.args.verbose:
print('{} changes for prefix {}'.format(
len(changes), p))
did_review += self.process_changes(changes)
if did_review > 0:
self.update_stats()
if not self.args.daemon:
break
if self.args.verbose:
print('Finished! Going to sleep until next run')
except (requests.exceptions.HTTPError, OSError) as e:
sys.stderr.write('Error getting changes: ({})\n'.format(
str(e)))
time.sleep(60)
time.sleep(120)
class Troll(object):
STRING_HEADER = '''
-- Automated message --
'''
STRING_SUCCESS = '''
This change does not differ from its upstream source. It is certified {}
by review-o-matic!
'''
STRING_INCORRECT_PREFIX = '''
This change has a BACKPORT prefix, however it does not differ from its upstream
source. The BACKPORT prefix should be primarily used for patches which were
altered during the cherry-pick (due to conflicts or downstream inconsistencies).
Consider changing your subject prefix to UPSTREAM (or FROMGIT/FROMLIST as
appropriate) to better reflect the contents of this patch.
'''
STRING_MISSING_FIELDS = '''
Your commit message is missing the following required field(s):
{}
'''
STRING_MISSING_FIELDS_SUCCESS = '''
Don't worry, there is good news! Your patch does not differ from its upstream
source. Once the missing fields are present, it will be certified {} (or some
other similarly official-sounding certification) by review-o-matic!
'''
STRING_MISSING_FIELDS_DIFF = '''
In addition to the missing fields, this patch differs from its upstream source.
This may be expected, this message is posted to make reviewing backports easier.
'''
STRING_MISSING_HASH_HEADER = '''
Your commit message is missing the upstream commit hash. It should be in the
form:
'''
STRING_MISSING_HASH_FMT_FROMGIT = '''
(cherry picked from commit <commit SHA>
<remote git url> <remote git tree>)
'''
STRING_MISSING_HASH_FMT_UPSTREAM = '''
(cherry picked from commit <commit SHA>)
'''
STRING_MISSING_HASH_FOOTER = '''
Hint: Use the '-x' argument of git cherry-pick to add this automagically
'''
STRING_MISSING_AM = '''
Your commit message is missing the patchwork URL. It should be in the
form:
(am from https://patchwork.kernel.org/.../)
'''
STRING_UNSUCCESSFUL_HEADER = '''
This patch differs from the source commit.
'''
STRING_UPSTREAM_DIFF = '''
Since this is an UPSTREAM labeled patch, it shouldn't. Either this reviewing
script is incorrect (totally possible, pls send patches!), or something changed
when this was backported. If the backport required changes, please consider
using the BACKPORT label with a description of you downstream changes in your
commit message.
'''
STRING_BACKPORT_DIFF = '''
This is expected, and this message is posted to make reviewing backports easier.
'''
STRING_FROMGIT_DIFF = '''
This may be expected, this message is posted to make reviewing backports easier.
'''
STRING_UNSUCCESSFUL_FOOTER = '''
Below is a diff of the upstream patch referenced in this commit message, vs this
patch.
'''
STRING_FOUND_FIXES_REF = '''
!! NOTE: This patch has been referenced in the Fixes: tag of another commit. If
!! you haven't already, consider backporting the following patch:
!! {}
'''
STRING_FOOTER = '''
---
To learn more about backporting kernel patches to Chromium OS, check out:
https://chromium.googlesource.com/chromiumos/docs/+/master/kernel_faq.md#UPSTREAM_BACKPORT_FROMLIST_and-you
If you're curious about how this message was generated, head over to:
https://github.com/atseanpaul/review-o-matic
This link is not useful:
https://thats.poorly.run/
'''
SWAG = [
'Frrrresh', 'Crisper Than Cabbage', 'Awesome', 'Ahhhmazing',
'Cool As A Cucumber', 'Most Excellent', 'Eximious', 'Prestantious',
'Supernacular', 'Bodacious', 'Blue Chip', 'Blue Ribbon', 'Cracking',
'Dandy', 'Dynamite', 'Fab', 'Fabulous', 'Fantabulous',
'Scrumtrulescent', 'First Class', 'First Rate', 'First String',
'Five Star', 'Gangbusters', 'Grand', 'Groovy', 'HYPE', 'Jim-Dandy',
'Snazzy', 'Marvelous', 'Nifty', 'Par Excellence', 'Peachy Keen',
'PHAT', 'Prime', 'Prizewinning', 'Quality', 'Radical', 'Righteous',
'Sensational', 'Slick', 'Splendid', 'Lovely', 'Stellar', 'Sterling',
'Superb', 'Superior', 'Superlative', 'Supernal', 'Swell', 'Terrific',
'Tip-Top', 'Top Notch', 'Top Shelf', 'Unsurpassed', 'Wonderful'
]
def __init__(self, url, args):
self.url = url
self.args = args
self.gerrit = Gerrit(url)
self.tag = 'autogenerated:review-o-matic'
self.blacklist = []
self.stats = {
ReviewType.SUCCESS: 0,
ReviewType.BACKPORT: 0,
ReviewType.ALTERED_UPSTREAM: 0,
ReviewType.MISSING_FIELDS: 0,
ReviewType.MISSING_HASH: 0,
ReviewType.INCORRECT_PREFIX: 0,
ReviewType.FIXES_REF: 0
}
def do_review(self, review_type, change, fixes_ref, msg, notify, vote):
final_msg = self.STRING_HEADER
if fixes_ref:
print('Adding fixes ref for change {}'.format(change.url()))
self.stats[ReviewType.FIXES_REF] += 1
final_msg += self.STRING_FOUND_FIXES_REF.format(fixes_ref)
final_msg += msg
final_msg += self.STRING_FOOTER
self.stats[review_type] += 1
if not self.args.dry_run:
self.gerrit.review(change,
self.tag,
final_msg,
notify,
vote_code_review=vote)
else:
print('Review for change: {}'.format(change.url()))
print(' Type:{}, Vote:{}, Notify:{}'.format(
review_type, vote, notify))
print(final_msg)
print('------')
def handle_successful_review(self, change, prefix, fixes_ref):
# TODO: We should tag FROMLIST: BACKPORT: patches as incorrect, if needed
if prefix == 'BACKPORT':
print('Adding incorrect prefix review for change {}'.format(
change.url()))
msg = self.STRING_INCORRECT_PREFIX
self.do_review(ReviewType.INCORRECT_PREFIX, change, fixes_ref, msg,
True, 0)
else:
print('Adding successful review for change {}'.format(
change.url()))
msg = self.STRING_SUCCESS.format(random.choice(self.SWAG))
self.do_review(ReviewType.SUCCESS, change, fixes_ref, msg, True, 1)
def handle_missing_fields_review(self, change, fields, result, fixes_ref):
print('Adding missing fields review for change {}'.format(
change.url()))
missing = []
if not fields['bug']:
missing.append('BUG=')
if not fields['test']:
missing.append('TEST=')
if not fields['sob']:
cur_rev = change.current_revision
missing.append('Signed-off-by: {} <{}>'.format(
cur_rev.uploader_name, cur_rev.uploader_email))
msg = self.STRING_MISSING_FIELDS.format(', '.join(missing))
if len(result) == 0:
msg += self.STRING_MISSING_FIELDS_SUCCESS.format(
random.choice(self.SWAG))
else:
msg += self.STRING_MISSING_FIELDS_DIFF
msg += self.STRING_UNSUCCESSFUL_FOOTER
for l in result:
msg += '{}\n'.format(l)
self.do_review(ReviewType.MISSING_FIELDS, change, fixes_ref, msg, True,
-1)
def handle_missing_hash_review(self, change, prefix):
print('Adding missing hash review for change {}'.format(change.url()))
msg = self.STRING_MISSING_HASH_HEADER
if prefix == 'FROMGIT':
msg += self.STRING_MISSING_HASH_FMT_FROMGIT
else:
msg += self.STRING_MISSING_HASH_FMT_UPSTREAM
msg += self.STRING_MISSING_HASH_FOOTER
self.do_review(ReviewType.MISSING_HASH, change, None, msg, True, -1)
def handle_missing_am_review(self, change, prefix):
print('Adding missing am URL for change {}'.format(change.url()))
self.do_review(ReviewType.MISSING_HASH, change, None,
self.STRING_MISSING_AM, True, -1)
def handle_unsuccessful_review(self, change, prefix, result, fixes_ref):
vote = 0
notify = False
review_type = ReviewType.BACKPORT
msg = self.STRING_UNSUCCESSFUL_HEADER
if prefix == 'UPSTREAM':
review_type = ReviewType.ALTERED_UPSTREAM
vote = -1
notify = True
msg += self.STRING_UPSTREAM_DIFF
elif prefix == 'BACKPORT':
msg += self.STRING_BACKPORT_DIFF
elif prefix == 'FROMGIT' or prefix == 'FROMLIST':
msg += self.STRING_FROMGIT_DIFF
msg += self.STRING_UNSUCCESSFUL_FOOTER
for l in result:
msg += '{}\n'.format(l)
print('Adding unsuccessful review (vote={}) for change {}'.format(
vote, change.url()))
self.do_review(review_type, change, fixes_ref, msg, notify, vote)
def get_changes(self, prefix):
message = '{}:'.format(prefix)
after = datetime.date.today() - datetime.timedelta(days=5)
changes = self.gerrit.query_changes(
status='open',
message=message,
after=after,
project='chromiumos/third_party/kernel')
return changes
def print_error(self, error):
if self.args.verbose:
sys.stderr.write('\n')
sys.stderr.write(error)
def process_changes(self, prefix, changes):
rev = Reviewer(git_dir=self.args.git_dir,
verbose=self.args.verbose,
chatty=self.args.chatty)
num_changes = len(changes)
cur_change = 1
line_feed = False
ret = False
for c in changes:
cur_rev = c.current_revision
if self.args.chatty:
print('Processing change {}'.format(c.url()))
elif self.args.verbose:
sys.stdout.write('{}Processing change {}/{}'.format(
'\r' if line_feed else '', cur_change, num_changes))
cur_change += 1
line_feed = True
if c in self.blacklist:
continue
if not c.subject.startswith(prefix):
continue
skip = False
for m in c.messages:
if m.tag == self.tag and m.revision_num == cur_rev.number:
skip = True
if skip and not self.args.force_cl:
continue
ret = True
line_feed = False
if self.args.verbose:
print('')
gerrit_patch = rev.get_commit_from_remote('cros', cur_rev.ref)
if prefix == 'FROMLIST':
upstream_patchworks = rev.get_am_from_from_patch(gerrit_patch)
if not upstream_patchworks:
self.handle_missing_am_review(c, prefix)
continue
upstream_patch = None
for u in reversed(upstream_patchworks):
try:
upstream_patch = rev.get_commit_from_patchwork(u)
break
except:
continue
if not upstream_patch:
self.print_error(
'ERROR: patch missing from patchwork, or patchwork host '
'not whitelisted for {} ({})\n'.format(
c, upstream_patchworks))
self.blacklist.append(c)
continue
else:
upstream_shas = rev.get_cherry_pick_shas_from_patch(
gerrit_patch)
if not upstream_shas:
self.handle_missing_hash_review(c, prefix)
continue
upstream_patch = None
upstream_sha = None
for s in reversed(upstream_shas):
try:
upstream_patch = rev.get_commit_from_sha(s)
upstream_sha = s
break
except:
continue
if not upstream_patch:
self.print_error(
'ERROR: SHA missing from git for {} ({})\n'.format(
c, upstream_shas))
self.blacklist.append(c)
continue
if prefix != 'FROMLIST':
fixes_ref = rev.find_fixes_reference(upstream_sha)
else:
fixes_ref = None
result = rev.compare_diffs(upstream_patch, gerrit_patch)
fields = {'sob': False, 'bug': False, 'test': False}
sob_re = re.compile('Signed-off-by:\s+{}'.format(
cur_rev.uploader_name))
for l in cur_rev.commit_message.splitlines():
if l.startswith('BUG='):
fields['bug'] = True
continue
if l.startswith('TEST='):
fields['test'] = True
continue
if sob_re.match(l):
fields['sob'] = True
continue
if not fields['bug'] or not fields['test'] or not fields['sob']:
self.handle_missing_fields_review(c, fields, result, fixes_ref)
continue
if len(result) == 0:
self.handle_successful_review(c, prefix, fixes_ref)
continue
self.handle_unsuccessful_review(c, prefix, result, fixes_ref)
if self.args.verbose:
print('')
return ret
def run(self):
if self.args.force_cl != None:
c = self.gerrit.get_change(self.args.force_cl)
prefix = c.subject.split(':')[0]
print('Force reviewing change {}'.format(c))
self.process_changes(prefix, [c])
return
while True:
try:
prefixes = ['UPSTREAM', 'BACKPORT', 'FROMGIT']
did_review = False
for p in prefixes:
changes = self.get_changes(p)
if self.args.verbose:
print('{} changes for prefix {}'.format(
len(changes), p))
did_review |= self.process_changes(p, changes)
if did_review:
print('--')
summary = ' Summary: '
for k, v in self.stats.items():
summary += '{}={} '.format(k, v)
print(summary)
print('')
if not self.args.daemon:
break
if self.args.verbose:
print('Finished! Going to sleep until next run')
except requests.exceptions.HTTPError as e:
self.print_error('HTTPError ({})\n'.format(
e.response.status_code))
time.sleep(60)
time.sleep(120)
class Submitter(object):
def __init__(self, last_cid, review, verify, ready, abandon, force_review,
dry_run):
self.abandon = abandon
self.vote_review = 2 if review else None
self.vote_verify = 1 if verify else None
self.vote_cq_ready = ready
self.force_review = force_review
self.dry_run = dry_run
self.tag = 'autogenerated:submit-o-matic'
self.max_in_flight = 100 # 50 for the cq, 50 for the pre-cq
self.in_flight = []
self.changes = []
self.gerrit = Gerrit('https://chromium-review.googlesource.com',
use_internal=False)
last_change = self.gerrit.get_change(last_cid)
ancestor_changes = self.gerrit.get_ancestor_changes(last_change)
for c in reversed(ancestor_changes):
if c.status == 'NEW':
self.changes.append(c)
self.changes.append(last_change)
def change_needs_action(self, change):
return change.is_merged() or \
(self.vote_review and not change.is_reviewed()) or \
(self.vote_verify and not change.is_verified()) or \
(self.vote_cq_ready and not change.is_cq_ready())
def num_changes(self):
return len(self.changes)
def num_in_flight(self):
return len(self.in_flight)
def review_changes(self):
if not self.vote_review and not self.vote_verify and not self.abandon:
return
for i, c in enumerate(self.changes):
sys.stdout.write('\rRunning reviewer (%d/%d)' %
(i, self.num_changes()))
c = self.gerrit.get_change(c.number)
if self.abandon:
if not self.dry_run:
self.gerrit.abandon(c)
else:
print('DRYRUN abandon {}'.format(c))
continue
if (c.is_merged() or
not self.change_needs_action(c)) and not self.force_review:
continue
if not self.dry_run:
self.gerrit.review(c, self.tag, '', False, self.vote_review,
self.vote_verify, None)
else:
print('DRYRUN review (r={}, v={}) {}'.format(
self.vote_review, self.vote_verify, c))
def submit_changes(self):
if self.abandon:
return
self.in_flight = []
merged = 0
for i, c in enumerate(self.changes):
if self.num_in_flight() >= self.max_in_flight:
break
sys.stdout.write('\rRunning submitter (%d/%d)' %
(i, self.num_changes()))
c = self.gerrit.get_change(c.id)
if c.is_merged():
merged += 1
continue
if self.change_needs_action(c):
if not self.dry_run:
self.gerrit.review(c, self.tag, '', False,
self.vote_review, self.vote_verify,
self.vote_cq_ready)
else:
print('DRYRUN review (r={}, v={} cq={}) {}'.format(
self.vote_review, self.vote_verify, self.vote_cq_ready,
c))
self.in_flight.append(c)
sys.stdout.write(
'\r%d Changes: \n' %
self.num_changes())
sys.stdout.write('-- %d merged\n' % merged)
sys.stdout.write('-- %d in flight\n' % self.num_in_flight())
def detect_change(self):
if self.num_in_flight(
) == 0: # everything is merged, so no detection needed
return True
c = self.in_flight[0]
sys.stdout.write('\rDetecting: %s' % c.url())
c = self.gerrit.get_change(c.id)
if self.change_needs_action(c):
return True
return False
