def setRepoLabels (gh, repo_name, all_labels, dryRun=False):
repos = []
if not "/" in repo_name:
user = gh.get_user(repo_name)
for repo in user.get_repos():
repos.append(repo)
else:
repos.append(gh.get_repo(repo_name))
api_rate_limits(gh)
for repo in repos:
print "Checking repository ", repo.full_name
cur_labels = {}
for lab in repo.get_labels():
cur_labels [lab.name]=lab
api_rate_limits(gh)
for lab in all_labels:
if not lab in cur_labels:
print " Creating new label ",lab,"=>",all_labels[lab]
if not dryRun:
repo.create_label(lab, all_labels[lab])
api_rate_limits(gh)
elif cur_labels[lab].color != all_labels[lab]:
if not dryRun:
cur_labels[lab].edit(lab, all_labels[lab])
api_rate_limits(gh)
print " Label ",lab," color updatd: ",cur_labels[lab].color ," => ",all_labels[lab]
def setRepoLabels(gh, repo_name, all_labels, dryRun=False):
repos = []
if not "/" in repo_name:
user = gh.get_user(repo_name)
for repo in user.get_repos():
repos.append(repo)
else:
repos.append(gh.get_repo(repo_name))
api_rate_limits(gh)
for repo in repos:
print "Checking repository ", repo.full_name
cur_labels = {}
for lab in repo.get_labels():
cur_labels[lab.name] = lab
api_rate_limits(gh)
for lab in all_labels:
if not lab in cur_labels:
print " Creating new label ", lab, "=>", all_labels[lab]
if not dryRun:
repo.create_label(lab, all_labels[lab])
api_rate_limits(gh)
elif cur_labels[lab].color != all_labels[lab]:
if not dryRun:
cur_labels[lab].edit(lab, all_labels[lab])
api_rate_limits(gh)
print " Label ", lab, " color updatd: ", cur_labels[
lab].color, " => ", all_labels[lab]
def setRepoLabels (gh, repo_name, all_labels, dryRun=False, ignore=[]):
repos = []
if not "/" in repo_name:
user = gh.get_user(repo_name)
for repo in user.get_repos():
skip = False
if repo.full_name in ignore:
skip = True
elif repo_name==GH_CMSSW_ORGANIZATION:
if repo.name not in VALID_CMS_SW_REPOS_FOR_TESTS:
skip = True
if skip:
print("Ignoring repo:",repo.full_name)
continue
repos.append(repo)
else:
repos.append(gh.get_repo(repo_name))
api_rate_limits(gh)
for repo in repos:
print("Checking repository ", repo.full_name, ", DryRun:",dryRun)
cur_labels = {}
for lab in repo.get_labels():
cur_labels [lab.name]=lab
api_rate_limits(gh)
for lab in all_labels:
if not lab in cur_labels:
print(" Creating new label ",lab,"=>",all_labels[lab])
if not dryRun:
repo.create_label(lab, all_labels[lab])
api_rate_limits(gh)
elif cur_labels[lab].color != all_labels[lab]:
if not dryRun:
cur_labels[lab].edit(lab, all_labels[lab])
api_rate_limits(gh)
print(" Label ",lab," color updated: ",cur_labels[lab].color ," => ",all_labels[lab])
def clean_labels(repo):
deflabels = [
'bug', 'documentation', 'duplicate', 'enhancement', 'good first idea',
'help wanted', 'invalid', 'question', 'wontfix'
]
api_rate_limits(gh)
print repo.name
api_rate_limits(gh)
for label in repo.get_labels():
if not label.name in deflabels:
print label.name
label.delete()
api_rate_limits(gh)
}
#################################
parser = ArgumentParser()
parser.add_argument("-o", "--organization", dest="organization", help="Github Organization name e.g. cms-sw. Default is * i.e. all cms origanizations", type=str, default="*")
parser.add_argument("-n", "-dry-run", dest="dryRun", default=False, action="store_true")
args = parser.parse_args()
GH_TOKEN = open(expanduser("~/.github-token")).read().strip()
gh = Github(login_or_token=GH_TOKEN)
cache = {"users" : {}}
total_changes=0
err_code=0
for org_name in CMS_ORGANIZATIONS:
if args.organization!="*" and org_name!=args.organization: continue
print "Wroking on Organization ",org_name
api_rate_limits(gh)
org = gh.get_organization(org_name)
ok_mems = []
print " Looking for owners:",REPO_OWNERS[org_name]
chg_flag=0
for mem in org.get_members(role="admin"):
login = mem.login.encode("ascii", "ignore")
if not login in cache["users"]: cache["users"][login] = mem
if not login in REPO_OWNERS[org_name]:
if not args.dryRun: add_organization_member(GH_TOKEN, org_name, login, role="member")
print " =>Remove owner:",login
chg_flag+=1
else:
ok_mems.append(login)
for login in [ l for l in REPO_OWNERS[org_name] if not l in ok_mems ]:
if not args.dryRun: add_organization_member(GH_TOKEN, org_name, login, role="admin")
print('you have to choose an action')
parser.print_help()
exit()
ACTION = args[0]
if (ACTION == 'prBot.py'):
print('you have to choose an action')
parser.print_help()
exit()
print('you chose the action %s' % ACTION)
TOKEN = open(expanduser(repo_config.GH_TOKEN)).read().strip()
github = Github( login_or_token = TOKEN )
api_rate_limits(github)
if (options.pr_number == -1 ):
complain_missing_param('pull request number')
exit()
else:
pr_number = options.pr_number
if (options.pr_job_id == -1 ):
complain_missing_param( 'pull request job id' )
exit()
else:
pr_job_id=options.pr_job_id
destination_repo = github.get_repo( options.custom_repo )
COMMIT_STATUS_BASE_URL = 'https://api.github.com/repos/'+destination_repo.full_name+'/statuses/%s'
GH_TOKEN = open(expanduser("~/.github-token")).read().strip()
gh = Github(login_or_token=GH_TOKEN)
cache = {"users": {}}
total_changes = 0
err_code = 0
for org_name in CMS_ORGANIZATIONS:
if args.organization != "*" and org_name != args.organization: continue
print("Wroking on Organization ", org_name)
pending_members = []
for user in get_pending_members(GH_TOKEN, org_name):
user = user['login'].encode("ascii", "ignore")
pending_members.append(user)
print("Pending Invitatiosn: %s" %
",".join(["@%s" % u for u in pending_members]))
api_rate_limits(gh)
org = gh.get_organization(org_name)
ok_mems = []
print(" Looking for owners:", REPO_OWNERS[org_name])
chg_flag = 0
for mem in org.get_members(role="admin"):
login = mem.login.encode("ascii", "ignore")
if not login in cache["users"]: cache["users"][login] = mem
if not login in REPO_OWNERS[org_name]:
if not args.dryRun:
add_organization_member(GH_TOKEN,
org_name,
login,
role="member")
print(" =>Remove owner:", login)
chg_flag += 1
print 'you have to choose an action'
parser.print_help()
exit()
ACTION = args[0]
if (ACTION == 'prBot.py'):
print 'you have to choose an action'
parser.print_help()
exit()
print 'you chose the action %s' % ACTION
TOKEN = open(expanduser(repo_config.GH_TOKEN)).read().strip()
github = Github(login_or_token=TOKEN)
api_rate_limits(github)
if (options.pr_number == -1):
complain_missing_param('pull request number')
exit()
else:
pr_number = options.pr_number
if (options.pr_job_id == -1):
complain_missing_param('pull request job id')
exit()
else:
pr_job_id = options.pr_job_id
destination_repo = github.get_repo(options.custom_repo)
COMMIT_STATUS_BASE_URL = 'https://api.github.com/repos/' + destination_repo.full_name + '/statuses/%s'
parser.add_option("-u", "--users", dest="users", action="store_true", help="Only process Users externals repositories", default=False)
parser.add_option("-c", "--cmssw", dest="cmssw", action="store_true", help="Only process "+",".join(CMSSW_REPOS)+" repository", default=False)
parser.add_option("-d", "--cmsdist", dest="cmsdist", action="store_true", help="Only process "+",".join(CMSDIST_REPOS)+" repository", default=False)
parser.add_option("-a", "--all", dest="all", action="store_true", help="Process all CMS repository i.e. externals, cmsdist and cmssw", default=False)
opts, args = parser.parse_args()
if opts.all:
opts.externals = True
opts.cmssw = True
opts.cmsdist = True
elif (not opts.externals) and (not opts.cmssw) and (not opts.cmsdist) and (not opts.users):
parser.error("Too few arguments, please use either -e, -c, -u or -d")
import repo_config
gh = Github(login_or_token=open(expanduser(repo_config.GH_TOKEN)).read().strip())
api_rate_limits(gh)
if opts.externals:
all_labels = COMMON_LABELS
for cat in COMMON_CATEGORIES+EXTERNAL_CATEGORIES:
for lab in LABEL_TYPES:
all_labels[cat+"-"+lab]=LABEL_TYPES[lab]
for repo_name in EXTERNAL_REPOS:
setRepoLabels (gh, repo_name, all_labels, opts.dryRun)
if opts.cmssw:
all_labels = COMMON_LABELS
for lab in COMPARISON_LABELS:
all_labels[lab] = COMPARISON_LABELS[lab]
for lab in CMSSW_BUILD_LABELS:
all_labels[lab] = CMSSW_BUILD_LABELS[lab]
for cat in COMMON_CATEGORIES+CMSSW_CATEGORIES.keys():
if len(args) != 0: parser.error("Too many/few arguments")
repo_dir = join(SCRIPT_DIR, 'repos', opts.repository)
if exists(join(repo_dir, "repo_config.py")): sys.path.insert(0, repo_dir)
import repo_config
from process_pr import get_backported_pr
gh = Github(
login_or_token=open(expanduser(repo_config.GH_TOKEN)).read().strip())
repo = gh.get_repo(opts.repository)
label = [repo.get_label("backport")]
issues = repo.get_issues(state="open", sort="updated", labels=label)
for issue in issues:
if not issue.pull_request: continue
api_rate_limits(gh)
backport_pr = None
issue_body = issue.body.encode("ascii", "ignore")
if (issue.user.login == CMSBUILD_GH_USER) and re.match(
ISSUE_SEEN_MSG,
issue_body.split("\n", 1)[0].strip()):
backport_pr = get_backported_pr(issue_body)
else:
for comment in issue.get_comments():
commenter = comment.user.login
comment_msg = comment.body.encode("ascii", "ignore")
# The first line is an invariant.
comment_lines = [
l.strip() for l in comment_msg.split("\n") if l.strip()
]
first_line = comment_lines[0:1]
def process_pr(repo_config,
gh,
repo,
issue,
dryRun,
cmsbuild_user=None,
force=False):
if (not force) and ignore_issue(repo_config, repo, issue): return
api_rate_limits(gh)
prId = issue.number
repository = repo.full_name
repo_org, repo_name = repository.split("/", 1)
if not cmsbuild_user: cmsbuild_user = repo_config.CMSBUILD_USER
print "Working on ", repo.full_name, " for PR/Issue ", prId, "with admin user", cmsbuild_user
cmssw_repo = (repo_name == GH_CMSSW_REPO)
external_repo = len([
e for e in EXTERNAL_REPOS + CMSDIST_REPOS
if (repository == e) or (repo_org == e)
]) > 0
official_repo = (repo_org == GH_CMSSW_ORGANIZATION)
create_test_property = False
packages = set([])
create_external_issue = False
add_external_category = False
signing_categories = set([])
new_package_message = ""
mustClose = False
releaseManagers = []
signatures = {}
watchers = []
#Process Pull Request
pkg_categories = set([])
REGEX_EX_CMDS = "^type\s+(bug(-fix|fix|)|(new-|)feature)|urgent|backport\s+(of\s+|)(#|http(s|):/+github\.com/+%s/+pull/+)\d+$" % (
repo.full_name)
last_commit_date = None
if issue.pull_request:
pr = repo.get_pull(prId)
if pr.changed_files == 0:
print "Ignoring: PR with no files changed"
return
if cmssw_repo and official_repo and (pr.base.ref
== CMSSW_DEVEL_BRANCH):
if pr.state != "closed":
print "This pull request must go in to master branch"
if not dryRun:
edit_pr(get_token(gh), repo.full_name, prId, base="master")
msg = format(
"@%(user)s, %(dev_branch)s branch is closed for direct updates. cms-bot is going to move this PR to master branch.\n"
"In future, please use cmssw master branch to submit your changes.\n",
user=issue.user.login.encode("ascii", "ignore"),
dev_branch=CMSSW_DEVEL_BRANCH)
issue.create_comment(msg)
return
# A pull request is by default closed if the branch is a closed one.
if pr.base.ref in RELEASE_BRANCH_CLOSED: mustClose = True
# Process the changes for the given pull request so that we can determine the
# signatures it requires.
if cmssw_repo or not external_repo:
if cmssw_repo and (pr.base.ref == "master"):
signing_categories.add("code-checks")
packages = sorted([
x for x in set([
cmssw_file2Package(repo_config, f)
for f in get_changed_files(repo, pr)
])
])
print "First Package: ", packages[0]
if cmssw_repo: updateMilestone(repo, issue, pr, dryRun)
create_test_property = True
else:
add_external_category = True
packages = set(["externals/" + repository])
if repo_name != GH_CMSDIST_REPO:
if repo_name != "cms-bot":
create_external_issue = repo_config.CREATE_EXTERNAL_ISSUE
else:
create_test_property = True
if not re.match(VALID_CMSDIST_BRANCHES, pr.base.ref):
print "Skipping PR as it does not belong to valid CMSDIST branch"
return
print "Following packages affected:"
print "\n".join(packages)
pkg_categories = set([
category for package in packages
for category, category_packages in CMSSW_CATEGORIES.items()
if package in category_packages
])
signing_categories.update(pkg_categories)
# For PR, we always require tests.
signing_categories.add("tests")
if add_external_category: signing_categories.add("externals")
# We require ORP approval for releases which are in production.
# or all externals package
if official_repo and ((not cmssw_repo) or
(pr.base.ref in RELEASE_BRANCH_PRODUCTION)):
print "This pull request requires ORP approval"
signing_categories.add("orp")
for l1 in CMSSW_L1:
if not l1 in CMSSW_L2: CMSSW_L2[l1] = []
if not "orp" in CMSSW_L2[l1]: CMSSW_L2[l1].append("orp")
print "Following categories affected:"
print "\n".join(signing_categories)
if cmssw_repo:
# If there is a new package, add also a dummy "new" category.
all_packages = [
package for category_packages in CMSSW_CATEGORIES.values()
for package in category_packages
]
has_category = all(
[package in all_packages for package in packages])
if not has_category:
new_package_message = "\nThe following packages do not have a category, yet:\n\n"
new_package_message += "\n".join([
package
for package in packages if not package in all_packages
]) + "\n"
new_package_message += "Please create a PR for https://github.com/cms-sw/cms-bot/blob/master/categories_map.py to assign category\n"
print new_package_message
signing_categories.add("new-package")
# Add watchers.yaml information to the WATCHERS dict.
WATCHERS = read_repo_file(repo_config, "watchers.yaml", {})
# Given the packages check if there are additional developers watching one or more.
author = pr.user.login
watchers = set([
user for package in packages
for user, watched_regexp in WATCHERS.items()
for regexp in watched_regexp
if re.match("^" + regexp + ".*", package) and user != author
])
#Handle category watchers
catWatchers = read_repo_file(repo_config, "category-watchers.yaml", {})
for user, cats in catWatchers.items():
for cat in cats:
if cat in signing_categories:
print "Added ", user, " to watch due to cat", cat
watchers.add(user)
# Handle watchers
watchingGroups = read_repo_file(repo_config, "groups.yaml", {})
for watcher in [x for x in watchers]:
if not watcher in watchingGroups: continue
watchers.remove(watcher)
watchers.update(set(watchingGroups[watcher]))
watchers = set(["@" + u for u in watchers])
print "Watchers " + ", ".join(watchers)
last_commit = None
try:
# This requires at least PyGithub 1.23.0. Making it optional for the moment.
last_commit = pr.get_commits().reversed[0].commit
except:
# This seems to fail for more than 250 commits. Not sure if the
# problem is github itself or the bindings.
try:
last_commit = pr.get_commits()[pr.commits - 1].commit
except IndexError:
print "Index error: May be PR with no commits"
return
last_commit_date = last_commit.committer.date
print "Latest commit by ", last_commit.committer.name.encode(
"ascii", "ignore"), " at ", last_commit_date
print "Latest commit message: ", last_commit.message.encode(
"ascii", "ignore")
print "Latest commit sha: ", last_commit.sha
releaseManagers = list(
set(
RELEASE_MANAGERS.get(pr.base.ref, []) +
SPECIAL_RELEASE_MANAGERS))
# Process the issue comments
signatures = dict([(x, "pending") for x in signing_categories])
pre_checks = ("code-checks" in signing_categories)
already_seen = None
pull_request_updated = False
comparison_done = False
comparison_notrun = False
tests_already_queued = False
tests_requested = False
mustMerge = False
external_issue_number = ""
trigger_test_on_signature = True
has_categories_approval = False
cmsdist_pr = ''
cmssw_prs = ''
extra_wfs = ''
assign_cats = {}
hold = {}
extra_labels = {}
last_test_start_time = None
body_firstline = issue.body.encode("ascii", "ignore").split("\n",
1)[0].strip()
abort_test = False
need_external = False
trigger_code_checks = False
triggerred_code_ckecks = False
backport_pr_num = ""
if (issue.user.login == cmsbuild_user) and \
re.match(ISSUE_SEEN_MSG,body_firstline):
already_seen = issue
backport_pr_num = get_backported_pr(
issue.body.encode("ascii", "ignore"))
elif re.match(REGEX_EX_CMDS, body_firstline, re.I):
check_extra_labels(body_firstline.lower(), extra_labels)
all_comments = [issue]
for c in issue.get_comments():
all_comments.append(c)
for comment in all_comments:
commenter = comment.user.login
comment_msg = comment.body.encode("ascii", "ignore")
# The first line is an invariant.
comment_lines = [
l.strip() for l in comment_msg.split("\n") if l.strip()
]
first_line = comment_lines[0:1]
if not first_line: continue
first_line = first_line[0]
if (commenter == cmsbuild_user) and re.match(ISSUE_SEEN_MSG,
first_line):
already_seen = comment
backport_pr_num = get_backported_pr(comment_msg)
if issue.pull_request and last_commit_date and (
comment.created_at >= last_commit_date):
pull_request_updated = False
if create_external_issue:
external_issue_number = comment_msg.split(
"external issue " + CMSDIST_REPO_NAME + "#",
2)[-1].split("\n")[0]
if not re.match("^[1-9][0-9]*$", external_issue_number):
print "ERROR: Unknow external issue PR format:", external_issue_number
external_issue_number = ""
continue
assign_type, new_cats = get_assign_categories(first_line)
if new_cats:
if (assign_type
== "new categories assigned:") and (commenter
== cmsbuild_user):
for ex_cat in new_cats:
if ex_cat in assign_cats: assign_cats[ex_cat] = 1
if ((commenter in CMSSW_L2)
or (commenter in CMSSW_ISSUES_TRACKERS + CMSSW_L1)):
if assign_type == "assign":
for ex_cat in new_cats:
if not ex_cat in signing_categories:
assign_cats[ex_cat] = 0
signing_categories.add(ex_cat)
signatures[ex_cat] = "pending"
elif assign_type == "unassign":
for ex_cat in new_cats:
if ex_cat in assign_cats:
assign_cats.pop(ex_cat)
signing_categories.remove(ex_cat)
signatures.pop(ex_cat)
continue
# Some of the special users can say "hold" prevent automatic merging of
# fully signed PRs.
if re.match("^hold$", first_line, re.I):
if commenter in CMSSW_L1 + CMSSW_L2.keys(
) + releaseManagers + PR_HOLD_MANAGERS:
hold[commenter] = 1
continue
if re.match(REGEX_EX_CMDS, first_line, re.I):
if commenter in CMSSW_L1 + CMSSW_L2.keys() + releaseManagers + [
issue.user.login
]:
check_extra_labels(first_line.lower(), extra_labels)
continue
if re.match("^unhold$", first_line, re.I):
if commenter in CMSSW_L1:
hold = {}
elif commenter in CMSSW_L2.keys(
) + releaseManagers + PR_HOLD_MANAGERS:
if hold.has_key(commenter): del hold[commenter]
continue
if (commenter == cmsbuild_user) and (re.match("^" + HOLD_MSG + ".+",
first_line)):
for u in first_line.split(HOLD_MSG, 2)[1].split(","):
u = u.strip().lstrip("@")
if hold.has_key(u): hold[u] = 0
if re.match("^close$", first_line, re.I):
if (not issue.pull_request
and (commenter in CMSSW_ISSUES_TRACKERS + CMSSW_L1)):
mustClose = True
continue
# Ignore all other messages which are before last commit.
if issue.pull_request and (comment.created_at < last_commit_date):
pull_request_updated = True
continue
if ("code-checks" == first_line) and ("code-checks" in signatures):
signatures["code-checks"] = "pending"
trigger_code_checks = True
continue
# Check for cmsbuild_user comments and tests requests only for pull requests
if commenter == cmsbuild_user:
if not issue.pull_request: continue
sec_line = comment_lines[1:2]
if not sec_line: sec_line = ""
else: sec_line = sec_line[0]
if re.match("Comparison is ready", first_line):
if ('tests'
in signatures) and signatures["tests"] != 'pending':
comparison_done = True
elif "-code-checks" == first_line:
signatures["code-checks"] = "rejected"
trigger_code_checks = False
triggerred_code_ckecks = False
elif "+code-checks" == first_line:
signatures["code-checks"] = "approved"
trigger_code_checks = False
triggerred_code_ckecks = False
elif TRIGERING_CODE_CHECK_MSG == first_line:
trigger_code_checks = False
triggerred_code_ckecks = True
signatures["code-checks"] = "pending"
elif re.match("^Comparison not run.+", first_line):
if ('tests'
in signatures) and signatures["tests"] != 'pending':
comparison_notrun = True
elif re.match(FAILED_TESTS_MSG, first_line) or re.match(
IGNORING_TESTS_MSG, first_line):
tests_already_queued = False
tests_requested = False
signatures["tests"] = "pending"
trigger_test_on_signature = False
elif re.match("Pull request ([^ #]+|)[#][0-9]+ was updated[.].*",
first_line):
pull_request_updated = False
elif re.match(TRIGERING_TESTS_MSG, first_line):
tests_already_queued = True
tests_requested = False
signatures["tests"] = "started"
trigger_test_on_signature = False
last_test_start_time = comment.created_at
abort_test = False
need_external = False
if sec_line.startswith("Using externals from cms-sw/cmsdist#"):
need_external = True
elif re.match(TESTS_RESULTS_MSG, first_line):
test_sha = sec_line.replace("Tested at: ", "").strip()
if (test_sha != last_commit.sha) and (
test_sha != 'UNKNOWN') and (not "I had the issue "
in first_line):
print "Ignoring test results for sha:", test_sha
continue
trigger_test_on_signature = False
tests_already_queued = False
tests_requested = False
comparison_done = False
comparison_notrun = False
if "+1" in first_line:
signatures["tests"] = "approved"
elif "-1" in first_line:
signatures["tests"] = "rejected"
else:
signatures["tests"] = "pending"
print 'Previous tests already finished, resetting test request state to ', signatures[
"tests"]
elif re.match(TRIGERING_TESTS_ABORT_MSG, first_line):
abort_test = False
continue
if issue.pull_request:
# Check if the release manager asked for merging this.
if (commenter in releaseManagers + CMSSW_L1) and re.match(
"^\s*(merge)\s*$", first_line, re.I):
mustMerge = True
mustClose = False
if (commenter in CMSSW_L1) and ("orp" in signatures):
signatures["orp"] = "approved"
continue
# Check if the someone asked to trigger the tests
if commenter in TRIGGER_PR_TESTS + CMSSW_L2.keys(
) + CMSSW_L1 + releaseManagers + [repo_org]:
ok, cmsdist_pr, cmssw_prs, extra_wfs = check_test_cmd(
first_line)
if ok:
print 'Tests requested:', commenter, 'asked to test this PR with cmsdist_pr=%s, cmssw_prs=%s and workflows=%s' % (
cmsdist_pr, cmssw_prs, extra_wfs)
trigger_test_on_signature = False
if tests_already_queued:
print "Test results not obtained in ", comment.created_at - last_test_start_time
diff = time.mktime(
comment.created_at.timetuple()) - time.mktime(
last_test_start_time.timetuple())
if diff >= TEST_WAIT_GAP:
print "Looks like tests are stuck, will try to re-queue"
tests_already_queued = False
if not tests_already_queued:
print 'cms-bot will request test for this PR'
tests_requested = True
comparison_done = False
comparison_notrun = False
if not cmssw_repo:
cmsdist_pr = ''
cmssw_prs = ''
signatures["tests"] = "pending"
else:
print 'Tests already request for this PR'
continue
elif (REGEX_TEST_ABORT.match(first_line)
and ((signatures["tests"] == "started") or
((signatures["tests"] != "pending") and
(not comparison_done)))):
tests_already_queued = False
abort_test = True
signatures["tests"] = "pending"
# Check L2 signoff for users in this PR signing categories
if commenter in CMSSW_L2 and [
x for x in CMSSW_L2[commenter] if x in signing_categories
]:
ctype = ""
selected_cats = []
if re.match("^([+]1|approve[d]?|sign|signed)$", first_line, re.I):
ctype = "+1"
selected_cats = CMSSW_L2[commenter]
elif re.match("^([-]1|reject|rejected)$", first_line, re.I):
ctype = "-1"
selected_cats = CMSSW_L2[commenter]
elif re.match("^[+-][a-z][a-z0-9]+$", first_line, re.I):
category_name = first_line[1:].lower()
if category_name in CMSSW_L2[commenter]:
ctype = first_line[0] + "1"
selected_cats = [category_name]
elif re.match("^(reopen)$", first_line, re.I):
ctype = "reopen"
if ctype == "+1":
for sign in selected_cats:
signatures[sign] = "approved"
has_categories_approval = True
if sign == "orp": mustClose = False
elif ctype == "-1":
for sign in selected_cats:
signatures[sign] = "rejected"
has_categories_approval = False
if sign == "orp": mustClose = False
elif ctype == "reopen":
if "orp" in CMSSW_L2[commenter]:
signatures["orp"] = "pending"
mustClose = False
continue
is_hold = len(hold) > 0
new_blocker = False
blockers = ""
for u in hold:
blockers += " @" + u + ","
if hold[u]: new_blocker = True
blockers = blockers.rstrip(",")
new_assign_cats = []
for ex_cat in assign_cats:
if assign_cats[ex_cat] == 1: continue
new_assign_cats.append(ex_cat)
print "All assigned cats:", ",".join(assign_cats.keys())
print "Newly assigned cats:", ",".join(new_assign_cats)
# Labels coming from signature.
labels = []
for cat in signing_categories:
l = cat + "-pending"
if cat in signatures: l = cat + "-" + signatures[cat]
labels.append(l)
if not issue.pull_request and len(signing_categories) == 0:
labels.append("pending-assignment")
# Additional labels.
if is_hold: labels.append("hold")
dryRunOrig = dryRun
if pre_checks and ((not already_seen) or pull_request_updated):
for cat in ["code-checks"]:
if (cat in signatures) and (signatures[cat] != "approved"):
dryRun = True
break
old_labels = set([x.name.encode("ascii", "ignore") for x in issue.labels])
print "Stats:", backport_pr_num, extra_labels
print "Old Labels:", sorted(old_labels)
if "backport" in extra_labels:
if backport_pr_num != extra_labels["backport"][1]:
try:
bp_pr = repo.get_pull(int(extra_labels["backport"][1]))
backport_pr_num = extra_labels["backport"][1]
if bp_pr.merged: extra_labels["backport"][0] = "backport-ok"
except Exception, e:
print "Error: Unknown PR", backport_pr_num, "\n", e
backport_pr_num = ""
extra_labels.pop("backport")
if already_seen:
if dryRun:
print "Update PR seen message to include backport PR number", backport_pr_num
else:
new_msg = ""
for l in already_seen.body.encode("ascii",
"ignore").split("\n"):
if BACKPORT_STR in l: continue
new_msg += l + "\n"
if backport_pr_num:
new_msg = "%s%s%s\n" % (new_msg, BACKPORT_STR,
backport_pr_num)
already_seen.edit(body=new_msg)
elif ("backport-ok" in old_labels):
extra_labels["backport"][0] = "backport-ok"
def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=False):
if (not force) and ignore_issue(repo_config, repo, issue): return
api_rate_limits(gh)
prId = issue.number
repository = repo.full_name
repo_org, repo_name = repository.split("/",1)
if not cmsbuild_user: cmsbuild_user=repo_config.CMSBUILD_USER
print "Working on ",repo.full_name," for PR/Issue ",prId,"with admin user",cmsbuild_user
cmssw_repo = (repo_name==GH_CMSSW_REPO)
official_repo = (repo_org==GH_CMSSW_ORGANIZATION)
external_repo = (repository!=CMSSW_REPO_NAME) and (len([e for e in EXTERNAL_REPOS if repo_org==e])>0)
create_test_property = False
packages = set([])
create_external_issue = False
add_external_category = False
signing_categories = set([])
new_package_message = ""
mustClose = False
releaseManagers = []
signatures = {}
watchers = []
#Process Pull Request
pkg_categories = set([])
REGEX_EX_CMDS="^type\s+(bug(-fix|fix|)|(new-|)feature)|urgent|backport\s+(of\s+|)(#|http(s|):/+github\.com/+%s/+pull/+)\d+$" % (repo.full_name)
known_ignore_tests='build-warnings|clang-warnings|none'
REGEX_EX_IGNORE_CHKS='^ignore\s+(%s)(\s*,\s*(%s))*$' % (known_ignore_tests, known_ignore_tests)
last_commit_date = None
push_test_issue = False
requestor = issue.user.login.encode("ascii", "ignore")
ignore_tests = []
if issue.pull_request:
pr = repo.get_pull(prId)
if pr.changed_files==0:
print "Ignoring: PR with no files changed"
return
if cmssw_repo and official_repo and (pr.base.ref == CMSSW_DEVEL_BRANCH):
if pr.state != "closed":
print "This pull request must go in to master branch"
if not dryRun:
edit_pr(get_token(gh), repo.full_name, prId, base="master")
msg = format("@%(user)s, %(dev_branch)s branch is closed for direct updates. cms-bot is going to move this PR to master branch.\n"
"In future, please use cmssw master branch to submit your changes.\n",
user=requestor,
dev_branch=CMSSW_DEVEL_BRANCH)
issue.create_comment(msg)
return
# A pull request is by default closed if the branch is a closed one.
if pr.base.ref in RELEASE_BRANCH_CLOSED: mustClose = True
# Process the changes for the given pull request so that we can determine the
# signatures it requires.
if cmssw_repo or not external_repo:
if cmssw_repo and (pr.base.ref=="master"): signing_categories.add("code-checks")
packages = sorted([x for x in set([cmssw_file2Package(repo_config, f)
for f in get_changed_files(repo, pr)])])
print "First Package: ",packages[0]
if cmssw_repo: updateMilestone(repo, issue, pr, dryRun)
create_test_property = True
else:
add_external_category = True
packages = set (["externals/"+repository])
if repo_name != GH_CMSDIST_REPO:
if not repo_name in ["cms-bot", "cmssdt-web"]: create_external_issue = repo_config.CREATE_EXTERNAL_ISSUE
else:
create_test_property = True
if not re.match(VALID_CMSDIST_BRANCHES,pr.base.ref):
print "Skipping PR as it does not belong to valid CMSDIST branch"
return
print "Following packages affected:"
print "\n".join(packages)
pkg_categories = set([category for package in packages
for category, category_packages in CMSSW_CATEGORIES.items()
if package in category_packages])
signing_categories.update(pkg_categories)
# For PR, we always require tests.
signing_categories.add("tests")
if add_external_category: signing_categories.add("externals")
# We require ORP approval for releases which are in production.
# or all externals package
if official_repo and ((not cmssw_repo) or (pr.base.ref in RELEASE_BRANCH_PRODUCTION)):
print "This pull request requires ORP approval"
signing_categories.add("orp")
for l1 in CMSSW_L1:
if not l1 in CMSSW_L2: CMSSW_L2[l1]=[]
if not "orp" in CMSSW_L2[l1]: CMSSW_L2[l1].append("orp")
print "Following categories affected:"
print "\n".join(signing_categories)
if cmssw_repo:
# If there is a new package, add also a dummy "new" category.
all_packages = [package for category_packages in CMSSW_CATEGORIES.values()
for package in category_packages]
has_category = all([package in all_packages for package in packages])
if not has_category:
new_package_message = "\nThe following packages do not have a category, yet:\n\n"
new_package_message += "\n".join([package for package in packages if not package in all_packages]) + "\n"
new_package_message += "Please create a PR for https://github.com/cms-sw/cms-bot/blob/master/categories_map.py to assign category\n"
print new_package_message
signing_categories.add("new-package")
# Add watchers.yaml information to the WATCHERS dict.
WATCHERS = read_repo_file(repo_config, "watchers.yaml", {})
# Given the packages check if there are additional developers watching one or more.
author = pr.user.login
watchers = set([user for package in packages
for user, watched_regexp in WATCHERS.items()
for regexp in watched_regexp
if re.match("^" + regexp + ".*", package) and user != author])
#Handle category watchers
catWatchers = read_repo_file(repo_config, "category-watchers.yaml", {})
for user, cats in catWatchers.items():
for cat in cats:
if cat in signing_categories:
print "Added ",user, " to watch due to cat",cat
watchers.add(user)
# Handle watchers
watchingGroups = read_repo_file(repo_config, "groups.yaml", {})
for watcher in [x for x in watchers]:
if not watcher in watchingGroups: continue
watchers.remove(watcher)
watchers.update(set(watchingGroups[watcher]))
watchers = set(["@" + u for u in watchers])
print "Watchers " + ", ".join(watchers)
last_commit = None
try:
# This requires at least PyGithub 1.23.0. Making it optional for the moment.
last_commit = pr.get_commits().reversed[0].commit
except:
# This seems to fail for more than 250 commits. Not sure if the
# problem is github itself or the bindings.
try:
last_commit = pr.get_commits()[pr.commits - 1].commit
except IndexError:
print "Index error: May be PR with no commits"
return
last_commit_date = last_commit.committer.date
print "Latest commit by ",last_commit.committer.name.encode("ascii", "ignore")," at ",last_commit_date
print "Latest commit message: ",last_commit.message.encode("ascii", "ignore")
print "Latest commit sha: ",last_commit.sha
extra_rm = RELEASE_MANAGERS.get(pr.base.ref, [])
if repository==CMSDIST_REPO_NAME:
br = "_".join(pr.base.ref.split("/")[:2][-1].split("_")[:3])+"_X"
if br: extra_rm=extra_rm+RELEASE_MANAGERS.get(br, [])
releaseManagers=list(set(extra_rm+SPECIAL_RELEASE_MANAGERS))
else:
try:
if (repo_config.OPEN_ISSUE_FOR_PUSH_TESTS) and (requestor == cmsbuild_user) and re.match(PUSH_TEST_ISSUE_MSG,issue.title):
signing_categories.add("tests")
push_test_issue = True
except: pass
# Process the issue comments
signatures = dict([(x, "pending") for x in signing_categories])
pre_checks = ("code-checks" in signing_categories)
already_seen = None
pull_request_updated = False
comparison_done = False
comparison_notrun = False
tests_already_queued = False
tests_requested = False
mustMerge = False
external_issue_number=""
trigger_test_on_signature = True
has_categories_approval = False
cmsdist_pr = ''
cmssw_prs = ''
extra_wfs = ''
assign_cats = {}
hold = {}
extra_labels = {}
last_test_start_time = None
body_firstline = issue.body.encode("ascii", "ignore").split("\n",1)[0].strip() if issue.body else ""
abort_test = False
need_external = False
trigger_code_checks=False
triggerred_code_checks=False
backport_pr_num = ""
comp_warnings = False
if (requestor == cmsbuild_user) and \
re.match(ISSUE_SEEN_MSG,body_firstline):
already_seen = issue
backport_pr_num = get_backported_pr(issue.body.encode("ascii", "ignore")) if issue.body else ""
elif re.match(REGEX_EX_CMDS, body_firstline, re.I):
check_extra_labels(body_firstline.lower(), extra_labels)
all_comments = [issue]
for c in issue.get_comments(): all_comments.append(c)
for comment in all_comments:
commenter = comment.user.login.encode("ascii", "ignore")
valid_commenter = commenter in TRIGGER_PR_TESTS + CMSSW_L2.keys() + CMSSW_L1 + releaseManagers + [repo_org]
if (not valid_commenter) and (requestor!=commenter): continue
comment_msg = comment.body.encode("ascii", "ignore") if comment.body else ""
if (commenter in COMMENT_CONVERSION) and (comment.created_at<=COMMENT_CONVERSION[commenter]['comments_before']):
orig_msg = comment_msg
for cmt in COMMENT_CONVERSION[commenter]['comments']:
comment_msg = comment_msg.replace(cmt[0],cmt[1])
if (orig_msg != comment_msg): print "==>Updated Comment:",commenter,comment.created_at,"\n",comment_msg
# The first line is an invariant.
comment_lines = [ l.strip() for l in comment_msg.split("\n") if l.strip() ]
first_line = comment_lines[0:1]
if not first_line: continue
first_line = first_line[0]
if (commenter == cmsbuild_user) and re.match(ISSUE_SEEN_MSG, first_line):
already_seen = comment
backport_pr_num = get_backported_pr(comment_msg)
if issue.pull_request and last_commit_date and (comment.created_at >= last_commit_date): pull_request_updated = False
if create_external_issue:
external_issue_number=comment_msg.split("external issue "+CMSDIST_REPO_NAME+"#",2)[-1].split("\n")[0]
if not re.match("^[1-9][0-9]*$",external_issue_number):
print "ERROR: Unknow external issue PR format:",external_issue_number
external_issue_number=""
continue
assign_type, new_cats = get_assign_categories(first_line)
if new_cats:
if (assign_type == "new categories assigned:") and (commenter == cmsbuild_user):
for ex_cat in new_cats:
if ex_cat in assign_cats: assign_cats[ex_cat] = 1
if ((commenter in CMSSW_L2) or (commenter in CMSSW_ISSUES_TRACKERS + CMSSW_L1)):
if assign_type == "assign":
for ex_cat in new_cats:
if not ex_cat in signing_categories:
assign_cats[ex_cat] = 0
signing_categories.add(ex_cat)
signatures[ex_cat]="pending"
elif assign_type == "unassign":
for ex_cat in new_cats:
if ex_cat in assign_cats:
assign_cats.pop(ex_cat)
signing_categories.remove(ex_cat)
signatures.pop(ex_cat)
continue
# Some of the special users can say "hold" prevent automatic merging of
# fully signed PRs.
if re.match("^hold$", first_line, re.I):
if commenter in CMSSW_L1 + CMSSW_L2.keys() + releaseManagers + PR_HOLD_MANAGERS: hold[commenter]=1
continue
if re.match(REGEX_EX_CMDS, first_line, re.I):
if commenter in CMSSW_L1 + CMSSW_L2.keys() + releaseManagers + [requestor]:
check_extra_labels(first_line.lower(), extra_labels)
continue
if re.match(REGEX_EX_IGNORE_CHKS, first_line, re.I):
if commenter in CMSSW_L1 + CMSSW_L2.keys() + releaseManagers:
ignore_tests = check_ignore_test (first_line.upper())
if 'NONE' in ignore_tests: ignore_tests=[]
continue
if re.match("^unhold$", first_line, re.I):
if commenter in CMSSW_L1:
hold = {}
elif commenter in CMSSW_L2.keys() + releaseManagers + PR_HOLD_MANAGERS:
if hold.has_key(commenter): del hold[commenter]
continue
if (commenter == cmsbuild_user) and (re.match("^"+HOLD_MSG+".+", first_line)):
for u in first_line.split(HOLD_MSG,2)[1].split(","):
u = u.strip().lstrip("@")
if hold.has_key(u): hold[u]=0
if re.match("^close$", first_line, re.I):
if (not issue.pull_request and (commenter in CMSSW_ISSUES_TRACKERS + CMSSW_L1)):
mustClose = True
continue
# Ignore all other messages which are before last commit.
if issue.pull_request and (comment.created_at < last_commit_date):
pull_request_updated = True
continue
if ("code-checks"==first_line) and ("code-checks" in signatures):
signatures["code-checks"] = "pending"
trigger_code_checks=True
triggerred_code_checks=False
continue
# Check for cmsbuild_user comments and tests requests only for pull requests
if commenter == cmsbuild_user:
if not issue.pull_request and not push_test_issue: continue
sec_line = comment_lines[1:2]
if not sec_line: sec_line = ""
else: sec_line = sec_line[0]
if re.match("Comparison is ready", first_line):
if ('tests' in signatures) and signatures["tests"]!='pending': comparison_done = True
elif "-code-checks" == first_line:
signatures["code-checks"] = "rejected"
trigger_code_checks=False
triggerred_code_checks=False
elif "+code-checks" == first_line:
signatures["code-checks"] = "approved"
trigger_code_checks=False
triggerred_code_checks=False
elif TRIGERING_CODE_CHECK_MSG == first_line:
trigger_code_checks=False
triggerred_code_checks=True
signatures["code-checks"] = "pending"
elif re.match("^Comparison not run.+",first_line):
if ('tests' in signatures) and signatures["tests"]!='pending': comparison_notrun = True
elif re.match( FAILED_TESTS_MSG, first_line) or re.match(IGNORING_TESTS_MSG, first_line):
tests_already_queued = False
tests_requested = False
signatures["tests"] = "pending"
trigger_test_on_signature = False
elif re.match("Pull request ([^ #]+|)[#][0-9]+ was updated[.].*", first_line):
pull_request_updated = False
elif re.match( TRIGERING_TESTS_MSG, first_line) or re.match( TRIGERING_TESTS_MSG1, first_line):
tests_already_queued = True
tests_requested = False
signatures["tests"] = "started"
trigger_test_on_signature = False
last_test_start_time = comment.created_at
abort_test = False
need_external = False
if sec_line.startswith("Using externals from cms-sw/cmsdist#"): need_external = True
elif re.match( TESTS_RESULTS_MSG, first_line):
test_sha = sec_line.replace("Tested at: ","").strip()
if (not push_test_issue) and (test_sha != last_commit.sha) and (test_sha != 'UNKNOWN') and (not "I had the issue " in first_line):
print "Ignoring test results for sha:",test_sha
continue
trigger_test_on_signature = False
tests_already_queued = False
tests_requested = False
comparison_done = False
comparison_notrun = False
comp_warnings = False
if "+1" in first_line:
signatures["tests"] = "approved"
comp_warnings = len([1 for l in comment_lines if 'Compilation Warnings: Yes' in l ])>0
elif "-1" in first_line:
signatures["tests"] = "rejected"
else:
signatures["tests"] = "pending"
print 'Previous tests already finished, resetting test request state to ',signatures["tests"]
elif re.match( TRIGERING_TESTS_ABORT_MSG, first_line):
abort_test = False
continue
if issue.pull_request or push_test_issue:
# Check if the release manager asked for merging this.
if (commenter in releaseManagers + CMSSW_L1) and re.match("^\s*(merge)\s*$", first_line, re.I):
mustMerge = True
mustClose = False
if (commenter in CMSSW_L1) and ("orp" in signatures): signatures["orp"] = "approved"
continue
# Check if the someone asked to trigger the tests
if valid_commenter:
ok, cmsdist_pr, cmssw_prs, extra_wfs = check_test_cmd(first_line)
if ok:
print 'Tests requested:', commenter, 'asked to test this PR with cmsdist_pr=%s, cmssw_prs=%s and workflows=%s' % (cmsdist_pr, cmssw_prs, extra_wfs)
print "Comment message:",first_line
trigger_test_on_signature = False
if tests_already_queued:
print "Test results not obtained in ",comment.created_at-last_test_start_time
diff = time.mktime(comment.created_at.timetuple()) - time.mktime(last_test_start_time.timetuple())
if diff>=TEST_WAIT_GAP:
print "Looks like tests are stuck, will try to re-queue"
tests_already_queued = False
if not tests_already_queued:
print 'cms-bot will request test for this PR'
tests_requested = True
comparison_done = False
comparison_notrun = False
if not cmssw_repo:
cmsdist_pr = ''
cmssw_prs = ''
signatures["tests"] = "pending"
else:
print 'Tests already request for this PR'
continue
elif (REGEX_TEST_ABORT.match(first_line) and
((signatures["tests"] == "started") or
((signatures["tests"] != "pending") and (not comparison_done) and (not push_test_issue)))):
tests_already_queued = False
abort_test = True
signatures["tests"] = "pending"
# Check L2 signoff for users in this PR signing categories
if commenter in CMSSW_L2 and [x for x in CMSSW_L2[commenter] if x in signing_categories]:
ctype = ""
selected_cats = []
if re.match("^([+]1|approve[d]?|sign|signed)$", first_line, re.I):
ctype = "+1"
selected_cats = CMSSW_L2[commenter]
elif re.match("^([-]1|reject|rejected)$", first_line, re.I):
ctype = "-1"
selected_cats = CMSSW_L2[commenter]
elif re.match("^[+-][a-z][a-z0-9]+$", first_line, re.I):
category_name = first_line[1:].lower()
if category_name in CMSSW_L2[commenter]:
ctype = first_line[0]+"1"
selected_cats = [ category_name ]
elif re.match("^(reopen)$", first_line, re.I):
ctype = "reopen"
if ctype == "+1":
for sign in selected_cats:
signatures[sign] = "approved"
has_categories_approval = True
if sign == "orp": mustClose = False
elif ctype == "-1":
for sign in selected_cats:
signatures[sign] = "rejected"
has_categories_approval = False
if sign == "orp": mustClose = False
elif ctype == "reopen":
if "orp" in CMSSW_L2[commenter]:
signatures["orp"] = "pending"
mustClose = False
continue
if push_test_issue:
auto_close_push_test_issue = True
try: auto_close_push_test_issue=repo_config.AUTO_CLOSE_PUSH_TESTS_ISSUE
except: pass
if auto_close_push_test_issue and (issue.state == "open") and ('tests' in signatures) and ((signatures["tests"] in ["approved","rejected"]) or abort_test):
print "Closing the issue as it has been tested/aborted"
if not dryRun: issue.edit(state="closed")
if abort_test:
job, bnum = get_jenkins_job(issue)
if job and bnum:
params = {}
params["JENKINS_PROJECT_TO_KILL"]=job
params["JENKINS_BUILD_NUMBER"]=bnum
create_property_file("trigger-abort-%s" % job, params, dryRun)
return
is_hold = len(hold)>0
new_blocker = False
blockers = ""
for u in hold:
blockers += " @"+u+","
if hold[u]: new_blocker = True
blockers = blockers.rstrip(",")
new_assign_cats = []
for ex_cat in assign_cats:
if assign_cats[ex_cat]==1: continue
new_assign_cats.append(ex_cat)
print "All assigned cats:",",".join(assign_cats.keys())
print "Newly assigned cats:",",".join(new_assign_cats)
print "Ignore tests:",ignore_tests
# Labels coming from signature.
labels = []
for cat in signing_categories:
l = cat+"-pending"
if cat in signatures: l = cat+"-"+signatures[cat]
labels.append(l)
if not issue.pull_request and len(signing_categories)==0:
labels.append("pending-assignment")
# Additional labels.
if is_hold: labels.append("hold")
dryRunOrig = dryRun
if pre_checks and ((not already_seen) or pull_request_updated):
for cat in ["code-checks"]:
if (cat in signatures) and (signatures[cat]!="approved"):
dryRun=True
break
old_labels = set([x.name.encode("ascii", "ignore") for x in issue.labels])
print "Stats:",backport_pr_num,extra_labels
print "Old Labels:",sorted(old_labels)
print "Compilation Warnings: ",comp_warnings
if "backport" in extra_labels:
if backport_pr_num!=extra_labels["backport"][1]:
try:
bp_pr = repo.get_pull(int(extra_labels["backport"][1]))
backport_pr_num=extra_labels["backport"][1]
if bp_pr.merged: extra_labels["backport"][0]="backport-ok"
except Exception, e :
print "Error: Unknown PR", backport_pr_num,"\n",e
backport_pr_num=""
extra_labels.pop("backport")
if already_seen:
if dryRun: print "Update PR seen message to include backport PR number",backport_pr_num
else:
new_msg = ""
for l in already_seen.body.encode("ascii", "ignore").split("\n"):
if BACKPORT_STR in l: continue
new_msg += l+"\n"
if backport_pr_num: new_msg="%s%s%s\n" % (new_msg, BACKPORT_STR, backport_pr_num)
already_seen.edit(body=new_msg)
def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=False):
api_rate_limits(gh)
if not issue.pull_request:
print("Ignoring: Not a PR")
return
prId = issue.number
pr = repo.get_pull(prId)
if pr.changed_files == 0:
print("Ignoring: PR with no files changed")
return
authorised_users = get_authorised_users(gh, repo)
not_seen_yet = True
last_time_seen = None
labels = []
# commit test states:
test_statuses = {}
test_triggered = {}
# did we already create a commit status?
test_status_exists = {}
# tests we'd like to trigger on this commit
tests_to_trigger = []
# tests we've already triggered
tests_already_triggered = []
# get PR changed libraries / packages
pr_files = pr.get_files()
# top-level folders of the Offline 'monorepo'
# that have been edited by this PR
modified_top_level_folders = get_modified(pr_files)
print ('Build Targets changed:')
print (
'\n'.join(['- %s' % s for s in modified_top_level_folders])
)
# TODO: print modified monorepo package folders in 'greeting' message
watchers = read_repo_file(repo_config, "watchers.yaml", {})
print ('watchers:', watchers)
# get required tests
test_requirements = test_suites.get_tests_for(modified_top_level_folders)
print ('Tests required: ', test_requirements)
# set their status to 'pending' (will be updated shortly after)
for test in test_requirements:
test_statuses[test] = 'pending'
test_triggered[test] = False
test_status_exists[test] = False
# this will be the commit of master that the PR is merged
# into for the CI tests (for a build test this is just the current HEAD.)
master_commit_sha = repo.get_branch("master").commit.sha
# get latest commit
last_commit = pr.get_commits().reversed[0]
git_commit = last_commit.commit
if git_commit is None:
return
last_commit_date = git_commit.committer.date
print(
"Latest commit by ",
git_commit.committer.name,
" at ",
last_commit_date,
)
print("Latest commit message: ", git_commit.message.encode("ascii", "ignore"))
print("Latest commit sha: ", git_commit.sha)
print("PR update time", pr.updated_at)
print("Time UTC:", datetime.utcnow())
if last_commit_date > datetime.utcnow():
print("==== Future commit found ====")
if (not dryRun) and repo_config.ADD_LABELS:
labels = [x.name for x in issue.labels]
if not "future commit" in labels:
labels.append("future commit")
issue.edit(labels=labels)
return
# now get commit statuses
# this is how we figure out the current state of tests
# on the latest commit of the PR.
commit_status = last_commit.get_statuses()
# we can translate git commit status API 'state' strings if needed.
state_labels = {
'error': 'error',
'failure': 'failed',
'success': 'finished',
}
state_labels_colors = {
'error': 'd73a4a',
'failure': 'd2222d',
'pending': 'ffbf00',
'running': 'a4e8f9',
'success': '238823',
'finish': '238823',
'stalled': 'ededed'
}
commit_status_time = {}
for stat in commit_status:
name = test_suites.get_test_name(stat.context)
if name == 'unrecognised':
continue
if name in commit_status_time and commit_status_time[name] > stat.updated_at:
continue
commit_status_time[name] = stat.updated_at
# error, failure, pending, success
test_statuses[name] = stat.state
if stat.state in state_labels:
test_statuses[name] = state_labels[stat.state]
test_status_exists[name] = True
if name in test_triggered and test_triggered[name]: # if already True, don't change it
continue
test_triggered[name] = ('has been triggered' in stat.description) or (stat.state == 'success' or stat.state == 'failure')
# some other labels, gleaned from the description (the status API
# doesn't support these states)
if ('running' in stat.description):
test_statuses[name] = 'running'
# check if we've stalled
if (test_statuses[name] in ['running', 'pending']) and (name in test_triggered) and test_triggered[name]:
if (datetime.utcnow() - stat.updated_at).total_seconds() > test_suites.get_stall_time(name):
test_triggered[name] = False # the test may be triggered again.
test_statuses[name] = 'stalled'
if (stat.context == 'mu2e/buildtest' and stat.description.startswith(':')):
# this is the commit SHA in master that we merged into
# this is important if we want to trigger a validation job
master_commit_sha = stat.description.replace(':','')
# now process PR comments that come after when
# the bot last did something, first figuring out when the bot last commented
pr_author = issue.user.login
comments = issue.get_comments()
for comment in comments:
# loop through once to ascertain when the bot last commented
if comment.user.login == repo_config.CMSBUILD_USER:
if last_time_seen is None or last_time_seen < comment.created_at:
not_seen_yet = False
last_time_seen = comment.created_at
print (comment.user.login, last_time_seen)
print ("Last time seen", last_time_seen)
# we might not have commented, but e.g. changed a label instead...
for event in pr.get_issue_events():
if event.actor.login == repo_config.CMSBUILD_USER and event.event in ['labeled', 'unlabeled']:
if last_time_seen is None or last_time_seen < event.created_at:
last_time_seen = event.created_at
print (last_time_seen, event)
print ("Last time seen", last_time_seen)
# now we process comments
for comment in comments:
# Ignore all messages which are before last commit.
if (comment.created_at < last_commit_date):
continue
# neglect comments we've already responded to
if (comment.created_at < last_time_seen):
print ("IGNORE COMMENT (seen)", comment.user.login, comment.created_at, '<', last_time_seen)
continue
# neglect comments by un-authorised users
if not comment.user.login in authorised_users or comment.user.login == repo_config.CMSBUILD_USER:
print("IGNORE COMMENT (unauthorised or bot user) - %s." % comment.user.login)
continue
# now look for bot triggers
# check if the comment has triggered a test
trigger_search = check_test_cmd_mu2e(comment.body, repo.full_name)
tests_already_triggered = []
if trigger_search is not None:
tests, _ = trigger_search
print ("Triggered! Comment: %s" % comment.body)
print ('current test(s): %r' % tests_to_trigger)
print ('adding these test(s): %r' % tests )
for test in tests:
# check that the test has been triggered on this commit first
if test in test_triggered and test_triggered[test]:
print ("The test has already been triggered for this ref. It will not be triggered again.")
tests_already_triggered.append(test)
continue
else:
test_triggered[test] = False
if not test_triggered[test]: # is the test already running?
# ok - now we can trigger the test
print ("The test has not been triggered yet. It will now be triggered.")
# update the 'state' of this commit
test_statuses[test] = 'pending'
test_triggered[test] = True
# add the test to the queue of tests to trigger
tests_to_trigger.append(test)
# now,
# - trigger tests if indicated (for this specific SHA.)
# - set the current status for this commit SHA
# - apply labels according to the state of the latest commit of the PR
# - make a comment if required
for test, state in test_statuses.items():
labels.append('%s %s' % (test, state))
if test in tests_to_trigger:
print ("TEST WILL NOW BE TRIGGERED: %s" % test)
# trigger the test in jenkins
create_properties_file_for_test(
test,
repo.full_name,
prId,
git_commit.sha,
master_commit_sha
)
if not dryRun:
if test == 'build':
# we need to store somewhere the master commit SHA
# that we merge into for the build test (for validation)
# this is overlapped with the next, more human readable message
last_commit.create_status(
state="pending",
target_url="https://github.com/mu2e/Offline",
description=":%s" % master_commit_sha,
context=test_suites.get_test_alias(test)
)
last_commit.create_status(
state="pending",
target_url="https://github.com/mu2e/Offline",
description="The test has been triggered in Jenkins",
context=test_suites.get_test_alias(test)
)
print ("Git status created for SHA %s test %s - since the test has been triggered." % (git_commit.sha, test))
elif state == 'pending' and test_status_exists[test]:
print ("Git status unchanged for SHA %s test %s - the existing one is up-to-date." % (git_commit.sha, test))
elif state == 'pending' and not test_triggered[test] and not test_status_exists[test]:
print (test_status_exists)
print ("Git status created for SHA %s test %s - since there wasn't one already." % (git_commit.sha, test))
# indicate that the test is pending but
# we're still waiting for someone to trigger the test
if not dryRun:
last_commit.create_status(
state="pending",
target_url="https://github.com/mu2e/Offline",
description="This test has not been triggered yet.",
context=test_suites.get_test_alias(test)
)
# don't do anything else with commit statuses
# the script handler that handles Jenkins job results will update the commits accordingly
# check if labels have changed
labelnames = [x.name for x in issue.labels]
if (set(labelnames) != set(labels)):
if not dryRun:
issue.edit(labels=labels)
print ("Labels have changed to: ", labels)
# check label colours
try:
for label in issue.labels:
if label.color == 'ededed':
# the label color isn't set
for labelcontent, col in state_labels_colors.items():
if labelcontent in label.name:
label.edit(label.name, col)
break
except:
print ("Failed to set label colours!")
# construct a reply if tests have been triggered.
tests_triggered_msg = ''
already_running_msg = ''
commitlink = 'https://github.com/%s/pull/%s/commits/%s' % (repo.full_name, prId, git_commit.sha)
if len(tests_to_trigger) > 0:
if len(tests_already_triggered) > 0:
already_running_msg = '(already triggered: %s)' % ','.join(tests_already_triggered)
tests_triggered_msg = TESTS_TRIGGERED_CONFIRMATION.format(
commit_link=commitlink,
test_list=', '.join(tests_to_trigger),
tests_already_running_msg=already_running_msg
)
# decide if we should issue a comment, and what comment to issue
if not_seen_yet:
print ("First time seeing this PR - send the user a salutation!")
if not dryRun:
issue.create_comment(PR_SALUTATION.format(
pr_author=pr_author,
changed_folders='\n'.join(['- %s' % s for s in modified_top_level_folders]),
tests_required=', '.join(test_requirements),
watchers='',
tests_triggered_msg=tests_triggered_msg
))
elif len(tests_to_trigger) > 0:
# tests were triggered, let people know about it
if not dryRun:
issue.create_comment(tests_triggered_msg)
elif len(tests_to_trigger) == 0 and len(tests_already_triggered) > 0:
if not dryRun:
issue.create_comment(TESTS_ALREADY_TRIGGERED.format(
commit_link=commitlink,
triggered_tests=', '.join(tests_already_triggered))
)
parser.add_option("-u",
"--user",
dest="user",
help="GH API user.",
type=str,
default="")
opts, args = parser.parse_args()
if len(args) != 1:
parser.error("Too many/few arguments")
prId = int(args[0])
ghtoken = ".github-token"
if opts.user: ghtoken = ".github-token-" + opts.user
gh = Github(login_or_token=open(expanduser("~/" + ghtoken)).read().strip())
if not opts.commit: api_rate_limits(gh)
repo = gh.get_repo(opts.repository)
issue = repo.get_issue(prId)
if not issue.pull_request:
print "ERROR: Only cache Pull requests, %s is an issue." % prId
exit(1)
pr = repo.get_pull(prId)
if not pr.merged:
print "ERROR: PR is not merged yet"
exit(1)
data = {}
data['user'] = issue.user.login.encode("ascii", "ignore")
data['title'] = issue.title.encode("ascii", "ignore")
data['body'] = issue.body.encode("ascii", "ignore")
data['branch'] = pr.base.ref.encode("ascii", "ignore")
data['created_at'] = pr.created_at.strftime("%s")