def github_community_pr_comment(pull_request, jira_issue, people=None):
"""
For a newly-created pull request from an open source contributor,
write a welcoming comment on the pull request. The comment should:
* contain a link to the JIRA issue
* check for contributor agreement
* contain a link to our process documentation
"""
people = people or get_people_file()
people = {user.lower(): values for user, values in people.items()}
pr_author = pull_request["user"]["login"].lower()
created_at = parse_date(pull_request["created_at"]).replace(tzinfo=None)
# does the user have a valid, signed contributor agreement?
has_signed_agreement = (
pr_author in people
and people[pr_author].get("expires_on", date.max) > created_at.date())
return render_template(
"github_community_pr_comment.md.j2",
user=pull_request["user"]["login"],
repo=pull_request["base"]["repo"]["full_name"],
number=pull_request["number"],
issue_key=jira_issue["key"],
has_signed_agreement=has_signed_agreement,
)
def check_contributors():
"""
Identify missing contributors: people who have commits in a repository,
but who are not listed in the AUTHORS file.
"""
repo = request.form.get("repo", "")
if repo:
repos = (repo, )
else:
repos = get_repos_file().keys()
people = get_people_file()
people_lower = {username.lower() for username in people.keys()}
missing_contributors = defaultdict(set)
for repo in repos:
sentry.client.extra_context({"repo": repo})
contributors_url = "/repos/{repo}/contributors".format(repo=repo)
contributors = paginated_get(contributors_url, session=github)
for contributor in contributors:
if contributor["login"].lower() not in people_lower:
missing_contributors[repo].add(contributor["login"])
# convert sets to lists, so jsonify can handle them
output = {
repo: list(contributors)
for repo, contributors in missing_contributors.items()
}
return jsonify(output)
def github_check_contributors():
if request.method == "GET":
return render_template("github_check_contributors.html")
repo = request.form.get("repo", "")
if repo:
repos = (repo,)
else:
repos = get_repos_file().keys()
people = get_people_file()
people_lower = {username.lower() for username in people.keys()}
missing_contributors = defaultdict(set)
for repo in repos:
bugsnag_context = {"repo": repo}
bugsnag.configure_request(meta_data=bugsnag_context)
contributors_url = "/repos/{repo}/contributors".format(repo=repo)
contributors = paginated_get(contributors_url, session=github)
for contributor in contributors:
if contributor["login"].lower() not in people_lower:
missing_contributors[repo].add(contributor["login"])
# convert sets to lists, so jsonify can handle them
output = {
repo: list(contributors)
for repo, contributors in missing_contributors.items()
}
return jsonify(output)
def check_contributors():
"""
Identify missing contributors: people who have commits in a repository,
but who are not listed in the AUTHORS file.
"""
repo = request.form.get("repo", "")
if repo:
repos = (repo,)
else:
repos = get_repos_file().keys()
people = get_people_file()
people_lower = {username.lower() for username in people.keys()}
missing_contributors = defaultdict(set)
for repo in repos:
sentry.client.extra_context({"repo": repo})
contributors_url = "/repos/{repo}/contributors".format(repo=repo)
contributors = paginated_get(contributors_url, session=github)
for contributor in contributors:
if contributor["login"].lower() not in people_lower:
missing_contributors[repo].add(contributor["login"])
# convert sets to lists, so jsonify can handle them
output = {
repo: list(contributors)
for repo, contributors in missing_contributors.items()
}
return jsonify(output)
def github_community_pr_comment(pull_request, jira_issue, people=None):
"""
For a newly-created pull request from an open source contributor,
write a welcoming comment on the pull request. The comment should:
* contain a link to the JIRA issue
* check for contributor agreement
* check for AUTHORS entry
* contain a link to our process documentation
"""
github = github_bp.session
people = people or get_people_file()
people = {user.lower(): values for user, values in people.items()}
pr_author = pull_request["user"]["login"].decode('utf-8').lower()
created_at = parse_date(pull_request["created_at"]).replace(tzinfo=None)
# does the user have a valid, signed contributor agreement?
has_signed_agreement = (
pr_author in people and
people[pr_author].get("expires_on", date.max) > created_at.date()
)
# is the user in the AUTHORS file?
in_authors_file = False
name = people.get(pr_author, {}).get("name", "")
if name:
authors_url = "https://raw.githubusercontent.com/{repo}/{branch}/AUTHORS".format(
repo=pull_request["head"]["repo"]["full_name"].decode('utf-8'),
branch=pull_request["head"]["ref"].decode('utf-8'),
)
authors_resp = github.get(authors_url)
if authors_resp.ok:
authors_content = authors_resp.text
if name in authors_content:
in_authors_file = True
return render_template("github_community_pr_comment.md.j2",
user=pull_request["user"]["login"].decode('utf-8'),
repo=pull_request["base"]["repo"]["full_name"].decode('utf-8'),
number=pull_request["number"],
issue_key=jira_issue["key"].decode('utf-8'),
has_signed_agreement=has_signed_agreement,
in_authors_file=in_authors_file,
)
def test_updated_person():
people = get_people_file()
created_at = datetime(2014, 1, 1)
updated_person = get_person_certain_time(people["raisingarizona"], created_at)
assert updated_person["agreement"] == "individual"
def test_updated_person_has_institution():
people = get_people_file()
created_at = datetime(2014, 1, 1)
updated_person = get_person_certain_time(people["jarv"], created_at)
assert updated_person["institution"] == "edX"
def test_current_person():
people = get_people_file()
created_at = datetime.today()
current_person = get_person_certain_time(people["raisingarizona"], created_at)
assert current_person["agreement"] == "none"
def test_current_person_no_institution():
people = get_people_file()
created_at = datetime.today()
current_person = get_person_certain_time(people["jarv"], created_at)
assert "institution" not in current_person
assert current_person["agreement"] == "individual"
def pull_request_opened(self,
pull_request,
ignore_internal=True,
check_contractor=True):
"""
Process a pull request. This is called when a pull request is opened, or
when the pull requests of a repo are re-scanned. By default, this function
will ignore internal pull requests (unless a repo has supplied .pr_cover_letter.md.j2),
and will add a comment to pull requests made by contractors (if if has not yet added
a comment). However, this function can be called in such a way that it processes those pull
requests anyway.
This function must be idempotent. Every time the repositories are re-scanned,
this function will be called for pull requests that have already been opened.
As a result, it should not comment on the pull request without checking to
see if it has *already* commented on the pull request.
Returns a 2-tuple. The first element in the tuple is the key of the JIRA
issue associated with the pull request, if any, as a string. The second
element in the tuple is a boolean indicating if this function did any
work, such as making a JIRA issue or commenting on the pull request.
"""
# Environment variable containing the Open edX release name
open_edx_release = os.environ.get('OPENEDX_RELEASE_NAME')
# Environment variable containing a string of comma separated Github usernames for testing
test_open_edx_release = os.environ.get(
'GITHUB_USERS_CHERRY_PICK_MESSAGE_TEST')
#test_open_edx_release = 'mduboseedx,nedbat,fakeuser'
github = github_bp.session
pr = pull_request
user = pr["user"]["login"]
repo = pr["base"]["repo"]["full_name"]
num = pr["number"]
is_internal_pr = is_internal_pull_request(pr)
has_cl = has_internal_cover_letter(pr)
is_beta = is_beta_tester_pull_request(pr)
msg = "Processing {} PR #{} by {}...".format(repo, num, user)
log_info(self.request, msg)
if is_bot_pull_request(pr):
# Bots never need OSPR attention.
return None, False
if is_internal_pr and not has_cl and is_beta:
msg = "Adding cover letter to PR #{num} against {repo}".format(
repo=repo, num=num)
log_info(self.request, msg)
coverletter = github_internal_cover_letter(pr)
if coverletter is not None:
comment = {"body": coverletter}
url = "/repos/{repo}/issues/{num}/comments".format(repo=repo,
num=num)
comment_resp = github.post(url, json=comment)
log_request_response(self.request, comment_resp)
comment_resp.raise_for_status()
if ignore_internal and is_internal_pr:
# not an open source pull request, don't create an issue for it
msg = "@{user} opened PR #{num} against {repo} (internal PR)".format(
user=user, repo=repo, num=num)
log_info(self.request, msg)
# new release candidate for Open edX is available, ask internal PR if should be cherry picked
do_cherry_pick_comment = False
if open_edx_release:
do_cherry_pick_comment = True
release_message = open_edx_release
elif test_open_edx_release:
if user in test_open_edx_release.split(','):
do_cherry_pick_comment = True
release_message = "Test Release"
if do_cherry_pick_comment:
github_post_cherry_pick_comment(self, github, pr, release_message)
return None, True
return None, False
if check_contractor and is_contractor_pull_request(pr):
# have we already left a contractor comment?
if has_contractor_comment(pr):
msg = "Already left contractor comment for PR #{}".format(num)
log_info(self.request, msg)
return None, False
# don't create a JIRA issue, but leave a comment
comment = {
"body": github_contractor_pr_comment(pr),
}
url = "/repos/{repo}/issues/{num}/comments".format(repo=repo, num=num)
msg = "Posting contractor comment to PR #{}".format(num)
log_info(self.request, msg)
comment_resp = github.post(url, json=comment)
log_request_response(self.request, comment_resp)
comment_resp.raise_for_status()
return None, True
issue_key = get_jira_issue_key(pr)
if issue_key:
msg = "Already created {key} for PR #{num} against {repo}".format(
key=issue_key,
num=pr["number"],
repo=pr["base"]["repo"]["full_name"],
)
log_info(self.request, msg)
return issue_key, False
repo = pr["base"]["repo"]["full_name"]
people = get_people_file()
custom_fields = get_jira_custom_fields(jira_bp.session)
user_name = None
if user in people:
user_name = people[user].get("name", "")
if not user_name:
user_resp = github.get(pr["user"]["url"])
if user_resp.ok:
user_name = user_resp.json().get("name", user)
else:
user_name = user
# create an issue on JIRA!
new_issue = {
"fields": {
"project": {
"key": "OSPR",
},
"issuetype": {
"name": "Pull Request Review",
},
"summary": pr["title"],
"description": pr["body"],
"customfield_10904":
pr["html_url"], # "URL" is ambiguous, use the internal name.
custom_fields["PR Number"]: pr["number"],
custom_fields["Repo"]: pr["base"]["repo"]["full_name"],
custom_fields["Contributor Name"]: user_name,
}
}
institution = people.get(user, {}).get("institution", None)
if institution:
new_issue["fields"][custom_fields["Customer"]] = [institution]
sentry_extra_context({"new_issue": new_issue})
log_info(self.request, 'Creating new JIRA issue...')
resp = jira_bp.session.post("/rest/api/2/issue", json=new_issue)
log_request_response(self.request, resp)
resp.raise_for_status()
new_issue_body = resp.json()
issue_key = new_issue_body["key"]
new_issue["key"] = issue_key
sentry_extra_context({"new_issue": new_issue})
# add a comment to the Github pull request with a link to the JIRA issue
comment = {
"body": github_community_pr_comment(pr, new_issue_body, people),
}
url = "/repos/{repo}/issues/{num}/comments".format(repo=repo,
num=pr["number"])
log_info(self.request, 'Creating new GitHub comment with JIRA issue...')
comment_resp = github.post(url, json=comment)
log_request_response(self.request, comment_resp)
comment_resp.raise_for_status()
# Add the "Needs Triage" label to the PR
issue_url = "/repos/{repo}/issues/{num}".format(repo=repo,
num=pr["number"])
labels = {'labels': ['needs triage', 'open-source-contribution']}
log_info(self.request, 'Updating GitHub labels...')
label_resp = github.patch(issue_url, data=json.dumps(labels))
log_request_response(self.request, label_resp)
label_resp.raise_for_status()
msg = "@{user} opened PR #{num} against {repo}, created {issue} to track it".format(
user=user,
repo=repo,
num=pr["number"],
issue=issue_key,
)
log_info(self.request, msg)
return issue_key, True
def github_community_pr_comment(pull_request, jira_issue, people=None):
"""
For a newly-created pull request from an open source contributor,
write a welcoming comment on the pull request. The comment should:
* contain a link to the JIRA issue
* check for contributor agreement
* check for AUTHORS entry
* contain a link to our process documentation
"""
people = people or get_people_file()
people = {user.lower(): values for user, values in people.items()}
pr_author = pull_request["user"]["login"].decode('utf-8').lower()
created_at = parse_date(pull_request["created_at"]).replace(tzinfo=None)
# does the user have a valid, signed contributor agreement?
has_signed_agreement = (
pr_author in people and
people[pr_author].get("expires_on", date.max) > created_at.date()
)
# is the user in the AUTHORS file?
in_authors_file = False
name = people.get(pr_author, {}).get("name", "")
if name:
authors_url = "https://raw.githubusercontent.com/{repo}/{branch}/AUTHORS".format(
repo=pull_request["head"]["repo"]["full_name"].decode('utf-8'),
branch=pull_request["head"]["ref"].decode('utf-8'),
)
authors_resp = github.get(authors_url)
if authors_resp.ok:
authors_content = authors_resp.text
if name in authors_content:
in_authors_file = True
doc_url = "http://edx-developer-guide.readthedocs.org/en/latest/process/overview.html"
issue_key = jira_issue["key"].decode('utf-8')
issue_url = "https://openedx.atlassian.net/browse/{key}".format(key=issue_key)
contributing_url = "https://github.com/edx/edx-platform/blob/master/CONTRIBUTING.rst"
agreement_url = "http://open.edx.org/sites/default/files/wysiwyg/individual-contributor-agreement.pdf"
authors_url = "https://github.com/{repo}/blob/master/AUTHORS".format(
repo=pull_request["base"]["repo"]["full_name"].decode('utf-8'),
)
comment = (
"Thanks for the pull request, @{user}! I've created "
"[{issue_key}]({issue_url}) to keep track of it in JIRA. "
"JIRA is a place for product owners to prioritize feature reviews "
"by the engineering development teams. "
"\n\nFeel free to add as much of the following information to the ticket:"
"\n- supporting documentation"
"\n- edx-code email threads"
"\n- timeline information ('this must be merged by XX date', and why that is)"
"\n- partner information ('this is a course on edx.org')"
"\n- any other information that can help Product understand the context for the PR"
"\n\nAll technical communication about the code itself will still be "
"done via the Github pull request interface. "
"As a reminder, [our process documentation is here]({doc_url})."
).format(
user=pull_request["user"]["login"].decode('utf-8'),
issue_key=issue_key,
issue_url=issue_url,
doc_url=doc_url,
)
if not has_signed_agreement or not in_authors_file:
todo = ""
if not has_signed_agreement:
todo += (
"submitted a [signed contributor agreement]({agreement_url}) "
"or indicated your institutional affiliation"
).format(
agreement_url=agreement_url,
)
if not has_signed_agreement and not in_authors_file:
todo += " and "
if not in_authors_file:
todo += "added yourself to the [AUTHORS]({authors_url}) file".format(
authors_url=authors_url,
)
comment += ("\n\n"
"We can't start reviewing your pull request until you've {todo}. "
"Please see the [CONTRIBUTING]({contributing_url}) file for "
"more information."
).format(todo=todo, contributing_url=contributing_url)
return comment
def pr_opened(pr, ignore_internal=True, check_contractor=True, bugsnag_context=None):
bugsnag_context = bugsnag_context or {}
user = pr["user"]["login"].decode('utf-8')
repo = pr["base"]["repo"]["full_name"]
num = pr["number"]
if ignore_internal and is_internal_pull_request(pr):
# not an open source pull request, don't create an issue for it
print(
"@{user} opened PR #{num} against {repo} (internal PR)".format(
user=user, repo=repo, num=num,
),
file=sys.stderr
)
return "internal pull request"
if check_contractor and is_contractor_pull_request(pr):
# don't create a JIRA issue, but leave a comment
comment = {
"body": github_contractor_pr_comment(pr),
}
url = "/repos/{repo}/issues/{num}/comments".format(
repo=repo, num=num,
)
comment_resp = github.post(url, json=comment)
comment_resp.raise_for_status()
return "contractor pull request"
issue_key = get_jira_issue_key(pr)
if issue_key:
msg = "Already created {key} for PR #{num} against {repo}".format(
key=issue_key,
num=pr["number"],
repo=pr["base"]["repo"]["full_name"],
)
print(msg, file=sys.stderr)
return msg
repo = pr["base"]["repo"]["full_name"].decode('utf-8')
people = get_people_file()
custom_fields = get_jira_custom_fields()
if user in people:
user_name = people[user].get("name", "")
else:
user_resp = github.get(pr["user"]["url"])
if user_resp.ok:
user_name = user_resp.json().get("name", user)
else:
user_name = user
# create an issue on JIRA!
new_issue = {
"fields": {
"project": {
"key": "OSPR",
},
"issuetype": {
"name": "Pull Request Review",
},
"summary": pr["title"],
"description": pr["body"],
custom_fields["URL"]: pr["html_url"],
custom_fields["PR Number"]: pr["number"],
custom_fields["Repo"]: pr["base"]["repo"]["full_name"],
custom_fields["Contributor Name"]: user_name,
}
}
institution = people.get(user, {}).get("institution", None)
if institution:
new_issue["fields"][custom_fields["Customer"]] = [institution]
bugsnag_context["new_issue"] = new_issue
bugsnag.configure_request(meta_data=bugsnag_context)
resp = jira.post("/rest/api/2/issue", json=new_issue)
resp.raise_for_status()
new_issue_body = resp.json()
issue_key = new_issue_body["key"].decode('utf-8')
bugsnag_context["new_issue"]["key"] = issue_key
bugsnag.configure_request(meta_data=bugsnag_context)
# add a comment to the Github pull request with a link to the JIRA issue
comment = {
"body": github_community_pr_comment(pr, new_issue_body, people),
}
url = "/repos/{repo}/issues/{num}/comments".format(
repo=repo, num=pr["number"],
)
comment_resp = github.post(url, json=comment)
comment_resp.raise_for_status()
# Add the "Needs Triage" label to the PR
issue_url = "/repos/{repo}/issues/{num}".format(repo=repo, num=pr["number"])
label_resp = github.patch(issue_url, data=json.dumps({"labels": ["needs triage", "open-source-contribution"]}))
label_resp.raise_for_status()
print(
"@{user} opened PR #{num} against {repo}, created {issue} to track it".format(
user=user, repo=repo,
num=pr["number"], issue=issue_key,
),
file=sys.stderr
)
return "created {key}".format(key=issue_key)
def pull_request_opened(pull_request, ignore_internal=True, check_contractor=True):
"""
Process a pull request. This is called when a pull request is opened, or
when the pull requests of a repo are re-scanned. By default, this function
will ignore internal pull requests, and will add a comment to pull requests
made by contractors (if if has not yet added a comment). However,
this function can be called in such a way that it processes those pull
requests anyway.
This function must be idempotent. Every time the repositories are re-scanned,
this function will be called for pull requests that have already been opened.
As a result, it should not comment on the pull request without checking to
see if it has *already* commented on the pull request.
Returns a 2-tuple. The first element in the tuple is the key of the JIRA
issue associated with the pull request, if any, as a string. The second
element in the tuple is a boolean indicating if this function did any
work, such as making a JIRA issue or commenting on the pull request.
"""
github = github_bp.session
pr = pull_request
user = pr["user"]["login"].decode('utf-8')
repo = pr["base"]["repo"]["full_name"]
num = pr["number"]
is_internal_pr = is_internal_pull_request(pr)
has_cl = has_internal_cover_letter(pr)
is_beta = is_beta_tester_pull_request(pr)
if is_internal_pr and not has_cl and is_beta:
logger.info(
"Adding cover letter template to PR #{num} against {repo}".format(
repo=repo, num=num,
),
)
comment = {
"body": github_internal_cover_letter(pr),
}
url = "/repos/{repo}/issues/{num}/comments".format(
repo=repo, num=num,
)
comment_resp = github.post(url, json=comment)
comment_resp.raise_for_status()
if ignore_internal and is_internal_pr:
# not an open source pull request, don't create an issue for it
logger.info(
"@{user} opened PR #{num} against {repo} (internal PR)".format(
user=user, repo=repo, num=num,
),
)
return None, False
if check_contractor and is_contractor_pull_request(pr):
# have we already left a contractor comment?
if has_contractor_comment(pr):
return None, False
# don't create a JIRA issue, but leave a comment
comment = {
"body": github_contractor_pr_comment(pr),
}
url = "/repos/{repo}/issues/{num}/comments".format(
repo=repo, num=num,
)
comment_resp = github.post(url, json=comment)
comment_resp.raise_for_status()
return None, True
issue_key = get_jira_issue_key(pr)
if issue_key:
msg = "Already created {key} for PR #{num} against {repo}".format(
key=issue_key,
num=pr["number"],
repo=pr["base"]["repo"]["full_name"],
)
logger.info(msg)
return issue_key, False
repo = pr["base"]["repo"]["full_name"].decode('utf-8')
people = get_people_file()
custom_fields = get_jira_custom_fields(jira_bp.session)
user_name = None
if user in people:
user_name = people[user].get("name", "")
if not user_name:
user_resp = github.get(pr["user"]["url"])
if user_resp.ok:
user_name = user_resp.json().get("name", user)
else:
user_name = user
# create an issue on JIRA!
new_issue = {
"fields": {
"project": {
"key": "OSPR",
},
"issuetype": {
"name": "Pull Request Review",
},
"summary": pr["title"],
"description": pr["body"],
custom_fields["URL"]: pr["html_url"],
custom_fields["PR Number"]: pr["number"],
custom_fields["Repo"]: pr["base"]["repo"]["full_name"],
custom_fields["Contributor Name"]: user_name,
}
}
institution = people.get(user, {}).get("institution", None)
if institution:
new_issue["fields"][custom_fields["Customer"]] = [institution]
sentry.client.extra_context({"new_issue": new_issue})
resp = jira_bp.session.post("/rest/api/2/issue", json=new_issue)
resp.raise_for_status()
new_issue_body = resp.json()
issue_key = new_issue_body["key"].decode('utf-8')
new_issue["key"] = issue_key
sentry.client.extra_context({"new_issue": new_issue})
# add a comment to the Github pull request with a link to the JIRA issue
comment = {
"body": github_community_pr_comment(pr, new_issue_body, people),
}
url = "/repos/{repo}/issues/{num}/comments".format(
repo=repo, num=pr["number"],
)
comment_resp = github.post(url, json=comment)
comment_resp.raise_for_status()
# Add the "Needs Triage" label to the PR
issue_url = "/repos/{repo}/issues/{num}".format(repo=repo, num=pr["number"])
label_resp = github.patch(issue_url, data=json.dumps({"labels": ["needs triage", "open-source-contribution"]}))
label_resp.raise_for_status()
logger.info(
"@{user} opened PR #{num} against {repo}, created {issue} to track it".format(
user=user, repo=repo,
num=pr["number"], issue=issue_key,
),
)
return issue_key, True