def rescan_issues():
"""
Re-scan all JIRA issues that are in the "Needs Triage" state. If any were
created by edX employees, they will be automatically transitioned to an
"Open" state.
Normally, issues are processed automatically. However, sometimes an issue
is skipped accidentally, either due to a network hiccup, a bug in JIRA,
or this bot going offline. This endpoint is used to clean up after these
kind of problems.
"""
jql = request.form.get("jql") or 'status = "Needs Triage" ORDER BY key'
sentry_extra_context({"jql": jql})
issues = jira_paginated_get(
"/rest/api/2/search",
jql=jql,
obj_name="issues",
session=jira,
)
results = {}
for issue in issues:
issue_key = to_unicode(issue["key"])
results[issue_key] = issue_opened(issue)
resp = make_response(json.dumps(results), 200)
resp.headers["Content-Type"] = "application/json"
return resp
def issue_created():
"""
Received an "issue created" event from JIRA. See `JIRA's webhook docs`_.
.. _JIRA's webhook docs: https://developer.atlassian.com/display/JIRADEV/JIRA+Webhooks+Overview
"""
try:
event = request.get_json()
except ValueError:
raise ValueError(
"Invalid JSON from JIRA: {data}".format(data=request.data))
sentry_extra_context({"event": event})
logger.debug("Jira issue created: {}".format(json.dumps(event)))
if current_app.debug:
print(json.dumps(event), file=sys.stderr)
if "issue" not in event:
# It's rare, but we occasionally see junk data from JIRA. For example,
# here's a real API request we've received on this handler:
# {"baseUrl": "https://openedx.atlassian.net",
# "key": "jira:1fec1026-b232-438f-adab-13b301059297",
# "newVersion": 64005, "oldVersion": 64003}
# If we don't have an "issue" key, it's junk.
return "What is this shit!?", 400
return issue_opened(event["issue"])
def jira_issue_rejected(issue):
issue_key = to_unicode(issue["key"])
pr_num = github_pr_num(issue)
pr_url = github_pr_url(issue)
issue_url = pr_url.replace("pulls", "issues")
gh_issue_resp = github.get(issue_url)
gh_issue_resp.raise_for_status()
gh_issue = gh_issue_resp.json()
sentry_extra_context({"github_issue": gh_issue})
if gh_issue["state"] == "closed":
# nothing to do
msg = "{key} was rejected, but PR #{num} was already closed".format(
key=issue_key, num=pr_num)
print(msg, file=sys.stderr)
return msg
# Comment on the PR to explain to look at JIRA
username = to_unicode(gh_issue["user"]["login"])
comment = {
"body": ("Hello @{username}: We are unable to continue with "
"review of your submission at this time. Please see the "
"associated JIRA ticket for more explanation.".format(
username=username))
}
comment_resp = github.post(issue_url + "/comments", json=comment)
comment_resp.raise_for_status()
# close the pull request on Github
close_resp = github.patch(pr_url, json={"state": "closed"})
close_resp.raise_for_status()
return "Closed PR #{num}".format(num=pr_num)
def rescan_repository(self, repo):
"""
rescans a single repo for new prs
"""
github = github_bp.session
sentry_extra_context({"repo": repo})
url = "/repos/{repo}/pulls".format(repo=repo)
created = {}
if not self.request.called_directly:
self.update_state(state='STARTED', meta={'repo': repo})
def page_callback(response):
if not response.ok or self.request.called_directly:
return
current_url = URLObject(response.url)
current_page = int(current_url.query_dict.get("page", 1))
link_last = response.links.get("last")
if link_last:
last_url = URLObject(link_last['url'])
last_page = int(last_url.query_dict["page"])
else:
last_page = current_page
state_meta = {
"repo": repo,
"current_page": current_page,
"last_page": last_page
}
self.update_state(state='STARTED', meta=state_meta)
for pull_request in paginated_get(url,
session=github,
callback=page_callback):
sentry_extra_context({"pull_request": pull_request})
issue_key = get_jira_issue_key(pull_request)
is_internal = is_internal_pull_request(pull_request)
if not issue_key and not is_internal:
# `pull_request_opened()` is a celery task, but by calling it as
# a function instead of calling `.delay()` on it, we're eagerly
# executing the task now, instead of adding it to the task queue
# so it is executed later. As a result, this will return the values
# that the `pull_request_opened()` function returns, rather than
# return an AsyncResult object.
issue_key, issue_created = pull_request_opened(pull_request) # pylint: disable=no-value-for-parameter
if issue_created:
created[pull_request["number"]] = issue_key
logger.info(
"Created {num} JIRA issues on repo {repo}. PRs are {prs}".format(
num=len(created),
repo=repo,
prs=created.keys(),
), )
info = {"repo": repo}
if created:
info["created"] = created
return info
def rescan_users(domain_groups):
jira = jira_bp.session
failures = defaultdict(dict)
for groupname, domain in domain_groups.items():
users_in_group = jira_group_members(groupname,
session=jira,
debug=True)
usernames_in_group = set(u["name"] for u in users_in_group)
sentry_extra_context({
"groupname": groupname,
"usernames_in_group": usernames_in_group,
})
for user in jira_users(filter=domain, session=jira, debug=True):
if not user["email"].endswith(domain):
pass
username = user["name"]
if username not in usernames_in_group:
# add the user to the group!
resp = jira.post(
"/rest/api/2/group/user?groupname={}".format(groupname),
json={"name": username},
)
if not resp.ok:
# Is this a failure saying that the user is already in
# the group? If so, ignore it.
nothing_to_do_msg = (
"Cannot add user '{username}', "
"user is already a member of '{groupname}'").format(
username=username, groupname=groupname)
error = resp.json()
if error.get("errorMessages", []) == [nothing_to_do_msg]:
continue
# it's some other kind of failure, so log it
failures[groupname][username] = resp.text
if failures:
logger.error(
"Failures in trying to rescan JIRA users: {}".format(failures))
return failures
def issue_updated():
"""
Received an "issue updated" event from JIRA. See `JIRA's webhook docs`_.
.. _JIRA's webhook docs: https://developer.atlassian.com/display/JIRADEV/JIRA+Webhooks+Overview
"""
try:
event = request.get_json()
except ValueError:
raise ValueError(
"Invalid JSON from JIRA: {data}".format(data=request.data))
sentry_extra_context({"event": event})
if current_app.debug:
print(json.dumps(event), file=sys.stderr)
if "issue" not in event:
# It's rare, but we occasionally see junk data from JIRA. For example,
# here's a real API request we've received on this handler:
# {"baseUrl": "https://openedx.atlassian.net",
# "key": "jira:1fec1026-b232-438f-adab-13b301059297",
# "newVersion": 64005, "oldVersion": 64003}
# If we don't have an "issue" key, it's junk.
return "What is this shit!?", 400
# is this a comment?
comment = event.get("comment")
if comment:
return jira_issue_comment_added(event["issue"], comment)
# is the issue an open source pull request?
if event["issue"]["fields"]["project"]["key"] != "OSPR":
return "I don't care"
# is it a pull request against an edX repo?
pr_repo = github_pr_repo(event["issue"])
if pr_repo and not pr_repo.startswith("edx/"):
return "ignoring PR on external repo"
# we don't care about OSPR subtasks
if event["issue"]["fields"]["issuetype"]["subtask"]:
return "ignoring subtasks"
# don't care about feature proposals
if event["issue"]["fields"]["issuetype"]["name"] == "Feature Proposal":
return "ignoring feature propsals"
# is there a changelog?
changelog = event.get("changelog")
if not changelog:
# it was just someone adding a comment
return "I don't care"
# did the issue change status?
status_changelog_items = [
item for item in changelog["items"] if item["field"] == "status"
]
if len(status_changelog_items) == 0:
return "I don't care"
if not pr_repo:
issue_key = to_unicode(event["issue"]["key"])
fail_msg = '{key} is missing "Repo" field'.format(key=issue_key)
fail_msg += ' {0}'.format(event["issue"]["fields"]["issuetype"])
raise Exception(fail_msg)
repo_labels_resp = github.get("/repos/{repo}/labels".format(repo=pr_repo))
repo_labels_resp.raise_for_status()
# map of label name to label URL
repo_labels = {l["name"]: l["url"] for l in repo_labels_resp.json()}
# map of label name lowercased to label name in the case that it is on Github
repo_labels_lower = {name.lower(): name for name in repo_labels}
new_status = status_changelog_items[0]["toString"]
changes = []
if new_status == "Rejected":
change = jira_issue_rejected(event["issue"])
changes.append(change)
elif 'blocked' in new_status.lower():
print("New status is: {}".format(new_status))
print("repo_labels_lower: {}".format(repo_labels_lower))
if new_status.lower() in repo_labels_lower:
change = jira_issue_status_changed(event["issue"], event["changelog"])
changes.append(change)
if changes:
return "\n".join(changes)
else:
return "no change necessary"
def issue_opened(issue):
sentry_extra_context({"issue": issue})
issue_key = to_unicode(issue["key"])
issue_url = URLObject(issue["self"])
transitioned = False
if should_transition(issue):
# In JIRA, a "transition" is how an issue changes from one status
# to another, like going from "Open" to "In Progress". The workflow
# defines what transitions are allowed, and this API will tell us
# what transitions are currently allowed by the workflow.
# Ref: https://docs.atlassian.com/jira/REST/ondemand/#d2e4954
transitions_url = issue_url.with_path(issue_url.path + "/transitions")
transitions_resp = jira_get(transitions_url)
transitions_resp.raise_for_status()
# This transforms the API response into a simple mapping from the
# name of the transition (like "In Progress") to the ID of the transition.
# Note that a transition may not have the same name as the state that it
# goes to, so a transition to go from "Open" to "In Progress" may be
# named something like "Start Work".
transitions = {
t["name"]: t["id"]
for t in transitions_resp.json()["transitions"]
}
# We attempt to transition the issue into the "Open" state for the given project
# (some projects use a different name), so look for a transition with the right name
new_status = None
action = None
for state_name in ["Open", "Design Backlog", "To Do"]:
if state_name in transitions:
new_status = state_name
action = "Transitioned to '{}'".format(state_name)
if not new_status:
# If it's an OSPR subtask (used by teams to manage reviews), transition to team backlog
if to_unicode(
issue["fields"]["project"]
["key"]) == "OSPR" and issue["fields"]["issuetype"]["subtask"]:
new_status = "To Backlog"
action = "Transitioned to 'To Backlog'"
else:
raise ValueError(
"No valid transition! Possibilities are {}".format(
transitions.keys()))
# This creates a new API request to tell JIRA to move the issue from
# one status to another using the specified transition. We have to
# tell JIRA the transition ID, so we use that mapping we set up earlier.
body = {
"transition": {
"id": transitions[new_status],
}
}
transition_resp = jira.post(transitions_url, json=body)
transition_resp.raise_for_status()
transitioned = True
# log to stderr
if transitioned and not action:
action = "Transitioned to Open"
else:
action = "ignored"
print(
"{key} created by {name} ({account}), {action}".format(
key=issue_key,
name=to_unicode(issue["fields"]["creator"]["displayName"]),
account=to_unicode(issue["fields"]["creator"]["accountId"]),
action="Transitioned to Open" if transitioned else "ignored",
),
file=sys.stderr,
)
return action
def pull_request_opened(self,
pull_request,
ignore_internal=True,
check_contractor=True):
"""
Process a pull request. This is called when a pull request is opened, or
when the pull requests of a repo are re-scanned. By default, this function
will ignore internal pull requests (unless a repo has supplied .pr_cover_letter.md.j2),
and will add a comment to pull requests made by contractors (if if has not yet added
a comment). However, this function can be called in such a way that it processes those pull
requests anyway.
This function must be idempotent. Every time the repositories are re-scanned,
this function will be called for pull requests that have already been opened.
As a result, it should not comment on the pull request without checking to
see if it has *already* commented on the pull request.
Returns a 2-tuple. The first element in the tuple is the key of the JIRA
issue associated with the pull request, if any, as a string. The second
element in the tuple is a boolean indicating if this function did any
work, such as making a JIRA issue or commenting on the pull request.
"""
# Environment variable containing the Open edX release name
open_edx_release = os.environ.get('OPENEDX_RELEASE_NAME')
# Environment variable containing a string of comma separated Github usernames for testing
test_open_edx_release = os.environ.get(
'GITHUB_USERS_CHERRY_PICK_MESSAGE_TEST')
#test_open_edx_release = 'mduboseedx,nedbat,fakeuser'
github = github_bp.session
pr = pull_request
user = pr["user"]["login"]
repo = pr["base"]["repo"]["full_name"]
num = pr["number"]
is_internal_pr = is_internal_pull_request(pr)
has_cl = has_internal_cover_letter(pr)
is_beta = is_beta_tester_pull_request(pr)
msg = "Processing {} PR #{} by {}...".format(repo, num, user)
log_info(self.request, msg)
if is_bot_pull_request(pr):
# Bots never need OSPR attention.
return None, False
if is_internal_pr and not has_cl and is_beta:
msg = "Adding cover letter to PR #{num} against {repo}".format(
repo=repo, num=num)
log_info(self.request, msg)
coverletter = github_internal_cover_letter(pr)
if coverletter is not None:
comment = {"body": coverletter}
url = "/repos/{repo}/issues/{num}/comments".format(repo=repo,
num=num)
comment_resp = github.post(url, json=comment)
log_request_response(self.request, comment_resp)
comment_resp.raise_for_status()
if ignore_internal and is_internal_pr:
# not an open source pull request, don't create an issue for it
msg = "@{user} opened PR #{num} against {repo} (internal PR)".format(
user=user, repo=repo, num=num)
log_info(self.request, msg)
# new release candidate for Open edX is available, ask internal PR if should be cherry picked
do_cherry_pick_comment = False
if open_edx_release:
do_cherry_pick_comment = True
release_message = open_edx_release
elif test_open_edx_release:
if user in test_open_edx_release.split(','):
do_cherry_pick_comment = True
release_message = "Test Release"
if do_cherry_pick_comment:
github_post_cherry_pick_comment(self, github, pr, release_message)
return None, True
return None, False
if check_contractor and is_contractor_pull_request(pr):
# have we already left a contractor comment?
if has_contractor_comment(pr):
msg = "Already left contractor comment for PR #{}".format(num)
log_info(self.request, msg)
return None, False
# don't create a JIRA issue, but leave a comment
comment = {
"body": github_contractor_pr_comment(pr),
}
url = "/repos/{repo}/issues/{num}/comments".format(repo=repo, num=num)
msg = "Posting contractor comment to PR #{}".format(num)
log_info(self.request, msg)
comment_resp = github.post(url, json=comment)
log_request_response(self.request, comment_resp)
comment_resp.raise_for_status()
return None, True
issue_key = get_jira_issue_key(pr)
if issue_key:
msg = "Already created {key} for PR #{num} against {repo}".format(
key=issue_key,
num=pr["number"],
repo=pr["base"]["repo"]["full_name"],
)
log_info(self.request, msg)
return issue_key, False
repo = pr["base"]["repo"]["full_name"]
people = get_people_file()
custom_fields = get_jira_custom_fields(jira_bp.session)
user_name = None
if user in people:
user_name = people[user].get("name", "")
if not user_name:
user_resp = github.get(pr["user"]["url"])
if user_resp.ok:
user_name = user_resp.json().get("name", user)
else:
user_name = user
# create an issue on JIRA!
new_issue = {
"fields": {
"project": {
"key": "OSPR",
},
"issuetype": {
"name": "Pull Request Review",
},
"summary": pr["title"],
"description": pr["body"],
"customfield_10904":
pr["html_url"], # "URL" is ambiguous, use the internal name.
custom_fields["PR Number"]: pr["number"],
custom_fields["Repo"]: pr["base"]["repo"]["full_name"],
custom_fields["Contributor Name"]: user_name,
}
}
institution = people.get(user, {}).get("institution", None)
if institution:
new_issue["fields"][custom_fields["Customer"]] = [institution]
sentry_extra_context({"new_issue": new_issue})
log_info(self.request, 'Creating new JIRA issue...')
resp = jira_bp.session.post("/rest/api/2/issue", json=new_issue)
log_request_response(self.request, resp)
resp.raise_for_status()
new_issue_body = resp.json()
issue_key = new_issue_body["key"]
new_issue["key"] = issue_key
sentry_extra_context({"new_issue": new_issue})
# add a comment to the Github pull request with a link to the JIRA issue
comment = {
"body": github_community_pr_comment(pr, new_issue_body, people),
}
url = "/repos/{repo}/issues/{num}/comments".format(repo=repo,
num=pr["number"])
log_info(self.request, 'Creating new GitHub comment with JIRA issue...')
comment_resp = github.post(url, json=comment)
log_request_response(self.request, comment_resp)
comment_resp.raise_for_status()
# Add the "Needs Triage" label to the PR
issue_url = "/repos/{repo}/issues/{num}".format(repo=repo,
num=pr["number"])
labels = {'labels': ['needs triage', 'open-source-contribution']}
log_info(self.request, 'Updating GitHub labels...')
label_resp = github.patch(issue_url, data=json.dumps(labels))
log_request_response(self.request, label_resp)
label_resp.raise_for_status()
msg = "@{user} opened PR #{num} against {repo}, created {issue} to track it".format(
user=user,
repo=repo,
num=pr["number"],
issue=issue_key,
)
log_info(self.request, msg)
return issue_key, True
def pull_request_closed(self, pull_request):
"""
A GitHub pull request has been merged or closed. Synchronize the JIRA issue
to also be in the "merged" or "closed" state. Returns a boolean: True
if the JIRA issue was correctly synchronized, False otherwise. (However,
these booleans are ignored.)
"""
jira = jira_bp.session
pr = pull_request
repo = pr["base"]["repo"]["full_name"]
merged = pr["merged"]
issue_key = get_jira_issue_key(pr)
if not issue_key:
msg = "Couldn't find JIRA issue for PR #{num} against {repo}".format(
num=pr["number"],
repo=repo,
)
log_info(self.request, msg)
return "no JIRA issue :("
sentry_extra_context({"jira_key": issue_key})
# close the issue on JIRA
transition_url = ("/rest/api/2/issue/{key}/transitions"
"?expand=transitions.fields".format(key=issue_key))
log_info(self.request, 'Closing the issue on JIRA...')
transitions_resp = jira.get(transition_url)
log_request_response(self.request, transitions_resp)
if transitions_resp.status_code == requests.codes.not_found:
# JIRA issue has been deleted
return False
transitions_resp.raise_for_status()
transitions = transitions_resp.json()["transitions"]
sentry_extra_context({"transitions": transitions})
transition_name = "Merged" if merged else "Rejected"
transition_id = None
for t in transitions:
if t["to"]["name"] == transition_name:
transition_id = t["id"]
break
if not transition_id:
# maybe the issue is *already* in the right status?
issue_url = "/rest/api/2/issue/{key}".format(key=issue_key)
issue_resp = jira.get(issue_url)
issue_resp.raise_for_status()
issue = issue_resp.json()
sentry_extra_context({"jira_issue": issue})
current_status = issue["fields"]["status"]["name"]
if current_status == transition_name:
msg = "{key} is already in status {status}".format(
key=issue_key, status=transition_name)
log_info(self.request, msg)
return False
# nope, raise an error message
fail_msg = (
"{key} cannot be transitioned directly from status {curr_status} "
"to status {new_status}. Valid status transitions are: {valid}".
format(
key=issue_key,
new_status=transition_name,
curr_status=current_status,
valid=", ".join(t["to"]["name"] for t in transitions),
))
log_error(self.request, fail_msg)
raise Exception(fail_msg)
log_info(self.request, 'Changing JIRA issue status...')
transition_resp = jira.post(transition_url,
json={"transition": {
"id": transition_id,
}})
log_request_response(self.request, transition_resp)
transition_resp.raise_for_status()
msg = "PR #{num} against {repo} was {action}, moving {issue} to status {status}".format(
num=pr["number"],
repo=repo,
action="merged" if merged else "closed",
issue=issue_key,
status="Merged" if merged else "Rejected",
)
log_info(self.request, msg)
return True