def jira_issue_created():
"""
Received an "issue created" event from JIRA. See `JIRA's webhook docs`_.
Ideally, this should be handled in a task queue, but we want to stay within
Heroku's free plan, so it will be handled inline instead.
(A worker dyno costs money.)
.. _JIRA's webhook docs: https://developer.atlassian.com/display/JIRADEV/JIRA+Webhooks+Overview
"""
try:
event = request.get_json()
except ValueError:
raise ValueError("Invalid JSON from JIRA: {data}".format(
data=request.data.decode('utf-8')
))
bugsnag_context = {"event": event}
bugsnag.configure_request(meta_data=bugsnag_context)
if app.debug:
print(json.dumps(event), file=sys.stderr)
if "issue" not in event:
# It's rare, but we occasionally see junk data from JIRA. For example,
# here's a real API request we've received on this handler:
# {"baseUrl": "https://openedx.atlassian.net",
# "key": "jira:1fec1026-b232-438f-adab-13b301059297",
# "newVersion": 64005, "oldVersion": 64003}
# If we don't have an "issue" key, it's junk.
return "What is this shit!?", 400
return issue_opened(event["issue"], bugsnag_context)
def milestone(owner, repo, number):
"""
Load a single milestone from Github into WebhookDB.
:query inline: process the request inline instead of creating a task
on the task queue. Defaults to ``false``.
:statuscode 200: milestone successfully loaded inline
:statuscode 202: task successfully queued
:statuscode 404: specified milestone was not found on Github
"""
inline = bool(request.args.get("inline", False))
children = bool(request.args.get("children", False))
bugsnag_ctx = {"owner": owner, "repo": repo, "number": number, "inline": inline}
bugsnag.configure_request(meta_data=bugsnag_ctx)
if inline and not children:
try:
sync_milestone(
owner, repo, number, children=children,
requestor_id=current_user.get_id(),
)
except NotFound as exc:
return jsonify({"message": exc.message}), 404
else:
return jsonify({"message": "success"})
else:
result = sync_milestone.delay(
owner, repo, number, children=children,
requestor_id=current_user.get_id(),
)
resp = jsonify({"message": "queued"})
resp.status_code = 202
resp.headers["Location"] = url_for("tasks.status", task_id=result.id)
return resp
def user_repositories(username):
"""
Queue tasks to load all of the given user's repositories into WebhookDB.
:query children: scan all children objects. Defaults to false
:query type: one of ``all``, ``owner``, ``member``. Default: ``owner``.
This parameter is proxied to the `Github API for listing user repositories`_.
:statuscode 202: task successfully queued
.. _Github API for listing user repositories: https://developer.github.com/v3/repos/#list-user-repositories
"""
bugsnag_ctx = {"username": username}
bugsnag.configure_request(meta_data=bugsnag_ctx)
children = bool(request.args.get("children", False))
type = request.args.get("type", "owner")
user = User.get(username)
if not user:
# queue a task to load the user
sync_user.delay(username)
result = spawn_page_tasks_for_user_repositories.delay(
username,
type=type,
children=children,
requestor_id=current_user.get_id(),
)
resp = jsonify({"message": "queued"})
resp.status_code = 202
resp.headers["Location"] = url_for("tasks.status", task_id=result.id)
return resp
def github_check_contributors():
if request.method == "GET":
return render_template("github_check_contributors.html")
repo = request.form.get("repo", "")
if repo:
repos = (repo,)
else:
repos = get_repos_file().keys()
people = get_people_file()
people_lower = {username.lower() for username in people.keys()}
missing_contributors = defaultdict(set)
for repo in repos:
bugsnag_context = {"repo": repo}
bugsnag.configure_request(meta_data=bugsnag_context)
contributors_url = "/repos/{repo}/contributors".format(repo=repo)
contributors = paginated_get(contributors_url, session=github)
for contributor in contributors:
if contributor["login"].lower() not in people_lower:
missing_contributors[repo].add(contributor["login"])
# convert sets to lists, so jsonify can handle them
output = {
repo: list(contributors)
for repo, contributors in missing_contributors.items()
}
return jsonify(output)
def github_rescan():
"""
Used to pick up PRs that might not have tickets associated with them.
"""
if request.method == "GET":
# just render the form
return render_template("github_rescan.html")
repo = request.form.get("repo") or "edx/edx-platform"
bugsnag_context = {"repo": repo}
bugsnag.configure_request(meta_data=bugsnag_context)
url = "/repos/{repo}/pulls".format(repo=repo)
created = {}
for pull_request in paginated_get(url, session=github):
bugsnag_context["pull_request"] = pull_request
bugsnag.configure_request(meta_data=bugsnag_context)
if not get_jira_issue_key(
pull_request) and not is_internal_pull_request(pull_request):
text = pr_opened(pull_request, bugsnag_context=bugsnag_context)
if "created" in text:
jira_key = text[8:]
created[pull_request["number"]] = jira_key
print("Created {num} JIRA issues. PRs are {prs}".format(
num=len(created),
prs=created.keys(),
),
file=sys.stderr)
resp = make_response(json.dumps(created), 200)
resp.headers["Content-Type"] = "application/json"
return resp
def issues(owner, repo):
"""
Queue tasks to load all issues on a single Github repository
into WebhookDB.
:query children: scan all children objects. Defaults to ``false``
:query state: one of ``all``, ``open``, or ``closed``. This parameter
is proxied to the `Github API for listing issues`_.
:statuscode 202: task successfully queued
.. _Github API for listing issues: https://developer.github.com/v3/issues/#list-issues-for-a-repository
"""
bugsnag_ctx = {"owner": owner, "repo": repo}
bugsnag.configure_request(meta_data=bugsnag_ctx)
state = request.args.get("state", "open")
children = bool(request.args.get("children", False))
result = spawn_page_tasks_for_issues.delay(
owner,
repo,
state,
children=children,
requestor_id=current_user.get_id(),
)
resp = jsonify({"message": "queued"})
resp.status_code = 202
resp.headers["Location"] = url_for("tasks.status", task_id=result.id)
return resp
def rescan_repo(repo):
"""
rescans a single repo for new prs
"""
bugsnag_context = {"repo": repo}
bugsnag.configure_request(meta_data=bugsnag_context)
url = "/repos/{repo}/pulls".format(repo=repo)
created = {}
for pull_request in paginated_get(url, session=github):
bugsnag_context["pull_request"] = pull_request
bugsnag.configure_request(meta_data=bugsnag_context)
if not get_jira_issue_key(pull_request) and not is_internal_pull_request(pull_request):
text = pr_opened(pull_request, bugsnag_context=bugsnag_context)
if "created" in text:
jira_key = text[8:]
created[pull_request["number"]] = jira_key
print(
"Created {num} JIRA issues on repo {repo}. PRs are {prs}".format(
num=len(created), repo=repo, prs=created.keys(),
),
file=sys.stderr
)
return created
def github_pull_request():
try:
event = request.get_json()
except ValueError:
raise ValueError("Invalid JSON from Github: {data}".format(data=request.data))
bugsnag_context = {"event": event}
bugsnag.configure_request(meta_data=bugsnag_context)
if "pull_request" not in event and "hook" in event and "zen" in event:
# this is a ping
repo = event.get("repository", {}).get("full_name")
print("ping from {repo}".format(repo=repo), file=sys.stderr)
return "PONG"
pr = event["pull_request"]
repo = pr["base"]["repo"]["full_name"]
if event["action"] == "opened":
return pr_opened(pr, bugsnag_context)
if event["action"] == "closed":
return pr_closed(pr, bugsnag_context)
print(
"Received {action} event on PR #{num} against {repo}, don't know how to handle it".format(
action=event["action"], repo=pr["base"]["repo"]["full_name"],
num=pr["number"]
),
file=sys.stderr
)
return "Don't know how to handle this.", 400
def pull_request():
"""
Webhook endpoint for ``pull_request`` events on Github.
"""
payload = request.get_json()
bugsnag.configure_request(meta_data={"payload": payload})
pr_data = payload.get("pull_request")
if not pr_data:
resp = jsonify({"error": "no pull_request in payload"})
resp.status_code = 400
return resp
try:
pr = process_pull_request(pr_data)
except MissingData as err:
return jsonify({"error": err.message, "obj": err.obj}), 400
except StaleData:
return jsonify({"message": "stale data"})
# Fetch the pull request files, too!
if pr.changed_files < 100:
# If there are fewer than 100, do it inline
PullRequestFile.query.filter_by(pull_request_id=pr.id).delete()
sync_page_of_pull_request_files(
owner=pr.base_repo.owner_login, repo=pr.base_repo.name,
number=pr.number, pull_request_id=pr.id,
)
else:
# otherwise, spawn tasks
spawn_page_tasks_for_pull_request_files.delay(
pr.base_repo.owner_login, pr.base_repo.name, pr.number
)
return jsonify({"message": "success"})
def github_check_contributors():
if request.method == "GET":
return render_template("github_check_contributors.html")
repo = request.form.get("repo", "")
if repo:
repos = (repo,)
else:
repos = get_repos_file().keys()
people = get_people_file()
people_lower = {username.lower() for username in people.keys()}
missing_contributors = defaultdict(set)
for repo in repos:
bugsnag_context = {"repo": repo}
bugsnag.configure_request(meta_data=bugsnag_context)
contributors_url = "/repos/{repo}/contributors".format(repo=repo)
contributors = paginated_get(contributors_url, session=github)
for contributor in contributors:
if contributor["login"].lower() not in people_lower:
missing_contributors[repo].add(contributor["login"])
# convert sets to lists, so jsonify can handle them
output = {
repo: list(contributors)
for repo, contributors in missing_contributors.items()
}
return jsonify(output)
def __init__(self, environ, start_response):
bugsnag.configure_request(user={
"id": "5",
"email": "*****@*****.**",
"name": "conrad",
})
raise SentinelError("oops")
def label(owner, repo, name):
"""
Load a single label from Github into WebhookDB.
:query children: scan all children objects. Defaults to ``false``
:query inline: process the request inline instead of creating a task
on the task queue. Defaults to ``false``.
:statuscode 200: label successfully loaded inline
:statuscode 202: task successfully queued
:statuscode 404: specified label was not found on Github
"""
inline = bool(request.args.get("inline", False))
children = bool(request.args.get("children", False))
bugsnag_ctx = {"owner": owner, "repo": repo, "name": name, "inline": inline}
bugsnag.configure_request(meta_data=bugsnag_ctx)
if inline and not children:
try:
sync_label(
owner, repo, name, children=False,
requestor_id=current_user.get_id(),
)
except NotFound as exc:
return jsonify({"message": exc.message}), 404
else:
return jsonify({"message": "success"})
else:
result = sync_label.delay(owner, repo, name, children=children)
resp = jsonify({"message": "queued"})
resp.status_code = 202
resp.headers["Location"] = url_for("tasks.status", task_id=result.id)
return resp
def __init__(self, environ, start_response):
bugsnag.configure_request(user={
"id": "5",
"email": "*****@*****.**",
"name": "conrad",
})
raise SentinelError("oops")
def github_rescan():
"""
Used to pick up PRs that might not have tickets associated with them.
"""
if request.method == "GET":
# just render the form
return render_template("github_rescan.html")
repo = request.form.get("repo") or "edx/edx-platform"
bugsnag_context = {"repo": repo}
bugsnag.configure_request(meta_data=bugsnag_context)
url = "/repos/{repo}/pulls".format(repo=repo)
created = {}
for pull_request in paginated_get(url, session=github):
bugsnag_context["pull_request"] = pull_request
bugsnag.configure_request(meta_data=bugsnag_context)
if not get_jira_issue_key(pull_request) and not is_edx_pull_request(pull_request):
text = pr_opened(pull_request, bugsnag_context=bugsnag_context)
if "created" in text:
jira_key = text[8:]
created[pull_request["number"]] = jira_key
print(
"Created {num} JIRA issues. PRs are {prs}".format(
num=len(created), prs=created.keys(),
),
file=sys.stderr
)
resp = make_response(json.dumps(created), 200)
resp.headers["Content-Type"] = "application/json"
return resp
def issues(owner, repo):
"""
Queue tasks to load all issues on a single Github repository
into WebhookDB.
:query children: scan all children objects. Defaults to ``false``
:query state: one of ``all``, ``open``, or ``closed``. This parameter
is proxied to the `Github API for listing issues`_.
:statuscode 202: task successfully queued
.. _Github API for listing issues: https://developer.github.com/v3/issues/#list-issues-for-a-repository
"""
bugsnag_ctx = {"owner": owner, "repo": repo}
bugsnag.configure_request(meta_data=bugsnag_ctx)
state = request.args.get("state", "open")
children = bool(request.args.get("children", False))
result = spawn_page_tasks_for_issues.delay(
owner, repo, state, children=children,
requestor_id=current_user.get_id(),
)
resp = jsonify({"message": "queued"})
resp.status_code = 202
resp.headers["Location"] = url_for("tasks.status", task_id=result.id)
return resp
def rescan_repo(repo):
"""
rescans a single repo for new prs
"""
bugsnag_context = {"repo": repo}
bugsnag.configure_request(meta_data=bugsnag_context)
url = "/repos/{repo}/pulls".format(repo=repo)
created = {}
for pull_request in paginated_get(url, session=github):
bugsnag_context["pull_request"] = pull_request
bugsnag.configure_request(meta_data=bugsnag_context)
if not get_jira_issue_key(pull_request) and not is_internal_pull_request(pull_request):
text = pr_opened(pull_request, bugsnag_context=bugsnag_context)
if "created" in text:
jira_key = text[8:]
created[pull_request["number"]] = jira_key
print(
"Created {num} JIRA issues on repo {repo}. PRs are {prs}".format(
num=len(created), repo=repo, prs=created.keys(),
),
file=sys.stderr
)
return created
async def __call__(self, scope, receive, send):
bugsnag.configure_request(asgi_scope=scope)
try:
if bugsnag.configuration.auto_capture_sessions:
bugsnag.start_session()
await self.app(scope, receive, send)
except Exception as e:
bugsnag.auto_notify(e, severity_reason=SEVERITY_REASON)
raise
def __init__(self, application, environ, start_response):
self.environ = environ
bugsnag.configure_request(wsgi_environ=self.environ)
try:
self.app = application(environ, start_response)
except Exception as e:
bugsnag.auto_notify(e)
raise
def __init__(self, application, environ, start_response):
self.environ = environ
bugsnag.configure_request(wsgi_environ=self.environ)
try:
self.app = application(environ, start_response)
except Exception as e:
bugsnag.auto_notify(e)
raise
def jira_rescan_users():
"""
This task goes through all users on JIRA and ensures that they are assigned
to the correct group based on the user's email address. It's meant to be
run regularly: once an hour or so.
"""
# a mapping of group name to email domain
domain_groups = {
"edx-employees": "@edx.org",
"clarice": "@claricetechnologies.com",
"bnotions": "@bnotions.com",
"qualcomm": ".qualcomm.com",
"ubc": "@cs.ubc.ca",
"ubc": "@ubc.ca",
}
if request.method == "GET":
return render_template("jira_rescan_users.html", domain_groups=domain_groups)
failures = defaultdict(dict)
requested_group = request.form.get("group")
if requested_group:
if requested_group not in domain_groups:
resp = jsonify({"error": "Not found", "groups": domain_groups.keys()})
resp.status_code = 404
return resp
requested_groups = {requested_group: domain_groups[requested_group]}
else:
requested_groups = domain_groups
for groupname, domain in requested_groups.items():
users_in_group = jira_group_members(groupname, session=jira, debug=True)
usernames_in_group = set(u["name"] for u in users_in_group)
bugsnag_context = {
"groupname": groupname,
"usernames_in_group": usernames_in_group,
}
bugsnag.configure_request(meta_data=bugsnag_context)
for user in jira_users(filter=domain, session=jira, debug=True):
if not user["email"].endswith(domain):
pass
username = user["name"]
if username not in usernames_in_group:
# add the user to the group!
resp = jira.post(
"/rest/api/2/group/user?groupname={}".format(groupname),
json={"name": username},
)
if not resp.ok:
failures[groupname][username] = resp.text
resp = jsonify(failures)
resp.status_code = 502 if failures else 200
return resp
def __init__(self, application, environ, start_response):
self.environ = environ
bugsnag.configure_request(wsgi_environ=self.environ)
try:
if bugsnag.configuration.auto_capture_sessions:
bugsnag.start_session()
self.app = application(environ, start_response)
except Exception as e:
bugsnag.auto_notify(e, severity_reason=self.SEVERITY_REASON)
raise
async def test_async_task_metadata(self):
bugsnag.configure_request(wins={'total': 3, 'last': 2})
async def coro():
bugsnag.configure_request(wins={'total': 1, 'last': 1})
asyncio.ensure_future(coro())
data = RequestConfiguration.get_instance().wins
assert data['total'] == 3
assert data['last'] == 2
async def asgi(self, receive, send, asgi_scope):
bugsnag.configure_request(asgi_scope=asgi_scope)
inner = self.app(asgi_scope)
try:
await inner(receive, send)
except Exception as exc:
bugsnag.configure_request(last_frame_locals=self.get_locals(exc))
bugsnag.notify(exc)
raise exc from None
finally:
bugsnag.clear_request_config()
def _notify_bugsnag(self, exception):
try:
configure_request(context='%s %s' % (Request.method, Request.environ.get('PATH_INFO', '')),
user_id=Request.remote_addr,
request_data={'url': Request.url,
'headers': dict(Request.headers),
'cookies': dict(Request.cookies),
'params': dict(Request.params)},
environment_data=dict(Request.environ))
notify(exception)
clear_request_config()
except:
pass
def __handle_request_exception(sender, **kwargs):
request = kwargs.get('request', None)
if request is not None:
bugsnag.configure_request(django_request=request)
try:
bugsnag.auto_notify_exc_info(
severity_reason={
"type": "unhandledExceptionMiddleware",
"attributes": {
"framework": "Django"
}
})
except Exception:
bugsnag.logger.exception("Error in exception middleware")
def __init__(self, application, environ, start_response):
self.environ = environ
bugsnag.configure_request(wsgi_environ=self.environ)
try:
if bugsnag.configuration.auto_capture_sessions:
bugsnag.start_session()
self.app = application(environ, start_response)
except Exception as e:
bugsnag.auto_notify(
e,
severity_reason=self.SEVERITY_REASON
)
raise
def handle_exception(exception, env):
request = Request(env)
bugsnag.configure_request(
context="%s %s" % (request.method, request_path(env)),
user_id=request.remote_addr,
request_data={
"url": request.base_url,
"headers": dict(request.headers),
"cookies": dict(request.cookies),
"params": dict(request.form),
},
environment_data=dict(request.environ),
)
bugsnag.auto_notify(exception)
bugsnag.clear_request_config()
def _handle_request_exception(self, exc):
# Set the request info
bugsnag.configure_request(
user_id=self.request.remote_ip,
context=self._get_context(),
request_data={
"url": self.request.full_url(),
"method": self.request.method,
"arguments": self.request.arguments,
},
)
# Notify bugsnag
bugsnag.notify(exc)
# Call the parent handler
RequestHandler._handle_request_exception(self, exc)
def jira_rescan():
if request.method == "GET":
# just render the form
return render_template("jira_rescan.html")
jql = request.form.get("jql") or 'status = "Needs Triage" ORDER BY key'
bugsnag_context = {"jql": jql}
bugsnag.configure_request(meta_data=bugsnag_context)
issues = jira_paginated_get("/rest/api/2/search", "issues", jql=jql, session=jira)
results = {}
for issue in issues:
issue_key = issue["key"].decode('utf-8')
results[issue_key] = issue_opened(issue)
resp = make_response(json.dumps(results), 200)
resp.headers["Content-Type"] = "application/json"
return resp
def __log_exception(sender, exception, **extra):
bugsnag.configure_request(
context="%s %s" % (request.method, request_path(request.environ)),
user_id=request.remote_addr,
request_data={
"url": request.base_url,
"headers": dict(request.headers),
"cookies": dict(request.cookies),
"params": dict(request.form),
},
session_data=dict(session),
environment_data=dict(request.environ),
)
bugsnag.auto_notify(exception)
bugsnag.clear_request_config()
def jira_issue_rejected(issue, bugsnag_context=None):
bugsnag_context = bugsnag_context or {}
issue_key = to_unicode(issue["key"])
pr_num = github_pr_num(issue)
pr_url = github_pr_url(issue)
issue_url = pr_url.replace("pulls", "issues")
gh_issue_resp = github.get(issue_url)
if not gh_issue_resp.ok:
raise requests.exceptions.RequestException(gh_issue_resp.text)
gh_issue = gh_issue_resp.json()
bugsnag_context["github_issue"] = gh_issue
bugsnag.configure_request(meta_data=bugsnag_context)
if gh_issue["state"] == "closed":
# nothing to do
msg = "{key} was rejected, but PR #{num} was already closed".format(
key=issue_key, num=pr_num
)
print(msg, file=sys.stderr)
return msg
# Comment on the PR to explain to look at JIRA
username = to_unicode(gh_issue["user"]["login"])
comment = {"body": (
"Hello @{username}: We are unable to continue with "
"review of your submission at this time. Please see the "
"associated JIRA ticket for more explanation.".format(username=username)
)}
comment_resp = github.post(issue_url + "/comments", json=comment)
# close the pull request on Github
close_resp = github.patch(pr_url, json={"state": "closed"})
if not close_resp.ok or not comment_resp.ok:
bugsnag_context['request_headers'] = close_resp.request.headers
bugsnag_context['request_url'] = close_resp.request.url
bugsnag_context['request_method'] = close_resp.request.method
bugsnag.configure_request(meta_data=bugsnag_context)
bug_text = ''
if not close_resp.ok:
bug_text += "Failed to close; " + close_resp.text
if not comment_resp.ok:
bug_text += "Failed to comment on the PR; " + comment_resp.text
raise requests.exceptions.RequestException(bug_text)
return "Closed PR #{num}".format(num=pr_num)
async def bugsnag_app(self, scope: Scope, receive: Receive,
send: Send) -> None:
bugsnag.configure_request(scope=scope)
inner = self.app
try:
await inner(scope, receive, send)
except Exception as exc:
bugsnag.configure_request(frame_locals=self.get_locals(exc))
bugsnag.auto_notify(exc,
severity_reason={
"type": "unhandledExceptionMiddleware",
"attributes": {
"framework": "Starlette"
}
})
raise exc from None
finally:
bugsnag.clear_request_config()
def labels(owner, repo):
"""
Queue tasks to load all labels on a single Github repository
into WebhookDB.
:statuscode 202: task successfully queued
"""
bugsnag_ctx = {"owner": owner, "repo": repo}
bugsnag.configure_request(meta_data=bugsnag_ctx)
children = bool(request.args.get("children", False))
result = spawn_page_tasks_for_labels.delay(
owner, repo, requestor_id=current_user.get_id(),
)
resp = jsonify({"message": "queued"})
resp.status_code = 202
resp.headers["Location"] = url_for("tasks.status", task_id=result.id)
return resp
def repository():
payload = request.get_json()
bugsnag.configure_request(meta_data={"payload": payload})
repo_data = payload.get("repository")
if not repo_data:
resp = jsonify({"error": "no repository in payload"})
resp.status_code = 400
return resp
try:
process_repository(repo_data)
except MissingData as err:
return jsonify({"error": err.message, "obj": err.obj}), 400
except StaleData:
return jsonify({"message": "stale data"})
else:
return jsonify({"message": "success"})
def issue_opened(issue, bugsnag_context=None):
bugsnag_context = bugsnag_context or {}
bugsnag_context = {"issue": issue}
bugsnag.configure_request(meta_data=bugsnag_context)
issue_key = to_unicode(issue["key"])
issue_url = URLObject(issue["self"])
transitioned = False
if should_transition(issue):
transitions_url = issue_url.with_path(issue_url.path + "/transitions")
transitions_resp = jira_get(transitions_url)
if not transitions_resp.ok:
raise requests.exceptions.RequestException(transitions_resp.text)
transitions = {t["name"]: t["id"] for t in transitions_resp.json()["transitions"]}
if "Open" in transitions:
new_status = "Open"
elif "Design Backlog" in transitions:
new_status = "Design Backlog"
else:
raise ValueError("No valid transition! Possibilities are {}".format(transitions.keys()))
body = {
"transition": {
"id": transitions[new_status],
}
}
transition_resp = jira.post(transitions_url, json=body)
if not transition_resp.ok:
raise requests.exceptions.RequestException(transition_resp.text)
transitioned = True
# log to stderr
action = "Transitioned to Open" if transitioned else "ignored"
print(
"{key} created by {name} ({username}), {action}".format(
key=issue_key,
name=to_unicode(issue["fields"]["creator"]["displayName"]),
username=to_unicode(issue["fields"]["creator"]["name"]),
action="Transitioned to Open" if transitioned else "ignored",
),
file=sys.stderr,
)
return action
def github_install():
if request.method == "GET":
return render_template("install.html")
repo = request.form.get("repo", "")
if repo:
repos = (repo, )
else:
repos = get_repos_file().keys()
secure = request.is_secure or request.headers.get("X-Forwarded-Proto",
"http") == "https"
api_url = url_for(
"github_pull_request",
_external=True,
_scheme="https" if secure else "http",
)
success = []
failed = []
for repo in repos:
url = "/repos/{repo}/hooks".format(repo=repo)
body = {
"name": "web",
"events": ["pull_request"],
"config": {
"url": api_url,
"content_type": "json",
}
}
bugsnag_context = {"repo": repo, "body": body}
bugsnag.configure_request(meta_data=bugsnag_context)
hook_resp = github.post(url, json=body)
if hook_resp.ok:
success.append(repo)
else:
failed.append((repo, hook_resp.text))
if failed:
resp = make_response(json.dumps(failed), 502)
else:
resp = make_response(json.dumps(success), 200)
resp.headers["Content-Type"] = "application/json"
return resp
def pull_request_files(owner, repo, number):
"""
Queue tasks to load the pull request files (diffs) for a single pull request
into WebhookDB.
:statuscode 202: task successfully queued
"""
bugsnag_ctx = {"owner": owner, "repo": repo, "number": number}
bugsnag.configure_request(meta_data=bugsnag_ctx)
children = bool(request.args.get("children", False))
result = spawn_page_tasks_for_pull_request_files.delay(
owner, repo, number, children=children,
requestor_id=current_user.get_id(),
)
resp = jsonify({"message": "queued"})
resp.status_code = 202
resp.headers["Location"] = url_for("tasks.status", task_id=result.id)
return resp
def jira_rescan_issues():
if request.method == "GET":
# just render the form
return render_template("jira_rescan_issues.html")
jql = request.form.get("jql") or 'status = "Needs Triage" ORDER BY key'
bugsnag_context = {"jql": jql}
bugsnag.configure_request(meta_data=bugsnag_context)
issues = jira_paginated_get(
"/rest/api/2/search", jql=jql, obj_name="issues", session=jira,
)
results = {}
for issue in issues:
issue_key = to_unicode(issue["key"])
results[issue_key] = issue_opened(issue)
resp = make_response(json.dumps(results), 200)
resp.headers["Content-Type"] = "application/json"
return resp
def install():
if request.method == "GET":
return render_template("install.html")
owner_login = request.values.get("owner", "")
if not owner_login:
return jsonify({"error": "missing owner param"}), 400
repo_name = request.values.get("repo", "")
if not repo_name:
return jsonify({"error": "missing repo param"}), 400
hook_url = "/repos/{owner}/{repo}/hooks".format(
owner=owner_login,
repo=repo_name,
)
body = {
"name": "web",
"events": ["pull_request", "issue"],
"config": {
"url": url_for("replication.main", _external=True),
"content_type": "json",
}
}
bugsnag_context = {"owner": owner_login, "repo": repo_name, "body": body}
bugsnag.configure_request(meta_data=bugsnag_context)
logging.info("POST {}".format(hook_url))
hook_resp = github.post(hook_url, json=body)
if not hook_resp.ok:
error_obj = hook_resp.json()
resp = jsonify({"error": error_obj["message"]})
resp.status_code = 503
return resp
else:
hook_data = hook_resp.json()
process_repository_hook(
hook_data,
via="api",
fetched_at=datetime.now(),
commit=True,
requestor_id=current_user.get_id(),
)
return jsonify({"message": "success"})
def labels(owner, repo):
"""
Queue tasks to load all labels on a single Github repository
into WebhookDB.
:statuscode 202: task successfully queued
"""
bugsnag_ctx = {"owner": owner, "repo": repo}
bugsnag.configure_request(meta_data=bugsnag_ctx)
children = bool(request.args.get("children", False))
result = spawn_page_tasks_for_labels.delay(
owner,
repo,
requestor_id=current_user.get_id(),
)
resp = jsonify({"message": "queued"})
resp.status_code = 202
resp.headers["Location"] = url_for("tasks.status", task_id=result.id)
return resp
def github_install():
if request.method == "GET":
return render_template("install.html")
repo = request.form.get("repo", "")
if repo:
repos = (repo,)
else:
repos = get_repos_file().keys()
secure = request.is_secure or request.headers.get("X-Forwarded-Proto", "http") == "https"
api_url = url_for(
"github_pull_request", _external=True,
_scheme="https" if secure else "http",
)
success = []
failed = []
for repo in repos:
url = "/repos/{repo}/hooks".format(repo=repo)
body = {
"name": "web",
"events": ["pull_request"],
"config": {
"url": api_url,
"content_type": "json",
}
}
bugsnag_context = {"repo": repo, "body": body}
bugsnag.configure_request(meta_data=bugsnag_context)
hook_resp = github.post(url, json=body)
if hook_resp.ok:
success.append(repo)
else:
failed.append((repo, hook_resp.text))
if failed:
resp = make_response(json.dumps(failed), 502)
else:
resp = make_response(json.dumps(success), 200)
resp.headers["Content-Type"] = "application/json"
return resp
def issue():
"""
Webhook endpoint for ``issues`` events on Github.
"""
payload = request.get_json()
bugsnag.configure_request(meta_data={"payload": payload})
issue_data = payload.get("issue")
if not issue_data:
resp = jsonify({"error": "no issue in payload"})
resp.status_code = 400
return resp
try:
issue = process_issue(issue_data)
except MissingData as err:
return jsonify({"error": err.message, "obj": err.obj}), 400
except StaleData:
return jsonify({"message": "stale data"})
return jsonify({"message": "success"})
def github_pull_request():
"""
Process a `PullRequestEvent`_ from Github.
.. _PullRequestEvent: https://developer.github.com/v3/activity/events/types/#pullrequestevent
"""
try:
event = request.get_json()
except ValueError:
raise ValueError("Invalid JSON from Github: {data}".format(data=request.data))
bugsnag_context = {"event": event}
bugsnag.configure_request(meta_data=bugsnag_context)
if "pull_request" not in event and "hook" in event and "zen" in event:
# this is a ping
repo = event.get("repository", {}).get("full_name")
print("ping from {repo}".format(repo=repo), file=sys.stderr)
return "PONG"
pr = event["pull_request"]
repo = pr["base"]["repo"]["full_name"].decode('utf-8')
if event["action"] == "opened":
return pr_opened(pr, bugsnag_context)
if event["action"] == "closed":
return pr_closed(pr, bugsnag_context)
if event["action"] == "labeled":
return "Ignoring labeling events from github", 200
if event["action"] == "synchronize":
return "Ignoring synchronize events from github", 200
print(
"Received {action} event on PR #{num} against {repo}, don't know how to handle it".format(
action=event["action"],
repo=pr["base"]["repo"]["full_name"].decode('utf-8'),
num=pr["number"],
),
file=sys.stderr
)
return "Don't know how to handle this.", 400
def github_pull_request():
"""
Process a `PullRequestEvent`_ from Github.
.. _PullRequestEvent: https://developer.github.com/v3/activity/events/types/#pullrequestevent
"""
try:
event = request.get_json()
except ValueError:
raise ValueError("Invalid JSON from Github: {data}".format(data=request.data))
bugsnag_context = {"event": event}
bugsnag.configure_request(meta_data=bugsnag_context)
if "pull_request" not in event and "hook" in event and "zen" in event:
# this is a ping
repo = event.get("repository", {}).get("full_name")
print("ping from {repo}".format(repo=repo), file=sys.stderr)
return "PONG"
pr = event["pull_request"]
repo = pr["base"]["repo"]["full_name"].decode('utf-8')
if event["action"] == "opened":
return pr_opened(pr, bugsnag_context)
if event["action"] == "closed":
return pr_closed(pr, bugsnag_context)
if event["action"] == "labeled":
return "Ignoring labeling events from github", 200
if event["action"] == "synchronize":
return "Ignoring synchronize events from github", 200
print(
"Received {action} event on PR #{num} against {repo}, don't know how to handle it".format(
action=event["action"],
repo=pr["base"]["repo"]["full_name"].decode('utf-8'),
num=pr["number"],
),
file=sys.stderr
)
return "Don't know how to handle this.", 400
def process_request(self, request):
if is_development_server(request):
bugsnag.configure(release_stage="development")
try:
bugsnag.configure_request(
context=request.path,
user_id=get_user_id(request),
session_data=dict(request.session),
request_data={
'path': request.path,
'encoding': request.encoding,
'params': dict(request.REQUEST),
'url': request.build_absolute_uri(),
},
environment_data=dict(request.META),
)
except Exception as exc:
bugsnag.log("Error in request middleware: %s" % exc)
return None
def pull_request_files(owner, repo, number):
"""
Queue tasks to load the pull request files (diffs) for a single pull request
into WebhookDB.
:statuscode 202: task successfully queued
"""
bugsnag_ctx = {"owner": owner, "repo": repo, "number": number}
bugsnag.configure_request(meta_data=bugsnag_ctx)
children = bool(request.args.get("children", False))
result = spawn_page_tasks_for_pull_request_files.delay(
owner,
repo,
number,
children=children,
requestor_id=current_user.get_id(),
)
resp = jsonify({"message": "queued"})
resp.status_code = 202
resp.headers["Location"] = url_for("tasks.status", task_id=result.id)
return resp
def repository(owner, repo):
"""
Load a single repository from Github into WebhookDB. Note that this does
not load issues, pull requests, etc for that repository into WebhookDB.
:query inline: process the request inline instead of creating a task
on the task queue. Defaults to ``false``.
:statuscode 200: repository successfully loaded inline
:statuscode 202: task successfully queued
:statuscode 404: specified repository was not found on Github
"""
inline = bool(request.args.get("inline", False))
children = bool(request.args.get("children", False))
bugsnag_ctx = {
"owner": owner, "repo": repo,
"inline": inline, "children": children,
}
bugsnag.configure_request(meta_data=bugsnag_ctx)
if inline and not children:
try:
sync_repository(
owner, repo, children=children,
requestor_id=current_user.get_id(),
)
except NotFound as exc:
return jsonify({"message": exc.message}), 404
else:
return jsonify({"message": "success"})
else:
result = sync_repository.delay(
owner, repo, children=children,
requestor_id=current_user.get_id(),
)
resp = jsonify({"message": "queued"})
resp.status_code = 202
resp.headers["Location"] = url_for("tasks.status", task_id=result.id)
return resp
def pull_request():
"""
Webhook endpoint for ``pull_request`` events on Github.
"""
payload = request.get_json()
bugsnag.configure_request(meta_data={"payload": payload})
pr_data = payload.get("pull_request")
if not pr_data:
resp = jsonify({"error": "no pull_request in payload"})
resp.status_code = 400
return resp
try:
pr = process_pull_request(pr_data)
except MissingData as err:
return jsonify({"error": err.message, "obj": err.obj}), 400
except StaleData:
return jsonify({"message": "stale data"})
# Fetch the pull request files, too!
if pr.changed_files < 100:
# If there are fewer than 100, do it inline
PullRequestFile.query.filter_by(pull_request_id=pr.id).delete()
sync_page_of_pull_request_files(
owner=pr.base_repo.owner_login,
repo=pr.base_repo.name,
number=pr.number,
pull_request_id=pr.id,
)
else:
# otherwise, spawn tasks
spawn_page_tasks_for_pull_request_files.delay(pr.base_repo.owner_login,
pr.base_repo.name,
pr.number)
return jsonify({"message": "success"})
def process_request(self, request):
bugsnag.configure_request(django_request=request)
return None
def pr_opened(pr, bugsnag_context=None):
bugsnag_context = bugsnag_context or {}
user = pr["user"]["login"]
repo = pr["base"]["repo"]["full_name"]
people = get_people_file()
if user in people and people[user].get("institution", "") == "edX":
# not an open source pull request, don't create an issue for it
print(
"@{user} opened PR #{num} against {repo} (internal PR)".format(
user=user, repo=pr["base"]["repo"]["full_name"],
num=pr["number"]
),
file=sys.stderr
)
return "internal pull request"
field_resp = jira.get("/rest/api/2/field")
if not field_resp.ok:
raise requests.exceptions.RequestException(field_resp.text)
field_map = dict(pop_dict_id(f) for f in field_resp.json())
custom_fields = {
value["name"]: id
for id, value in field_map.items()
if value["custom"]
}
user_resp = github.get(pr["user"]["url"])
if user_resp.ok:
user_name = user_resp.json().get("name", user)
else:
user_name = user
# create an issue on JIRA!
new_issue = {
"fields": {
"project": {
"key": "OSPR",
},
"issuetype": {
"name": "Pull Request Review",
},
"summary": pr["title"],
"description": pr["body"],
custom_fields["URL"]: pr["html_url"],
custom_fields["PR Number"]: pr["number"],
custom_fields["Repo"]: pr["base"]["repo"]["full_name"],
custom_fields["Contributor Name"]: user_name,
}
}
institution = people.get(user, {}).get("institution", None)
if institution:
new_issue["fields"][custom_fields["Customer"]] = [institution]
bugsnag_context["new_issue"] = new_issue
bugsnag.configure_request(meta_data=bugsnag_context)
resp = jira.post("/rest/api/2/issue", data=json.dumps(new_issue))
if not resp.ok:
raise requests.exceptions.RequestException(resp.text)
new_issue_body = resp.json()
bugsnag_context["new_issue"]["key"] = new_issue_body["key"]
bugsnag.configure_request(meta_data=bugsnag_context)
# add a comment to the Github pull request with a link to the JIRA issue
comment = {
"body": github_pr_comment(pr, new_issue_body, people),
}
url = "/repos/{repo}/issues/{num}/comments".format(
repo=repo, num=pr["number"],
)
comment_resp = github.post(url, data=json.dumps(comment))
if not comment_resp.ok:
raise requests.exceptions.RequestException(comment_resp.text)
print(
"@{user} opened PR #{num} against {repo}, created {issue} to track it".format(
user=user, repo=repo,
num=pr["number"], issue=new_issue_body["key"]
),
file=sys.stderr
)
return "created!"
def process_request(self, request):
bugsnag.configure_request(django_request=request)
return None
def index(request):
bugsnag.configure_request(extra_data={"james": "is testing"})
raise Exception("oh dear")
return HttpResponse("Hello, world. You're at the poll index.")
def issue_opened(issue, bugsnag_context=None):
bugsnag_context = bugsnag_context or {}
bugsnag_context = {"issue": issue}
bugsnag.configure_request(meta_data=bugsnag_context)
issue_key = to_unicode(issue["key"])
issue_url = URLObject(issue["self"])
transitioned = False
if should_transition(issue):
# In JIRA, a "transition" is how an issue changes from one status
# to another, like going from "Open" to "In Progress". The workflow
# defines what transitions are allowed, and this API will tell us
# what transitions are currently allowed by the workflow.
# Ref: https://docs.atlassian.com/jira/REST/ondemand/#d2e4954
transitions_url = issue_url.with_path(issue_url.path + "/transitions")
transitions_resp = jira_get(transitions_url)
if not transitions_resp.ok:
raise requests.exceptions.RequestException(transitions_resp.text)
# This transforms the API response into a simple mapping from the
# name of the transition (like "In Progress") to the ID of the transition.
# Note that a transition may not have the same name as the state that it
# goes to, so a transition to go from "Open" to "In Progress" may be
# named something like "Start Work".
transitions = {t["name"]: t["id"] for t in transitions_resp.json()["transitions"]}
# We attempt to transition the issue into the "Open" state for the given project
# (some projects use a different name), so look for a transition with the right name
new_status = None
action = None
for state_name in ["Open", "Design Backlog", "To Do"]:
if state_name in transitions:
new_status = state_name
action = "Transitioned to '{}'".format(state_name)
if not new_status:
# If it's an OSPR subtask (used by teams to manage reviews), transition to team backlog
if to_unicode(issue["fields"]["project"]["key"]) == "OSPR" and issue["fields"]["issuetype"]["subtask"]:
new_status = "To Backlog"
action = "Transitioned to 'To Backlog'"
else:
raise ValueError("No valid transition! Possibilities are {}".format(transitions.keys()))
# This creates a new API request to tell JIRA to move the issue from
# one status to another using the specified transition. We have to
# tell JIRA the transition ID, so we use that mapping we set up earlier.
body = {
"transition": {
"id": transitions[new_status],
}
}
transition_resp = jira.post(transitions_url, json=body)
if not transition_resp.ok:
raise requests.exceptions.RequestException(transition_resp.text)
transitioned = True
# log to stderr
if transitioned and not action:
action = "Transitioned to Open"
else:
action = "ignored"
print(
"{key} created by {name} ({username}), {action}".format(
key=issue_key,
name=to_unicode(issue["fields"]["creator"]["displayName"]),
username=to_unicode(issue["fields"]["creator"]["name"]),
action="Transitioned to Open" if transitioned else "ignored",
),
file=sys.stderr,
)
return action
def jira_issue_updated():
"""
Received an "issue updated" event from JIRA. See `JIRA's webhook docs`_.
.. _JIRA's webhook docs: https://developer.atlassian.com/display/JIRADEV/JIRA+Webhooks+Overview
"""
try:
event = request.get_json()
except ValueError:
raise ValueError("Invalid JSON from JIRA: {data}".format(
data=request.data.decode('utf-8')
))
bugsnag_context = {"event": event}
bugsnag.configure_request(meta_data=bugsnag_context)
if app.debug:
print(json.dumps(event), file=sys.stderr)
if "issue" not in event:
# It's rare, but we occasionally see junk data from JIRA. For example,
# here's a real API request we've received on this handler:
# {"baseUrl": "https://openedx.atlassian.net",
# "key": "jira:1fec1026-b232-438f-adab-13b301059297",
# "newVersion": 64005, "oldVersion": 64003}
# If we don't have an "issue" key, it's junk.
return "What is this shit!?", 400
# is this a comment?
comment = event.get("comment")
if comment:
return jira_issue_comment_added(event["issue"], comment, bugsnag_context)
# is the issue an open source pull request?
if event["issue"]["fields"]["project"]["key"] != "OSPR":
return "I don't care"
# we don't care about OSPR subtasks
if event["issue"]["fields"]["issuetype"]["subtask"]:
return "ignoring subtasks"
# don't care about feature proposals
if event["issue"]["fields"]["issuetype"]["name"] == "Feature Proposal":
return "ignoring feature propsals"
# is there a changelog?
changelog = event.get("changelog")
if not changelog:
# it was just someone adding a comment
return "I don't care"
# did the issue change status?
status_changelog_items = [item for item in changelog["items"] if item["field"] == "status"]
if len(status_changelog_items) == 0:
return "I don't care"
pr_repo = github_pr_repo(event["issue"])
if not pr_repo:
issue_key = to_unicode(event["issue"]["key"])
fail_msg = '{key} is missing "Repo" field'.format(key=issue_key)
fail_msg += ' {0}'.format(event["issue"]["fields"]["issuetype"])
raise Exception(fail_msg)
repo_labels_resp = github.get("/repos/{repo}/labels".format(repo=pr_repo))
if not repo_labels_resp.ok:
raise requests.exceptions.RequestException(repo_labels_resp.text)
# map of label name to label URL
repo_labels = {l["name"]: l["url"] for l in repo_labels_resp.json()}
# map of label name lowercased to label name in the case that it is on Github
repo_labels_lower = {name.lower(): name for name in repo_labels}
old_status = status_changelog_items[0]["fromString"]
new_status = status_changelog_items[0]["toString"]
changes = []
if new_status == "Rejected":
change = jira_issue_rejected(event["issue"], bugsnag_context)
changes.append(change)
elif 'blocked' in new_status.lower():
print("New status is: {}".format(new_status))
print("repo_labels_lower: {}".format(repo_labels_lower))
if new_status.lower() in repo_labels_lower:
change = jira_issue_status_changed(event["issue"], event["changelog"], bugsnag_context)
changes.append(change)
if changes:
return "\n".join(changes)
else:
return "no change necessary"