def rescan_repo(repo):
"""
rescans a single repo for new prs
"""
bugsnag_context = {"repo": repo}
bugsnag.configure_request(meta_data=bugsnag_context)
url = "/repos/{repo}/pulls".format(repo=repo)
created = {}
for pull_request in paginated_get(url, session=github):
bugsnag_context["pull_request"] = pull_request
bugsnag.configure_request(meta_data=bugsnag_context)
if not get_jira_issue_key(pull_request) and not is_internal_pull_request(pull_request):
text = pr_opened(pull_request, bugsnag_context=bugsnag_context)
if "created" in text:
jira_key = text[8:]
created[pull_request["number"]] = jira_key
print(
"Created {num} JIRA issues on repo {repo}. PRs are {prs}".format(
num=len(created), repo=repo, prs=created.keys(),
),
file=sys.stderr
)
return created
def github_check_contributors():
if request.method == "GET":
return render_template("github_check_contributors.html")
repo = request.form.get("repo", "")
if repo:
repos = (repo,)
else:
repos = get_repos_file().keys()
people = get_people_file()
people_lower = {username.lower() for username in people.keys()}
missing_contributors = defaultdict(set)
for repo in repos:
bugsnag_context = {"repo": repo}
bugsnag.configure_request(meta_data=bugsnag_context)
contributors_url = "/repos/{repo}/contributors".format(repo=repo)
contributors = paginated_get(contributors_url, session=github)
for contributor in contributors:
if contributor["login"].lower() not in people_lower:
missing_contributors[repo].add(contributor["login"])
# convert sets to lists, so jsonify can handle them
output = {
repo: list(contributors)
for repo, contributors in missing_contributors.items()
}
return jsonify(output)
def github_rescan():
"""
Used to pick up PRs that might not have tickets associated with them.
"""
if request.method == "GET":
# just render the form
return render_template("github_rescan.html")
repo = request.form.get("repo") or "edx/edx-platform"
bugsnag_context = {"repo": repo}
bugsnag.configure_request(meta_data=bugsnag_context)
url = "/repos/{repo}/pulls".format(repo=repo)
created = {}
for pull_request in paginated_get(url, session=github):
bugsnag_context["pull_request"] = pull_request
bugsnag.configure_request(meta_data=bugsnag_context)
if not get_jira_issue_key(pull_request) and not is_edx_pull_request(pull_request):
text = pr_opened(pull_request, bugsnag_context=bugsnag_context)
if "created" in text:
jira_key = text[8:]
created[pull_request["number"]] = jira_key
print(
"Created {num} JIRA issues. PRs are {prs}".format(
num=len(created), prs=created.keys(),
),
file=sys.stderr
)
resp = make_response(json.dumps(created), 200)
resp.headers["Content-Type"] = "application/json"
return resp
def github_rescan():
"""
Used to pick up PRs that might not have tickets associated with them.
"""
if request.method == "GET":
# just render the form
return render_template("github_rescan.html")
repo = request.form.get("repo") or "edx/edx-platform"
bugsnag_context = {"repo": repo}
bugsnag.configure_request(meta_data=bugsnag_context)
url = "/repos/{repo}/pulls".format(repo=repo)
created = {}
for pull_request in paginated_get(url, session=github):
bugsnag_context["pull_request"] = pull_request
bugsnag.configure_request(meta_data=bugsnag_context)
if not get_jira_issue_key(
pull_request) and not is_internal_pull_request(pull_request):
text = pr_opened(pull_request, bugsnag_context=bugsnag_context)
if "created" in text:
jira_key = text[8:]
created[pull_request["number"]] = jira_key
print("Created {num} JIRA issues. PRs are {prs}".format(
num=len(created),
prs=created.keys(),
),
file=sys.stderr)
resp = make_response(json.dumps(created), 200)
resp.headers["Content-Type"] = "application/json"
return resp
def check_contributors():
"""
Identify missing contributors: people who have commits in a repository,
but who are not listed in the AUTHORS file.
"""
repo = request.form.get("repo", "")
if repo:
repos = (repo, )
else:
repos = get_repos_file().keys()
people = get_people_file()
people_lower = {username.lower() for username in people.keys()}
missing_contributors = defaultdict(set)
for repo in repos:
sentry.client.extra_context({"repo": repo})
contributors_url = "/repos/{repo}/contributors".format(repo=repo)
contributors = paginated_get(contributors_url, session=github)
for contributor in contributors:
if contributor["login"].lower() not in people_lower:
missing_contributors[repo].add(contributor["login"])
# convert sets to lists, so jsonify can handle them
output = {
repo: list(contributors)
for repo, contributors in missing_contributors.items()
}
return jsonify(output)
def check_contributors():
"""
Identify missing contributors: people who have commits in a repository,
but who are not listed in the AUTHORS file.
"""
repo = request.form.get("repo", "")
if repo:
repos = (repo,)
else:
repos = get_repos_file().keys()
people = get_people_file()
people_lower = {username.lower() for username in people.keys()}
missing_contributors = defaultdict(set)
for repo in repos:
sentry.client.extra_context({"repo": repo})
contributors_url = "/repos/{repo}/contributors".format(repo=repo)
contributors = paginated_get(contributors_url, session=github)
for contributor in contributors:
if contributor["login"].lower() not in people_lower:
missing_contributors[repo].add(contributor["login"])
# convert sets to lists, so jsonify can handle them
output = {
repo: list(contributors)
for repo, contributors in missing_contributors.items()
}
return jsonify(output)
def github_check_contributors():
if request.method == "GET":
return render_template("github_check_contributors.html")
repo = request.form.get("repo", "")
if repo:
repos = (repo,)
else:
repos = get_repos_file().keys()
people = get_people_file()
people_lower = {username.lower() for username in people.keys()}
missing_contributors = defaultdict(set)
for repo in repos:
bugsnag_context = {"repo": repo}
bugsnag.configure_request(meta_data=bugsnag_context)
contributors_url = "/repos/{repo}/contributors".format(repo=repo)
contributors = paginated_get(contributors_url, session=github)
for contributor in contributors:
if contributor["login"].lower() not in people_lower:
missing_contributors[repo].add(contributor["login"])
# convert sets to lists, so jsonify can handle them
output = {
repo: list(contributors)
for repo, contributors in missing_contributors.items()
}
return jsonify(output)
def rescan_repo(repo):
"""
rescans a single repo for new prs
"""
bugsnag_context = {"repo": repo}
bugsnag.configure_request(meta_data=bugsnag_context)
url = "/repos/{repo}/pulls".format(repo=repo)
created = {}
for pull_request in paginated_get(url, session=github):
bugsnag_context["pull_request"] = pull_request
bugsnag.configure_request(meta_data=bugsnag_context)
if not get_jira_issue_key(pull_request) and not is_internal_pull_request(pull_request):
text = pr_opened(pull_request, bugsnag_context=bugsnag_context)
if "created" in text:
jira_key = text[8:]
created[pull_request["number"]] = jira_key
print(
"Created {num} JIRA issues on repo {repo}. PRs are {prs}".format(
num=len(created), repo=repo, prs=created.keys(),
),
file=sys.stderr
)
return created
def has_internal_cover_letter(pull_request):
"""
Given a pull request, this function returns a boolean indicating whether
the body has already been replaced with the cover letter template.
"""
me = github_whoami()
my_username = me["login"]
comment_url = "/repos/{repo}/issues/{num}/comments".format(
repo=pull_request["base"]["repo"]["full_name"].decode('utf-8'),
num=pull_request["number"],
)
for comment in paginated_get(comment_url, session=github_bp.session):
# I only care about comments I made
if comment["user"]["login"] != my_username:
continue
body = comment["body"].decode('utf-8')
magic_phrases = [
"# Sandbox",
"# Testing",
"# Reviewers",
"# DevOps assistance",
]
if all(magic_phrase in body for magic_phrase in magic_phrases):
return True
return False
def rescan_repository(self, repo):
"""
rescans a single repo for new prs
"""
github = github_bp.session
sentry_extra_context({"repo": repo})
url = "/repos/{repo}/pulls".format(repo=repo)
created = {}
if not self.request.called_directly:
self.update_state(state='STARTED', meta={'repo': repo})
def page_callback(response):
if not response.ok or self.request.called_directly:
return
current_url = URLObject(response.url)
current_page = int(current_url.query_dict.get("page", 1))
link_last = response.links.get("last")
if link_last:
last_url = URLObject(link_last['url'])
last_page = int(last_url.query_dict["page"])
else:
last_page = current_page
state_meta = {
"repo": repo,
"current_page": current_page,
"last_page": last_page
}
self.update_state(state='STARTED', meta=state_meta)
for pull_request in paginated_get(url,
session=github,
callback=page_callback):
sentry_extra_context({"pull_request": pull_request})
issue_key = get_jira_issue_key(pull_request)
is_internal = is_internal_pull_request(pull_request)
if not issue_key and not is_internal:
# `pull_request_opened()` is a celery task, but by calling it as
# a function instead of calling `.delay()` on it, we're eagerly
# executing the task now, instead of adding it to the task queue
# so it is executed later. As a result, this will return the values
# that the `pull_request_opened()` function returns, rather than
# return an AsyncResult object.
issue_key, issue_created = pull_request_opened(pull_request) # pylint: disable=no-value-for-parameter
if issue_created:
created[pull_request["number"]] = issue_key
logger.info(
"Created {num} JIRA issues on repo {repo}. PRs are {prs}".format(
num=len(created),
repo=repo,
prs=created.keys(),
), )
info = {"repo": repo}
if created:
info["created"] = created
return info
def rescan_repository(self, repo):
"""
rescans a single repo for new prs
"""
github = github_bp.session
sentry.client.extra_context({"repo": repo})
url = "/repos/{repo}/pulls".format(repo=repo)
created = {}
if not self.request.called_directly:
self.update_state(state='STARTED', meta={'repo': repo})
def page_callback(response):
if not response.ok or self.request.called_directly:
return
current_url = URLObject(response.url)
current_page = int(current_url.query_dict.get("page", 1))
link_last = response.links.get("last")
if link_last:
last_url = URLObject(link_last['url'])
last_page = int(last_url.query_dict["page"])
else:
last_page = current_page
state_meta = {
"repo": repo,
"current_page": current_page,
"last_page": last_page
}
self.update_state(state='STARTED', meta=state_meta)
for pull_request in paginated_get(url, session=github, callback=page_callback):
sentry.client.extra_context({"pull_request": pull_request})
issue_key = get_jira_issue_key(pull_request)
is_internal = is_internal_pull_request(pull_request)
if not issue_key and not is_internal:
# `pull_request_opened()` is a celery task, but by calling it as
# a function instead of calling `.delay()` on it, we're eagerly
# executing the task now, instead of adding it to the task queue
# so it is executed later. As a result, this will return the values
# that the `pull_request_opened()` function returns, rather than
# return an AsyncResult object.
issue_key, issue_created = pull_request_opened(pull_request)
if issue_created:
created[pull_request["number"]] = issue_key
logger.info(
"Created {num} JIRA issues on repo {repo}. PRs are {prs}".format(
num=len(created), repo=repo, prs=created.keys(),
),
)
info = {"repo": repo}
if created:
info["created"] = created
return info
def get_jira_issue_key(pull_request):
me = github_whoami()
my_username = me["login"]
comment_url = "/repos/{repo}/issues/{num}/comments".format(
repo=pull_request["base"]["repo"]["full_name"].decode("utf-8"), num=pull_request["number"]
)
for comment in paginated_get(comment_url, session=github_bp.session):
# I only care about comments I made
if comment["user"]["login"] != my_username:
continue
# search for the first occurrance of a JIRA ticket key in the comment body
match = re.search(r"\b([A-Z]{2,}-\d+)\b", comment["body"])
if match:
return match.group(0).decode("utf-8")
return None
def get_jira_issue_key(pull_request):
me = github_whoami()
my_username = me["login"]
comment_url = "/repos/{repo}/issues/{num}/comments".format(
repo=pull_request["base"]["repo"]["full_name"],
num=pull_request["number"],
)
for comment in paginated_get(comment_url, session=github_bp.session):
# I only care about comments I made
if comment["user"]["login"] != my_username:
continue
# search for the first occurrance of a JIRA ticket key in the comment body
match = re.search(r"\b([A-Z]{2,}-\d+)\b", comment["body"])
if match:
return match.group(0)
return None
def rescan():
"""
Re-scan GitHub repositories to find pull requests that need OSPR issues
on JIRA, and do not have them. If this method finds pull requests that are
missing JIRA issues, it will automatically process those pull requests
just as though the pull request was newly created.
Note that this rescan functionality is the reason why
:func:`~openedx_webhooks.tasks.github.pull_request_opened`
must be idempotent. It could run many times over the same pull request.
"""
repo = request.form.get("repo") or "edx/edx-platform"
inline = request.form.get("inline", False)
if repo == 'all' and inline:
return "Don't be silly."
if inline:
return jsonify(rescan_repository(repo))
if repo.startswith('all:'):
org = repo[4:]
org_url = "https://api.github.com/orgs/{org}/repos".format(org=org)
repo_names = [
repo_name['full_name'] for repo_name in paginated_get(org_url)
]
workflow = group(
rescan_repository.s(repository,
wsgi_environ=minimal_wsgi_environ())
for repository in repo_names)
group_result = workflow.delay()
group_result.save() # this is necessary for groups, for some reason
status_url = url_for("tasks.group_status",
group_id=group_result.id,
_external=True)
else:
result = rescan_repository.delay(repo,
wsgi_environ=minimal_wsgi_environ())
status_url = url_for("tasks.status", task_id=result.id, _external=True)
resp = jsonify({"message": "queued", "status_url": status_url})
resp.status_code = 202
resp.headers["Location"] = status_url
return resp
def has_contractor_comment(pull_request):
"""
Given a pull request, this function returns a boolean indicating whether
we have already left a comment on that pull request that suggests
making an OSPR issue for the pull request.
"""
me = github_whoami()
my_username = me["login"]
comment_url = "/repos/{repo}/issues/{num}/comments".format(
repo=pull_request["base"]["repo"]["full_name"].decode("utf-8"), num=pull_request["number"]
)
for comment in paginated_get(comment_url, session=github_bp.session):
# I only care about comments I made
if comment["user"]["login"] != my_username:
continue
magic_phrase = "It looks like you're a member of a company that does contract work for edX."
if magic_phrase in comment["body"]:
return True
return False
def has_contractor_comment(pull_request):
"""
Given a pull request, this function returns a boolean indicating whether
we have already left a comment on that pull request that suggests
making an OSPR issue for the pull request.
"""
me = github_whoami()
my_username = me["login"]
comment_url = "/repos/{repo}/issues/{num}/comments".format(
repo=pull_request["base"]["repo"]["full_name"],
num=pull_request["number"],
)
for comment in paginated_get(comment_url, session=github_bp.session):
# I only care about comments I made
if comment["user"]["login"] != my_username:
continue
magic_phrase = "It looks like you're a member of a company that does contract work for edX."
if magic_phrase in comment["body"]:
return True
return False
def has_internal_cover_letter(pull_request):
"""
Given a pull request, this function returns a boolean indicating whether
the body has already been replaced with the cover letter template.
"""
me = github_whoami()
my_username = me["login"]
comment_url = "/repos/{repo}/issues/{num}/comments".format(
repo=pull_request["base"]["repo"]["full_name"],
num=pull_request["number"],
)
for comment in paginated_get(comment_url, session=github_bp.session):
# I only care about comments I made
if comment["user"]["login"] != my_username:
continue
body = comment["body"]
if COVERLETTER_MARKER in body:
return True
return False
def has_internal_cover_letter(pull_request):
"""
Given a pull request, this function returns a boolean indicating whether
the body has already been replaced with the cover letter template.
"""
me = github_whoami()
my_username = me["login"]
comment_url = "/repos/{repo}/issues/{num}/comments".format(
repo=pull_request["base"]["repo"]["full_name"].decode('utf-8'),
num=pull_request["number"],
)
for comment in paginated_get(comment_url, session=github_bp.session):
# I only care about comments I made
if comment["user"]["login"] != my_username:
continue
body = comment["body"].decode('utf-8')
if COVERLETTER_MARKER in body:
return True
return False