def rescan_repo(repo): """ rescans a single repo for new prs """ bugsnag_context = {"repo": repo} bugsnag.configure_request(meta_data=bugsnag_context) url = "/repos/{repo}/pulls".format(repo=repo) created = {} for pull_request in paginated_get(url, session=github): bugsnag_context["pull_request"] = pull_request bugsnag.configure_request(meta_data=bugsnag_context) if not get_jira_issue_key(pull_request) and not is_internal_pull_request(pull_request): text = pr_opened(pull_request, bugsnag_context=bugsnag_context) if "created" in text: jira_key = text[8:] created[pull_request["number"]] = jira_key print( "Created {num} JIRA issues on repo {repo}. PRs are {prs}".format( num=len(created), repo=repo, prs=created.keys(), ), file=sys.stderr ) return created
def rescan_repository(self, repo): """ rescans a single repo for new prs """ github = github_bp.session sentry_extra_context({"repo": repo}) url = "/repos/{repo}/pulls".format(repo=repo) created = {} if not self.request.called_directly: self.update_state(state='STARTED', meta={'repo': repo}) def page_callback(response): if not response.ok or self.request.called_directly: return current_url = URLObject(response.url) current_page = int(current_url.query_dict.get("page", 1)) link_last = response.links.get("last") if link_last: last_url = URLObject(link_last['url']) last_page = int(last_url.query_dict["page"]) else: last_page = current_page state_meta = { "repo": repo, "current_page": current_page, "last_page": last_page } self.update_state(state='STARTED', meta=state_meta) for pull_request in paginated_get(url, session=github, callback=page_callback): sentry_extra_context({"pull_request": pull_request}) issue_key = get_jira_issue_key(pull_request) is_internal = is_internal_pull_request(pull_request) if not issue_key and not is_internal: # `pull_request_opened()` is a celery task, but by calling it as # a function instead of calling `.delay()` on it, we're eagerly # executing the task now, instead of adding it to the task queue # so it is executed later. As a result, this will return the values # that the `pull_request_opened()` function returns, rather than # return an AsyncResult object. issue_key, issue_created = pull_request_opened(pull_request) # pylint: disable=no-value-for-parameter if issue_created: created[pull_request["number"]] = issue_key logger.info( "Created {num} JIRA issues on repo {repo}. PRs are {prs}".format( num=len(created), repo=repo, prs=created.keys(), ), ) info = {"repo": repo} if created: info["created"] = created return info
def rescan_repository(self, repo): """ rescans a single repo for new prs """ github = github_bp.session sentry.client.extra_context({"repo": repo}) url = "/repos/{repo}/pulls".format(repo=repo) created = {} if not self.request.called_directly: self.update_state(state='STARTED', meta={'repo': repo}) def page_callback(response): if not response.ok or self.request.called_directly: return current_url = URLObject(response.url) current_page = int(current_url.query_dict.get("page", 1)) link_last = response.links.get("last") if link_last: last_url = URLObject(link_last['url']) last_page = int(last_url.query_dict["page"]) else: last_page = current_page state_meta = { "repo": repo, "current_page": current_page, "last_page": last_page } self.update_state(state='STARTED', meta=state_meta) for pull_request in paginated_get(url, session=github, callback=page_callback): sentry.client.extra_context({"pull_request": pull_request}) issue_key = get_jira_issue_key(pull_request) is_internal = is_internal_pull_request(pull_request) if not issue_key and not is_internal: # `pull_request_opened()` is a celery task, but by calling it as # a function instead of calling `.delay()` on it, we're eagerly # executing the task now, instead of adding it to the task queue # so it is executed later. As a result, this will return the values # that the `pull_request_opened()` function returns, rather than # return an AsyncResult object. issue_key, issue_created = pull_request_opened(pull_request) if issue_created: created[pull_request["number"]] = issue_key logger.info( "Created {num} JIRA issues on repo {repo}. PRs are {prs}".format( num=len(created), repo=repo, prs=created.keys(), ), ) info = {"repo": repo} if created: info["created"] = created return info
def test_never_heard_of_you(self, betamax_session): pr = make_pull_request("some_random_guy") assert not is_internal_pull_request(pr, session=betamax_session)
def test_ex_edx_employee_old_pr(self, betamax_session): created_at = datetime(2014, 1, 1) pr = make_pull_request("mmprandom", created_at=created_at) assert is_internal_pull_request(pr, session=betamax_session)
def test_ex_edx_employee(self, betamax_session): pr = make_pull_request("mmprandom") assert not is_internal_pull_request(pr, session=betamax_session)
def test_ex_edx_employee(): pr = make_pull_request("mmprandom") assert not is_internal_pull_request(pr)
def test_left_but_still_a_fan(): pr = make_pull_request("jarv") assert not is_internal_pull_request(pr)
def test_never_heard_of_you(): pr = make_pull_request("some_random_guy") assert not is_internal_pull_request(pr)
def test_committers(self, betamax_session): pr = make_pull_request("antoviaque") assert is_internal_pull_request(pr, session=betamax_session)
def pr_opened(pr, ignore_internal=True, check_contractor=True, bugsnag_context=None): bugsnag_context = bugsnag_context or {} user = pr["user"]["login"].decode('utf-8') repo = pr["base"]["repo"]["full_name"] num = pr["number"] if ignore_internal and is_internal_pull_request(pr): # not an open source pull request, don't create an issue for it print( "@{user} opened PR #{num} against {repo} (internal PR)".format( user=user, repo=repo, num=num, ), file=sys.stderr ) return "internal pull request" if check_contractor and is_contractor_pull_request(pr): # don't create a JIRA issue, but leave a comment comment = { "body": github_contractor_pr_comment(pr), } url = "/repos/{repo}/issues/{num}/comments".format( repo=repo, num=num, ) comment_resp = github.post(url, json=comment) comment_resp.raise_for_status() return "contractor pull request" issue_key = get_jira_issue_key(pr) if issue_key: msg = "Already created {key} for PR #{num} against {repo}".format( key=issue_key, num=pr["number"], repo=pr["base"]["repo"]["full_name"], ) print(msg, file=sys.stderr) return msg repo = pr["base"]["repo"]["full_name"].decode('utf-8') people = get_people_file() custom_fields = get_jira_custom_fields() if user in people: user_name = people[user].get("name", "") else: user_resp = github.get(pr["user"]["url"]) if user_resp.ok: user_name = user_resp.json().get("name", user) else: user_name = user # create an issue on JIRA! new_issue = { "fields": { "project": { "key": "OSPR", }, "issuetype": { "name": "Pull Request Review", }, "summary": pr["title"], "description": pr["body"], custom_fields["URL"]: pr["html_url"], custom_fields["PR Number"]: pr["number"], custom_fields["Repo"]: pr["base"]["repo"]["full_name"], custom_fields["Contributor Name"]: user_name, } } institution = people.get(user, {}).get("institution", None) if institution: new_issue["fields"][custom_fields["Customer"]] = [institution] bugsnag_context["new_issue"] = new_issue bugsnag.configure_request(meta_data=bugsnag_context) resp = jira.post("/rest/api/2/issue", json=new_issue) resp.raise_for_status() new_issue_body = resp.json() issue_key = new_issue_body["key"].decode('utf-8') bugsnag_context["new_issue"]["key"] = issue_key bugsnag.configure_request(meta_data=bugsnag_context) # add a comment to the Github pull request with a link to the JIRA issue comment = { "body": github_community_pr_comment(pr, new_issue_body, people), } url = "/repos/{repo}/issues/{num}/comments".format( repo=repo, num=pr["number"], ) comment_resp = github.post(url, json=comment) comment_resp.raise_for_status() # Add the "Needs Triage" label to the PR issue_url = "/repos/{repo}/issues/{num}".format(repo=repo, num=pr["number"]) label_resp = github.patch(issue_url, data=json.dumps({"labels": ["needs triage", "open-source-contribution"]})) label_resp.raise_for_status() print( "@{user} opened PR #{num} against {repo}, created {issue} to track it".format( user=user, repo=repo, num=pr["number"], issue=issue_key, ), file=sys.stderr ) return "created {key}".format(key=issue_key)
def pull_request_opened(pull_request, ignore_internal=True, check_contractor=True): """ Process a pull request. This is called when a pull request is opened, or when the pull requests of a repo are re-scanned. By default, this function will ignore internal pull requests, and will add a comment to pull requests made by contractors (if if has not yet added a comment). However, this function can be called in such a way that it processes those pull requests anyway. This function must be idempotent. Every time the repositories are re-scanned, this function will be called for pull requests that have already been opened. As a result, it should not comment on the pull request without checking to see if it has *already* commented on the pull request. Returns a 2-tuple. The first element in the tuple is the key of the JIRA issue associated with the pull request, if any, as a string. The second element in the tuple is a boolean indicating if this function did any work, such as making a JIRA issue or commenting on the pull request. """ github = github_bp.session pr = pull_request user = pr["user"]["login"].decode('utf-8') repo = pr["base"]["repo"]["full_name"] num = pr["number"] is_internal_pr = is_internal_pull_request(pr) has_cl = has_internal_cover_letter(pr) is_beta = is_beta_tester_pull_request(pr) if is_internal_pr and not has_cl and is_beta: logger.info( "Adding cover letter template to PR #{num} against {repo}".format( repo=repo, num=num, ), ) comment = { "body": github_internal_cover_letter(pr), } url = "/repos/{repo}/issues/{num}/comments".format( repo=repo, num=num, ) comment_resp = github.post(url, json=comment) comment_resp.raise_for_status() if ignore_internal and is_internal_pr: # not an open source pull request, don't create an issue for it logger.info( "@{user} opened PR #{num} against {repo} (internal PR)".format( user=user, repo=repo, num=num, ), ) return None, False if check_contractor and is_contractor_pull_request(pr): # have we already left a contractor comment? if has_contractor_comment(pr): return None, False # don't create a JIRA issue, but leave a comment comment = { "body": github_contractor_pr_comment(pr), } url = "/repos/{repo}/issues/{num}/comments".format( repo=repo, num=num, ) comment_resp = github.post(url, json=comment) comment_resp.raise_for_status() return None, True issue_key = get_jira_issue_key(pr) if issue_key: msg = "Already created {key} for PR #{num} against {repo}".format( key=issue_key, num=pr["number"], repo=pr["base"]["repo"]["full_name"], ) logger.info(msg) return issue_key, False repo = pr["base"]["repo"]["full_name"].decode('utf-8') people = get_people_file() custom_fields = get_jira_custom_fields(jira_bp.session) user_name = None if user in people: user_name = people[user].get("name", "") if not user_name: user_resp = github.get(pr["user"]["url"]) if user_resp.ok: user_name = user_resp.json().get("name", user) else: user_name = user # create an issue on JIRA! new_issue = { "fields": { "project": { "key": "OSPR", }, "issuetype": { "name": "Pull Request Review", }, "summary": pr["title"], "description": pr["body"], custom_fields["URL"]: pr["html_url"], custom_fields["PR Number"]: pr["number"], custom_fields["Repo"]: pr["base"]["repo"]["full_name"], custom_fields["Contributor Name"]: user_name, } } institution = people.get(user, {}).get("institution", None) if institution: new_issue["fields"][custom_fields["Customer"]] = [institution] sentry.client.extra_context({"new_issue": new_issue}) resp = jira_bp.session.post("/rest/api/2/issue", json=new_issue) resp.raise_for_status() new_issue_body = resp.json() issue_key = new_issue_body["key"].decode('utf-8') new_issue["key"] = issue_key sentry.client.extra_context({"new_issue": new_issue}) # add a comment to the Github pull request with a link to the JIRA issue comment = { "body": github_community_pr_comment(pr, new_issue_body, people), } url = "/repos/{repo}/issues/{num}/comments".format( repo=repo, num=pr["number"], ) comment_resp = github.post(url, json=comment) comment_resp.raise_for_status() # Add the "Needs Triage" label to the PR issue_url = "/repos/{repo}/issues/{num}".format(repo=repo, num=pr["number"]) label_resp = github.patch(issue_url, data=json.dumps({"labels": ["needs triage", "open-source-contribution"]})) label_resp.raise_for_status() logger.info( "@{user} opened PR #{num} against {repo}, created {issue} to track it".format( user=user, repo=repo, num=pr["number"], issue=issue_key, ), ) return issue_key, True
def test_committers(): pr = make_pull_request("antoviaque") assert is_internal_pull_request(pr)
def test_hourly_worker(): pr = make_pull_request("theJohnnyBrown") assert not is_internal_pull_request(pr)
def test_hourly_worker(self, betamax_session): pr = make_pull_request("theJohnnyBrown") assert not is_internal_pull_request(pr, session=betamax_session)
def test_left_but_still_a_fan(self, betamax_session): pr = make_pull_request("jarv") assert not is_internal_pull_request(pr, session=betamax_session)
def test_edx_employee(): pr = make_pull_request("nedbat") assert is_internal_pull_request(pr)
def test_ex_edx_employee_old_pr(): created_at = datetime(2014, 1, 1) pr = make_pull_request("mmprandom", created_at=created_at) assert is_internal_pull_request(pr)
def test_ex_edx_employee_old_pr(): created_at = datetime(2014, 1, 1) pr = make_pull_request("jarv", created_at=created_at) assert is_internal_pull_request(pr)
def pull_request_opened(self, pull_request, ignore_internal=True, check_contractor=True): """ Process a pull request. This is called when a pull request is opened, or when the pull requests of a repo are re-scanned. By default, this function will ignore internal pull requests (unless a repo has supplied .pr_cover_letter.md.j2), and will add a comment to pull requests made by contractors (if if has not yet added a comment). However, this function can be called in such a way that it processes those pull requests anyway. This function must be idempotent. Every time the repositories are re-scanned, this function will be called for pull requests that have already been opened. As a result, it should not comment on the pull request without checking to see if it has *already* commented on the pull request. Returns a 2-tuple. The first element in the tuple is the key of the JIRA issue associated with the pull request, if any, as a string. The second element in the tuple is a boolean indicating if this function did any work, such as making a JIRA issue or commenting on the pull request. """ # Environment variable containing the Open edX release name open_edx_release = os.environ.get('OPENEDX_RELEASE_NAME') # Environment variable containing a string of comma separated Github usernames for testing test_open_edx_release = os.environ.get( 'GITHUB_USERS_CHERRY_PICK_MESSAGE_TEST') #test_open_edx_release = 'mduboseedx,nedbat,fakeuser' github = github_bp.session pr = pull_request user = pr["user"]["login"] repo = pr["base"]["repo"]["full_name"] num = pr["number"] is_internal_pr = is_internal_pull_request(pr) has_cl = has_internal_cover_letter(pr) is_beta = is_beta_tester_pull_request(pr) msg = "Processing {} PR #{} by {}...".format(repo, num, user) log_info(self.request, msg) if is_bot_pull_request(pr): # Bots never need OSPR attention. return None, False if is_internal_pr and not has_cl and is_beta: msg = "Adding cover letter to PR #{num} against {repo}".format( repo=repo, num=num) log_info(self.request, msg) coverletter = github_internal_cover_letter(pr) if coverletter is not None: comment = {"body": coverletter} url = "/repos/{repo}/issues/{num}/comments".format(repo=repo, num=num) comment_resp = github.post(url, json=comment) log_request_response(self.request, comment_resp) comment_resp.raise_for_status() if ignore_internal and is_internal_pr: # not an open source pull request, don't create an issue for it msg = "@{user} opened PR #{num} against {repo} (internal PR)".format( user=user, repo=repo, num=num) log_info(self.request, msg) # new release candidate for Open edX is available, ask internal PR if should be cherry picked do_cherry_pick_comment = False if open_edx_release: do_cherry_pick_comment = True release_message = open_edx_release elif test_open_edx_release: if user in test_open_edx_release.split(','): do_cherry_pick_comment = True release_message = "Test Release" if do_cherry_pick_comment: github_post_cherry_pick_comment(self, github, pr, release_message) return None, True return None, False if check_contractor and is_contractor_pull_request(pr): # have we already left a contractor comment? if has_contractor_comment(pr): msg = "Already left contractor comment for PR #{}".format(num) log_info(self.request, msg) return None, False # don't create a JIRA issue, but leave a comment comment = { "body": github_contractor_pr_comment(pr), } url = "/repos/{repo}/issues/{num}/comments".format(repo=repo, num=num) msg = "Posting contractor comment to PR #{}".format(num) log_info(self.request, msg) comment_resp = github.post(url, json=comment) log_request_response(self.request, comment_resp) comment_resp.raise_for_status() return None, True issue_key = get_jira_issue_key(pr) if issue_key: msg = "Already created {key} for PR #{num} against {repo}".format( key=issue_key, num=pr["number"], repo=pr["base"]["repo"]["full_name"], ) log_info(self.request, msg) return issue_key, False repo = pr["base"]["repo"]["full_name"] people = get_people_file() custom_fields = get_jira_custom_fields(jira_bp.session) user_name = None if user in people: user_name = people[user].get("name", "") if not user_name: user_resp = github.get(pr["user"]["url"]) if user_resp.ok: user_name = user_resp.json().get("name", user) else: user_name = user # create an issue on JIRA! new_issue = { "fields": { "project": { "key": "OSPR", }, "issuetype": { "name": "Pull Request Review", }, "summary": pr["title"], "description": pr["body"], "customfield_10904": pr["html_url"], # "URL" is ambiguous, use the internal name. custom_fields["PR Number"]: pr["number"], custom_fields["Repo"]: pr["base"]["repo"]["full_name"], custom_fields["Contributor Name"]: user_name, } } institution = people.get(user, {}).get("institution", None) if institution: new_issue["fields"][custom_fields["Customer"]] = [institution] sentry_extra_context({"new_issue": new_issue}) log_info(self.request, 'Creating new JIRA issue...') resp = jira_bp.session.post("/rest/api/2/issue", json=new_issue) log_request_response(self.request, resp) resp.raise_for_status() new_issue_body = resp.json() issue_key = new_issue_body["key"] new_issue["key"] = issue_key sentry_extra_context({"new_issue": new_issue}) # add a comment to the Github pull request with a link to the JIRA issue comment = { "body": github_community_pr_comment(pr, new_issue_body, people), } url = "/repos/{repo}/issues/{num}/comments".format(repo=repo, num=pr["number"]) log_info(self.request, 'Creating new GitHub comment with JIRA issue...') comment_resp = github.post(url, json=comment) log_request_response(self.request, comment_resp) comment_resp.raise_for_status() # Add the "Needs Triage" label to the PR issue_url = "/repos/{repo}/issues/{num}".format(repo=repo, num=pr["number"]) labels = {'labels': ['needs triage', 'open-source-contribution']} log_info(self.request, 'Updating GitHub labels...') label_resp = github.patch(issue_url, data=json.dumps(labels)) log_request_response(self.request, label_resp) label_resp.raise_for_status() msg = "@{user} opened PR #{num} against {repo}, created {issue} to track it".format( user=user, repo=repo, num=pr["number"], issue=issue_key, ) log_info(self.request, msg) return issue_key, True
def test_edx_employee(self, betamax_session): pr = make_pull_request("nedbat") assert is_internal_pull_request(pr, session=betamax_session)