def github_community_pr_comment(pull_request, jira_issue, people=None): """ For a newly-created pull request from an open source contributor, write a welcoming comment on the pull request. The comment should: * contain a link to the JIRA issue * check for contributor agreement * contain a link to our process documentation """ people = people or get_people_file() people = {user.lower(): values for user, values in people.items()} pr_author = pull_request["user"]["login"].lower() created_at = parse_date(pull_request["created_at"]).replace(tzinfo=None) # does the user have a valid, signed contributor agreement? has_signed_agreement = ( pr_author in people and people[pr_author].get("expires_on", date.max) > created_at.date()) return render_template( "github_community_pr_comment.md.j2", user=pull_request["user"]["login"], repo=pull_request["base"]["repo"]["full_name"], number=pull_request["number"], issue_key=jira_issue["key"], has_signed_agreement=has_signed_agreement, )
def check_contributors(): """ Identify missing contributors: people who have commits in a repository, but who are not listed in the AUTHORS file. """ repo = request.form.get("repo", "") if repo: repos = (repo, ) else: repos = get_repos_file().keys() people = get_people_file() people_lower = {username.lower() for username in people.keys()} missing_contributors = defaultdict(set) for repo in repos: sentry.client.extra_context({"repo": repo}) contributors_url = "/repos/{repo}/contributors".format(repo=repo) contributors = paginated_get(contributors_url, session=github) for contributor in contributors: if contributor["login"].lower() not in people_lower: missing_contributors[repo].add(contributor["login"]) # convert sets to lists, so jsonify can handle them output = { repo: list(contributors) for repo, contributors in missing_contributors.items() } return jsonify(output)
def github_check_contributors(): if request.method == "GET": return render_template("github_check_contributors.html") repo = request.form.get("repo", "") if repo: repos = (repo,) else: repos = get_repos_file().keys() people = get_people_file() people_lower = {username.lower() for username in people.keys()} missing_contributors = defaultdict(set) for repo in repos: bugsnag_context = {"repo": repo} bugsnag.configure_request(meta_data=bugsnag_context) contributors_url = "/repos/{repo}/contributors".format(repo=repo) contributors = paginated_get(contributors_url, session=github) for contributor in contributors: if contributor["login"].lower() not in people_lower: missing_contributors[repo].add(contributor["login"]) # convert sets to lists, so jsonify can handle them output = { repo: list(contributors) for repo, contributors in missing_contributors.items() } return jsonify(output)
def check_contributors(): """ Identify missing contributors: people who have commits in a repository, but who are not listed in the AUTHORS file. """ repo = request.form.get("repo", "") if repo: repos = (repo,) else: repos = get_repos_file().keys() people = get_people_file() people_lower = {username.lower() for username in people.keys()} missing_contributors = defaultdict(set) for repo in repos: sentry.client.extra_context({"repo": repo}) contributors_url = "/repos/{repo}/contributors".format(repo=repo) contributors = paginated_get(contributors_url, session=github) for contributor in contributors: if contributor["login"].lower() not in people_lower: missing_contributors[repo].add(contributor["login"]) # convert sets to lists, so jsonify can handle them output = { repo: list(contributors) for repo, contributors in missing_contributors.items() } return jsonify(output)
def github_community_pr_comment(pull_request, jira_issue, people=None): """ For a newly-created pull request from an open source contributor, write a welcoming comment on the pull request. The comment should: * contain a link to the JIRA issue * check for contributor agreement * check for AUTHORS entry * contain a link to our process documentation """ github = github_bp.session people = people or get_people_file() people = {user.lower(): values for user, values in people.items()} pr_author = pull_request["user"]["login"].decode('utf-8').lower() created_at = parse_date(pull_request["created_at"]).replace(tzinfo=None) # does the user have a valid, signed contributor agreement? has_signed_agreement = ( pr_author in people and people[pr_author].get("expires_on", date.max) > created_at.date() ) # is the user in the AUTHORS file? in_authors_file = False name = people.get(pr_author, {}).get("name", "") if name: authors_url = "https://raw.githubusercontent.com/{repo}/{branch}/AUTHORS".format( repo=pull_request["head"]["repo"]["full_name"].decode('utf-8'), branch=pull_request["head"]["ref"].decode('utf-8'), ) authors_resp = github.get(authors_url) if authors_resp.ok: authors_content = authors_resp.text if name in authors_content: in_authors_file = True return render_template("github_community_pr_comment.md.j2", user=pull_request["user"]["login"].decode('utf-8'), repo=pull_request["base"]["repo"]["full_name"].decode('utf-8'), number=pull_request["number"], issue_key=jira_issue["key"].decode('utf-8'), has_signed_agreement=has_signed_agreement, in_authors_file=in_authors_file, )
def test_updated_person(): people = get_people_file() created_at = datetime(2014, 1, 1) updated_person = get_person_certain_time(people["raisingarizona"], created_at) assert updated_person["agreement"] == "individual"
def test_updated_person_has_institution(): people = get_people_file() created_at = datetime(2014, 1, 1) updated_person = get_person_certain_time(people["jarv"], created_at) assert updated_person["institution"] == "edX"
def test_current_person(): people = get_people_file() created_at = datetime.today() current_person = get_person_certain_time(people["raisingarizona"], created_at) assert current_person["agreement"] == "none"
def test_current_person_no_institution(): people = get_people_file() created_at = datetime.today() current_person = get_person_certain_time(people["jarv"], created_at) assert "institution" not in current_person assert current_person["agreement"] == "individual"
def pull_request_opened(self, pull_request, ignore_internal=True, check_contractor=True): """ Process a pull request. This is called when a pull request is opened, or when the pull requests of a repo are re-scanned. By default, this function will ignore internal pull requests (unless a repo has supplied .pr_cover_letter.md.j2), and will add a comment to pull requests made by contractors (if if has not yet added a comment). However, this function can be called in such a way that it processes those pull requests anyway. This function must be idempotent. Every time the repositories are re-scanned, this function will be called for pull requests that have already been opened. As a result, it should not comment on the pull request without checking to see if it has *already* commented on the pull request. Returns a 2-tuple. The first element in the tuple is the key of the JIRA issue associated with the pull request, if any, as a string. The second element in the tuple is a boolean indicating if this function did any work, such as making a JIRA issue or commenting on the pull request. """ # Environment variable containing the Open edX release name open_edx_release = os.environ.get('OPENEDX_RELEASE_NAME') # Environment variable containing a string of comma separated Github usernames for testing test_open_edx_release = os.environ.get( 'GITHUB_USERS_CHERRY_PICK_MESSAGE_TEST') #test_open_edx_release = 'mduboseedx,nedbat,fakeuser' github = github_bp.session pr = pull_request user = pr["user"]["login"] repo = pr["base"]["repo"]["full_name"] num = pr["number"] is_internal_pr = is_internal_pull_request(pr) has_cl = has_internal_cover_letter(pr) is_beta = is_beta_tester_pull_request(pr) msg = "Processing {} PR #{} by {}...".format(repo, num, user) log_info(self.request, msg) if is_bot_pull_request(pr): # Bots never need OSPR attention. return None, False if is_internal_pr and not has_cl and is_beta: msg = "Adding cover letter to PR #{num} against {repo}".format( repo=repo, num=num) log_info(self.request, msg) coverletter = github_internal_cover_letter(pr) if coverletter is not None: comment = {"body": coverletter} url = "/repos/{repo}/issues/{num}/comments".format(repo=repo, num=num) comment_resp = github.post(url, json=comment) log_request_response(self.request, comment_resp) comment_resp.raise_for_status() if ignore_internal and is_internal_pr: # not an open source pull request, don't create an issue for it msg = "@{user} opened PR #{num} against {repo} (internal PR)".format( user=user, repo=repo, num=num) log_info(self.request, msg) # new release candidate for Open edX is available, ask internal PR if should be cherry picked do_cherry_pick_comment = False if open_edx_release: do_cherry_pick_comment = True release_message = open_edx_release elif test_open_edx_release: if user in test_open_edx_release.split(','): do_cherry_pick_comment = True release_message = "Test Release" if do_cherry_pick_comment: github_post_cherry_pick_comment(self, github, pr, release_message) return None, True return None, False if check_contractor and is_contractor_pull_request(pr): # have we already left a contractor comment? if has_contractor_comment(pr): msg = "Already left contractor comment for PR #{}".format(num) log_info(self.request, msg) return None, False # don't create a JIRA issue, but leave a comment comment = { "body": github_contractor_pr_comment(pr), } url = "/repos/{repo}/issues/{num}/comments".format(repo=repo, num=num) msg = "Posting contractor comment to PR #{}".format(num) log_info(self.request, msg) comment_resp = github.post(url, json=comment) log_request_response(self.request, comment_resp) comment_resp.raise_for_status() return None, True issue_key = get_jira_issue_key(pr) if issue_key: msg = "Already created {key} for PR #{num} against {repo}".format( key=issue_key, num=pr["number"], repo=pr["base"]["repo"]["full_name"], ) log_info(self.request, msg) return issue_key, False repo = pr["base"]["repo"]["full_name"] people = get_people_file() custom_fields = get_jira_custom_fields(jira_bp.session) user_name = None if user in people: user_name = people[user].get("name", "") if not user_name: user_resp = github.get(pr["user"]["url"]) if user_resp.ok: user_name = user_resp.json().get("name", user) else: user_name = user # create an issue on JIRA! new_issue = { "fields": { "project": { "key": "OSPR", }, "issuetype": { "name": "Pull Request Review", }, "summary": pr["title"], "description": pr["body"], "customfield_10904": pr["html_url"], # "URL" is ambiguous, use the internal name. custom_fields["PR Number"]: pr["number"], custom_fields["Repo"]: pr["base"]["repo"]["full_name"], custom_fields["Contributor Name"]: user_name, } } institution = people.get(user, {}).get("institution", None) if institution: new_issue["fields"][custom_fields["Customer"]] = [institution] sentry_extra_context({"new_issue": new_issue}) log_info(self.request, 'Creating new JIRA issue...') resp = jira_bp.session.post("/rest/api/2/issue", json=new_issue) log_request_response(self.request, resp) resp.raise_for_status() new_issue_body = resp.json() issue_key = new_issue_body["key"] new_issue["key"] = issue_key sentry_extra_context({"new_issue": new_issue}) # add a comment to the Github pull request with a link to the JIRA issue comment = { "body": github_community_pr_comment(pr, new_issue_body, people), } url = "/repos/{repo}/issues/{num}/comments".format(repo=repo, num=pr["number"]) log_info(self.request, 'Creating new GitHub comment with JIRA issue...') comment_resp = github.post(url, json=comment) log_request_response(self.request, comment_resp) comment_resp.raise_for_status() # Add the "Needs Triage" label to the PR issue_url = "/repos/{repo}/issues/{num}".format(repo=repo, num=pr["number"]) labels = {'labels': ['needs triage', 'open-source-contribution']} log_info(self.request, 'Updating GitHub labels...') label_resp = github.patch(issue_url, data=json.dumps(labels)) log_request_response(self.request, label_resp) label_resp.raise_for_status() msg = "@{user} opened PR #{num} against {repo}, created {issue} to track it".format( user=user, repo=repo, num=pr["number"], issue=issue_key, ) log_info(self.request, msg) return issue_key, True
def github_community_pr_comment(pull_request, jira_issue, people=None): """ For a newly-created pull request from an open source contributor, write a welcoming comment on the pull request. The comment should: * contain a link to the JIRA issue * check for contributor agreement * check for AUTHORS entry * contain a link to our process documentation """ people = people or get_people_file() people = {user.lower(): values for user, values in people.items()} pr_author = pull_request["user"]["login"].decode('utf-8').lower() created_at = parse_date(pull_request["created_at"]).replace(tzinfo=None) # does the user have a valid, signed contributor agreement? has_signed_agreement = ( pr_author in people and people[pr_author].get("expires_on", date.max) > created_at.date() ) # is the user in the AUTHORS file? in_authors_file = False name = people.get(pr_author, {}).get("name", "") if name: authors_url = "https://raw.githubusercontent.com/{repo}/{branch}/AUTHORS".format( repo=pull_request["head"]["repo"]["full_name"].decode('utf-8'), branch=pull_request["head"]["ref"].decode('utf-8'), ) authors_resp = github.get(authors_url) if authors_resp.ok: authors_content = authors_resp.text if name in authors_content: in_authors_file = True doc_url = "http://edx-developer-guide.readthedocs.org/en/latest/process/overview.html" issue_key = jira_issue["key"].decode('utf-8') issue_url = "https://openedx.atlassian.net/browse/{key}".format(key=issue_key) contributing_url = "https://github.com/edx/edx-platform/blob/master/CONTRIBUTING.rst" agreement_url = "http://open.edx.org/sites/default/files/wysiwyg/individual-contributor-agreement.pdf" authors_url = "https://github.com/{repo}/blob/master/AUTHORS".format( repo=pull_request["base"]["repo"]["full_name"].decode('utf-8'), ) comment = ( "Thanks for the pull request, @{user}! I've created " "[{issue_key}]({issue_url}) to keep track of it in JIRA. " "JIRA is a place for product owners to prioritize feature reviews " "by the engineering development teams. " "\n\nFeel free to add as much of the following information to the ticket:" "\n- supporting documentation" "\n- edx-code email threads" "\n- timeline information ('this must be merged by XX date', and why that is)" "\n- partner information ('this is a course on edx.org')" "\n- any other information that can help Product understand the context for the PR" "\n\nAll technical communication about the code itself will still be " "done via the Github pull request interface. " "As a reminder, [our process documentation is here]({doc_url})." ).format( user=pull_request["user"]["login"].decode('utf-8'), issue_key=issue_key, issue_url=issue_url, doc_url=doc_url, ) if not has_signed_agreement or not in_authors_file: todo = "" if not has_signed_agreement: todo += ( "submitted a [signed contributor agreement]({agreement_url}) " "or indicated your institutional affiliation" ).format( agreement_url=agreement_url, ) if not has_signed_agreement and not in_authors_file: todo += " and " if not in_authors_file: todo += "added yourself to the [AUTHORS]({authors_url}) file".format( authors_url=authors_url, ) comment += ("\n\n" "We can't start reviewing your pull request until you've {todo}. " "Please see the [CONTRIBUTING]({contributing_url}) file for " "more information." ).format(todo=todo, contributing_url=contributing_url) return comment
def pr_opened(pr, ignore_internal=True, check_contractor=True, bugsnag_context=None): bugsnag_context = bugsnag_context or {} user = pr["user"]["login"].decode('utf-8') repo = pr["base"]["repo"]["full_name"] num = pr["number"] if ignore_internal and is_internal_pull_request(pr): # not an open source pull request, don't create an issue for it print( "@{user} opened PR #{num} against {repo} (internal PR)".format( user=user, repo=repo, num=num, ), file=sys.stderr ) return "internal pull request" if check_contractor and is_contractor_pull_request(pr): # don't create a JIRA issue, but leave a comment comment = { "body": github_contractor_pr_comment(pr), } url = "/repos/{repo}/issues/{num}/comments".format( repo=repo, num=num, ) comment_resp = github.post(url, json=comment) comment_resp.raise_for_status() return "contractor pull request" issue_key = get_jira_issue_key(pr) if issue_key: msg = "Already created {key} for PR #{num} against {repo}".format( key=issue_key, num=pr["number"], repo=pr["base"]["repo"]["full_name"], ) print(msg, file=sys.stderr) return msg repo = pr["base"]["repo"]["full_name"].decode('utf-8') people = get_people_file() custom_fields = get_jira_custom_fields() if user in people: user_name = people[user].get("name", "") else: user_resp = github.get(pr["user"]["url"]) if user_resp.ok: user_name = user_resp.json().get("name", user) else: user_name = user # create an issue on JIRA! new_issue = { "fields": { "project": { "key": "OSPR", }, "issuetype": { "name": "Pull Request Review", }, "summary": pr["title"], "description": pr["body"], custom_fields["URL"]: pr["html_url"], custom_fields["PR Number"]: pr["number"], custom_fields["Repo"]: pr["base"]["repo"]["full_name"], custom_fields["Contributor Name"]: user_name, } } institution = people.get(user, {}).get("institution", None) if institution: new_issue["fields"][custom_fields["Customer"]] = [institution] bugsnag_context["new_issue"] = new_issue bugsnag.configure_request(meta_data=bugsnag_context) resp = jira.post("/rest/api/2/issue", json=new_issue) resp.raise_for_status() new_issue_body = resp.json() issue_key = new_issue_body["key"].decode('utf-8') bugsnag_context["new_issue"]["key"] = issue_key bugsnag.configure_request(meta_data=bugsnag_context) # add a comment to the Github pull request with a link to the JIRA issue comment = { "body": github_community_pr_comment(pr, new_issue_body, people), } url = "/repos/{repo}/issues/{num}/comments".format( repo=repo, num=pr["number"], ) comment_resp = github.post(url, json=comment) comment_resp.raise_for_status() # Add the "Needs Triage" label to the PR issue_url = "/repos/{repo}/issues/{num}".format(repo=repo, num=pr["number"]) label_resp = github.patch(issue_url, data=json.dumps({"labels": ["needs triage", "open-source-contribution"]})) label_resp.raise_for_status() print( "@{user} opened PR #{num} against {repo}, created {issue} to track it".format( user=user, repo=repo, num=pr["number"], issue=issue_key, ), file=sys.stderr ) return "created {key}".format(key=issue_key)
def pull_request_opened(pull_request, ignore_internal=True, check_contractor=True): """ Process a pull request. This is called when a pull request is opened, or when the pull requests of a repo are re-scanned. By default, this function will ignore internal pull requests, and will add a comment to pull requests made by contractors (if if has not yet added a comment). However, this function can be called in such a way that it processes those pull requests anyway. This function must be idempotent. Every time the repositories are re-scanned, this function will be called for pull requests that have already been opened. As a result, it should not comment on the pull request without checking to see if it has *already* commented on the pull request. Returns a 2-tuple. The first element in the tuple is the key of the JIRA issue associated with the pull request, if any, as a string. The second element in the tuple is a boolean indicating if this function did any work, such as making a JIRA issue or commenting on the pull request. """ github = github_bp.session pr = pull_request user = pr["user"]["login"].decode('utf-8') repo = pr["base"]["repo"]["full_name"] num = pr["number"] is_internal_pr = is_internal_pull_request(pr) has_cl = has_internal_cover_letter(pr) is_beta = is_beta_tester_pull_request(pr) if is_internal_pr and not has_cl and is_beta: logger.info( "Adding cover letter template to PR #{num} against {repo}".format( repo=repo, num=num, ), ) comment = { "body": github_internal_cover_letter(pr), } url = "/repos/{repo}/issues/{num}/comments".format( repo=repo, num=num, ) comment_resp = github.post(url, json=comment) comment_resp.raise_for_status() if ignore_internal and is_internal_pr: # not an open source pull request, don't create an issue for it logger.info( "@{user} opened PR #{num} against {repo} (internal PR)".format( user=user, repo=repo, num=num, ), ) return None, False if check_contractor and is_contractor_pull_request(pr): # have we already left a contractor comment? if has_contractor_comment(pr): return None, False # don't create a JIRA issue, but leave a comment comment = { "body": github_contractor_pr_comment(pr), } url = "/repos/{repo}/issues/{num}/comments".format( repo=repo, num=num, ) comment_resp = github.post(url, json=comment) comment_resp.raise_for_status() return None, True issue_key = get_jira_issue_key(pr) if issue_key: msg = "Already created {key} for PR #{num} against {repo}".format( key=issue_key, num=pr["number"], repo=pr["base"]["repo"]["full_name"], ) logger.info(msg) return issue_key, False repo = pr["base"]["repo"]["full_name"].decode('utf-8') people = get_people_file() custom_fields = get_jira_custom_fields(jira_bp.session) user_name = None if user in people: user_name = people[user].get("name", "") if not user_name: user_resp = github.get(pr["user"]["url"]) if user_resp.ok: user_name = user_resp.json().get("name", user) else: user_name = user # create an issue on JIRA! new_issue = { "fields": { "project": { "key": "OSPR", }, "issuetype": { "name": "Pull Request Review", }, "summary": pr["title"], "description": pr["body"], custom_fields["URL"]: pr["html_url"], custom_fields["PR Number"]: pr["number"], custom_fields["Repo"]: pr["base"]["repo"]["full_name"], custom_fields["Contributor Name"]: user_name, } } institution = people.get(user, {}).get("institution", None) if institution: new_issue["fields"][custom_fields["Customer"]] = [institution] sentry.client.extra_context({"new_issue": new_issue}) resp = jira_bp.session.post("/rest/api/2/issue", json=new_issue) resp.raise_for_status() new_issue_body = resp.json() issue_key = new_issue_body["key"].decode('utf-8') new_issue["key"] = issue_key sentry.client.extra_context({"new_issue": new_issue}) # add a comment to the Github pull request with a link to the JIRA issue comment = { "body": github_community_pr_comment(pr, new_issue_body, people), } url = "/repos/{repo}/issues/{num}/comments".format( repo=repo, num=pr["number"], ) comment_resp = github.post(url, json=comment) comment_resp.raise_for_status() # Add the "Needs Triage" label to the PR issue_url = "/repos/{repo}/issues/{num}".format(repo=repo, num=pr["number"]) label_resp = github.patch(issue_url, data=json.dumps({"labels": ["needs triage", "open-source-contribution"]})) label_resp.raise_for_status() logger.info( "@{user} opened PR #{num} against {repo}, created {issue} to track it".format( user=user, repo=repo, num=pr["number"], issue=issue_key, ), ) return issue_key, True