def update_pull_comments(self, pull: PullRequest,
comments_data: List[Dict], dry_run: bool) -> None:
pull_id = pull.number
if dry_run:
print(f"Would update pull {pull_id} comments")
return
num_comments = len(comments_data)
existing_comments = list(pull.get_issue_comments())
# Create or update comments
for comment_num, comment_data in enumerate(comments_data):
print(
f"Set comment {comment_num + 1}/{num_comments} of github pull request #{pull_id}..."
)
comment_body = comment_data["body"]
if comment_num < len(existing_comments):
existing_comments[comment_num].edit(comment_body)
else:
pull.create_comment(comment_body)
# Delete comments in excess
comments_to_delete = existing_comments[num_comments:]
for i, gh_comment in enumerate(comments_to_delete):
print(
f"Delete extra github comment {i + 1}/{len(comments_to_delete)} of pull request #{pull_id}..."
)
gh_comment.delete()
def publish_comment(
self,
title: str,
stats: UnitTestRunResults,
pull_request: PullRequest,
check_run: Optional[CheckRun] = None,
cases: Optional[UnitTestCaseResults] = None) -> PullRequest:
# compare them with earlier stats
base_commit_sha = self.get_base_commit_sha(pull_request)
logger.debug(f'comparing against base={base_commit_sha}')
base_check_run = self.get_check_run(base_commit_sha)
base_stats = self.get_stats_from_check_run(
base_check_run) if base_check_run is not None else None
stats_with_delta = get_stats_delta(
stats, base_stats, 'base') if base_stats is not None else stats
logger.debug(f'stats with delta: {stats_with_delta}')
# gather test lists from check run and cases
before_all_tests, before_skipped_tests = self.get_test_lists_from_check_run(
base_check_run)
all_tests, skipped_tests = get_all_tests_list(
cases), get_skipped_tests_list(cases)
test_changes = SomeTestChanges(before_all_tests, all_tests,
before_skipped_tests, skipped_tests)
logger.info('creating comment')
details_url = check_run.html_url if check_run else None
summary = get_long_summary_md(stats_with_delta, details_url,
test_changes,
self._settings.test_changes_limit)
pull_request.create_issue_comment(f'## {title}\n{summary}')
return pull_request
def from_pull_request(cls, pr: PullRequest) \
-> Union['UpdateRequest', IgnoredRequest]:
"""
Construct from a Github pull request.
:param pr: The pull request.
:return: The constructed update request.
"""
if pr.is_merged():
return IgnoredRequest.merged
if pr.changed_files != 1:
return IgnoredRequest.invalid
# system file after merge
file = pr.get_files()[0]
# construct download url of the original file
orig_file = cls.FILE_URL.format(repo=pr.base.repo.full_name,
commit=pr.base.sha,
file=file.filename)
# retrieve the original and patched files
resp = requests.get(orig_file)
if not resp.ok:
raise ReconstructionError('Unable to retrieve the original file: '
'%d %s' %
(resp.status_code, resp.reason))
original = resp.text
resp = requests.get(file.raw_url)
if not resp.ok:
raise ReconstructionError('Unable to retrieve the original file: '
'%d %s' %
(resp.status_code, resp.reason))
patched = resp.text
# read xml and reconstruct system update object
update = None
try:
adapter = oec.Adapter()
update = data_compare(adapter.read_system(io.StringIO(original)),
adapter.read_system(io.StringIO(patched)))
if update is None:
logging.debug('No changes detected on ' + pr.number)
return IgnoredRequest.invalid
except Exception as e:
logging.debug(e)
return IgnoredRequest.invalid
message, reference = UpdateRequest._parse_description(pr.body)
return UpdateRequest(update,
title=pr.title,
message=message,
reference=reference,
pullreq_num=pr.number,
pullreq_url=pr.html_url,
branch=pr.head.label,
rejected=pr.state == "closed")
def convert_pr(pr: PullRequest) -> GprrPR:
"""
Converts PyGithub pull request representation to GPRR pull request representation
:param pr: PyGithub pull request
:return: GPRR pull request
"""
gprr_pr = GprrPR()
gprr_pr.id = pr.id
gprr_pr.number = pr.number
gprr_pr.url = pr.html_url
gprr_pr.repository = convert_repo(pr.head.repo)
gprr_pr.title = pr.title
gprr_pr.creator = convert_user(pr.user)
gprr_pr.created = pr.created_at
gprr_pr.updated = pr.updated_at
gprr_pr.since_updated = (datetime.today() - gprr_pr.updated).days
gprr_pr.flags.append(GprrPrFlag("Draft", str(pr.draft)))
gprr_pr.flags.append(GprrPrFlag("Mergeable", str(pr.mergeable)))
gprr_pr.flags.append(GprrPrFlag("Mergeable State",
str(pr.mergeable_state)))
gprr_pr.initial_branch = pr.head.ref
for assignee in pr.assignees:
gprr_pr.assignees.append(convert_user(assignee))
for label in pr.get_labels(
): # this makes additional call to Github REST API
gprr_pr.labels.append(convert_label(label))
active_reviewers = []
for review in pr.get_reviews(
): # this makes additional call to Github REST API
reviewer = convert_user(review.user)
if not known_review(active_reviewers, reviewer):
active_reviewers.append(reviewer)
gprr_review = GprrReview(
user=reviewer,
state=review.state,
submitted_at=review.submitted_at,
url=review.html_url,
)
gprr_pr.reviews.append(gprr_review)
pr_review_requests = pr.get_review_requests()
for revusr in pr_review_requests[
0]: # this makes additional call to Github REST API
usr = convert_user(revusr)
review = GprrReview(user=usr, state="PENDING")
gprr_pr.reviews_pending.append(review)
# for revteam in pr_review_requests[1]:
# team_as_user = convert_team_to_user(revteam)
# review = GprrReview(user=team_as_user, state="PENDING")
# gprr_pr.reviews_pending.append(review)
return gprr_pr
def update_or_create_links_comment(self, github_pr: PullRequest,
build_result: CiBuildResult):
github_comment = self._find_bot_pr_comment(github_pr)
if github_comment:
self._update_links_comment(github_pr, github_comment, build_result)
else:
new_bot_comment = self._generate_comment(github_pr, build_result)
github_pr.create_issue_comment(new_bot_comment.to_text(self._settings.comment))
def check_for_changelogs(pr: PullRequest) -> None:
for change in pr.get_files():
if change.filename == 'requirements.txt':
lines = build_changelog(change)
for comment in pr.get_issue_comments():
if comment.body.startswith('# Changelogs'):
# Changelog comment
comment.edit(body='\n'.join(lines))
break
else:
pr.create_issue_comment('\n'.join(lines))
def _merge_pr(self, pr: ghp.PullRequest) -> None:
assert self.repo_options.gh_auto_merge_pr is not None
if pr.update() is True:
if pr.state == 'closed' or not pr.mergeable:
return
if self.repo_options.gh_auto_merge_pr.required_label_name not in [l.name for l in pr.labels]:
return
status = pr.merge()
if status.merged:
print(f'GH:{self.repo_path}: PR#{pr.number} merged.')
else:
raise RuntimeError(f'GH:{self.repo_path}: PR#{pr.number} fail to merge - "{status.message}"!')
def store(self, pull_request: GithubPullRequest):
"""Override :func:`~Entity.store`."""
commits = pull_request.commits
created_at = int(pull_request.created_at.timestamp())
closed_at = int(pull_request.closed_at.timestamp()
) if pull_request.closed_at is not None else None
merged_at = int(pull_request.merged_at.timestamp()
) if pull_request.merged_at is not None else None
closed_by = pull_request.as_issue(
).closed_by.login if pull_request.as_issue(
).closed_by is not None else None
labels = [label.name for label in pull_request.get_labels()]
# Evaluate size of PR
pull_request_size = None
if labels:
pull_request_size = GitHubKnowledge.get_labeled_size(labels)
if not pull_request_size:
lines_changes = pull_request.additions + pull_request.deletions
pull_request_size = GitHubKnowledge.assign_pull_request_size(
lines_changes=lines_changes)
self.stored_entities[str(pull_request.number)] = {
"size":
pull_request_size,
"created_by":
pull_request.user.login,
"created_at":
created_at,
"closed_at":
closed_at,
"closed_by":
closed_by,
"merged_at":
merged_at,
"commits_number":
commits,
"referenced_issues":
PullRequest.get_referenced_issues(pull_request),
"interactions":
GitHubKnowledge.get_interactions(
pull_request.get_issue_comments()),
"reviews":
self.extract_pull_request_reviews(pull_request),
"requested_reviewers":
self.extract_pull_request_review_requests(pull_request),
"labels":
GitHubKnowledge.get_labels(pull_request.as_issue()),
}
def can_merge_pull_request(repo: Repository, pull: PullRequest) -> bool:
if pull.user.login not in USER_WHITELIST:
return False
if pull.mergeable_state != "clean":
return False
if not any(label.name == AUTO_MERGE_LABEL for label in pull.labels):
return False
if not any(review.state == "APPROVED" for review in pull.get_reviews()):
return False
return True
head_commit = repo.get_commit(pull.head.sha)
circle_success = False
percy_success = False
for status in head_commit.get_statuses():
if status.state == "success":
if status.context == "ci/circleci: build":
circle_success = True
elif status.context == "percy/web":
percy_success = True
return circle_success and percy_success
def get_reviews(repo_full_name):
gh = get_gh_client()
ghdb = get_github_db()
search_for = {
'base.repo.full_name': repo_full_name,
}
for page in get_next_page(ghdb.pulls.find(search_for)):
for raw_pull in page:
pull = PullRequest(gh._Github__requester, {},
raw_pull,
completed=True)
for review in pull.get_reviews():
store_document(review._rawData)
wait_for_rate(review)
def hide_orphaned_commit_comments(self, pull: PullRequest) -> None:
# rewriting history of branch removes commits
# we do not want to show test results for those commits anymore
# get commits of this pull request
commit_shas = set([commit.sha for commit in pull.get_commits()])
# get comments of this pull request
comments = self.get_pull_request_comments(pull)
# get all comments that come from this action and are not hidden
comments = self.get_action_comments(comments)
# get comment node ids and their commit sha (possibly abbreviated)
matches = [(comment.get('id'), re.search(r'^[Rr]esults for commit ([0-9a-f]{8,40})\.(?:\s.*)?$', comment.get('body'), re.MULTILINE))
for comment in comments]
comment_commits = [(node_id, match.group(1))
for node_id, match in matches
if match is not None]
# get those comment node ids whose commit is not part of this pull request any more
comment_ids = [(node_id, comment_commit_sha)
for (node_id, comment_commit_sha) in comment_commits
if not any([sha
for sha in commit_shas
if sha.startswith(comment_commit_sha)])]
# hide all those comments
for node_id, comment_commit_sha in comment_ids:
self._logger.info('hiding unit test result comment for commit {}'.format(comment_commit_sha))
self.hide_comment(node_id)
def _get_commit_hashes(pull: PullRequest) -> List[str]:
"""
Get a list of the commit hashes associated with this PR
:param pull:
:return:
"""
return [c.sha for c in pull.get_commits()]
def extract_pull_request_reviews(
pull_request: GithubPullRequest) -> Dict[str, Dict[str, Any]]:
"""Extract required features for each review from PR.
Arguments:
pull_request {PullRequest} -- Pull Request from which the reviews will be extracted
Returns:
Dict[str, Dict[str, Any]] -- dictionary of extracted reviews. Each review is stored
"""
reviews = pull_request.get_reviews()
_LOGGER.debug(" -num of reviews found: %d" % reviews.totalCount)
results = {}
for idx, review in enumerate(reviews, 1):
_LOGGER.info(" -analysing review no. %d/%d" %
(idx, reviews.totalCount))
results[str(review.id)] = {
"author":
review.user.login
if review.user and review.user.login else None,
"words_count":
len(review.body.split(" ")),
"submitted_at":
int(review.submitted_at.timestamp()),
"state":
review.state,
}
return results
def check_pr_for_mergability(pr: PullRequest) -> str:
repo = pr.base.repo
commit = repo.get_commit(pr.head.sha)
checks: Dict[str, str] = {}
for status in commit.get_statuses():
print(status)
if status.context == PDM_CHECK_CONTEXT:
continue
if checks.get(status.context) is None:
checks[status.context] = status.state
if status.state != 'success':
commit.create_status(state='pending', description=f'Waiting for {status.context}', context=PDM_CHECK_CONTEXT)
return f'Merge blocked by {status.context}'
travis_pr = 'continuous-integration/travis-ci/pr'
if travis_pr not in checks.keys():
# There's a lovely race condition where, if:
# 1. travis/push has completed before the PR was made
# 2. And the label is applied on creation (or author is whitelisted)
# The PR can be merged before travis is aware of the PR.
# The solution to this is to hardcode a check for /pr
commit.create_status(state='pending', description=f'Waiting for {travis_pr}', context=PDM_CHECK_CONTEXT)
return f'Merge blocked by {travis_pr}'
labels = [l.name for l in pr.as_issue().labels]
if 'do not merge' in labels:
commit.create_status(state='failure', description='Blocked by "do not merge"', context=PDM_CHECK_CONTEXT)
return 'Do not Merge'
whitelisted = pr.user in repo.get_collaborators()
if not whitelisted and not 'merge when ready' in labels:
commit.create_status(state='pending', description='Waiting for "merge when ready"', context=PDM_CHECK_CONTEXT)
return 'Waiting for label'
if 'beta test' in labels:
trying = repo.get_git_ref('heads/trying')
if trying.object.sha == commit.sha:
commit.create_status(state='success', description='Deployed to test branch', context=PDM_CHECK_CONTEXT)
return 'Already deployed'
trying.edit(commit.sha, True)
commit.create_status(state='success', description='Deployed to test branch', context=PDM_CHECK_CONTEXT)
return 'beta test'
commit.create_status(state='success', description='Ready to merge', context=PDM_CHECK_CONTEXT)
pr.merge()
return 'good to merge'
def publish_comment(self,
title: str,
stats: UnitTestRunResults,
pull_request: PullRequest,
check_run: Optional[CheckRun] = None) -> None:
# compare them with earlier stats
base_commit_sha = pull_request.base.sha if pull_request else None
self._logger.debug('comparing against base={}'.format(base_commit_sha))
base_stats = self.get_stats_from_commit(base_commit_sha)
stats_with_delta = get_stats_delta(
stats, base_stats, 'base') if base_stats is not None else stats
self._logger.debug('stats with delta: {}'.format(stats_with_delta))
self._logger.info('creating comment')
details_url = check_run.html_url if check_run else None
pull_request.create_issue_comment('## {}\n{}'.format(
title, get_long_summary_md(stats_with_delta, details_url)))
return pull_request
def _get_approved_reviews(pull: PullRequest) -> List[dict]:
"""
Get a list of the approved reviews
:param pull:
:return:
"""
return [{
"username": r.user.login,
"state": r.state
} for r in pull.get_reviews() if r.state == "APPROVED"]
def update_pr(pull: PullRequest) -> None:
repo = pull.base.repo
if 'update me' in [l.name for l in pull.as_issue().labels]:
print(f'Checking if #{pull.number} is up to date with master.')
master = repo.get_branch('master')
base, head = get_common_tree(repo, master.commit.sha, pull.head.sha)
if head.issuperset(base):
print('Up to date')
return
print(f'#{pull.number}: {pull.head.ref} is behind.')
repo.merge(pull.head.ref, 'master', f'Merge master into #{pull.number}')
def get_checks_summary_lines(self, pr: PullRequest) -> List[str]:
lines = []
commit = pr.get_commits().reversed[0]
for suite in commit.get_check_suites():
completed = suite.status == "completed"
success = suite.conclusion in ["success", "neutral", "skipped"]
if completed:
result = CHECK_PASSED if success else CHECK_FAILED
else:
result = CHECK_PENDING
lines.append(f"**{suite.app.name}** {result}")
return lines
def get_conversations(pull_request: PullRequest):
"""Get conversations for a pull_request."""
conversations = []
for c in pull_request.get_issue_comments():
comment = {
"user": c.user.login,
"text": c.body,
"created_at": int(c.created_at.timestamp()),
"reactions": [r.content for r in c.get_reactions()],
}
conversations.append(comment)
return conversations
def _pr_from_github_object(github_pr: GithubPullRequest) -> PullRequest:
return PullRequest(
title=github_pr.title,
id=github_pr.number,
status=PRStatus.merged
if github_pr.is_merged() else PRStatus[github_pr.state],
url=github_pr.html_url,
description=github_pr.body,
author=github_pr.user.name,
source_branch=github_pr.head.ref,
target_branch=github_pr.base.ref,
created=github_pr.created_at,
)
def _fetch_repo(self, repo, results):
for issue in repo.get_issues():
details = {
'repo': repo.full_name,
'number': issue.number,
'url': issue.html_url,
'title': issue.title,
'assignee': (issue.assignee.login if issue.assignee
else None),
'user': issue.user.login,
'blobs': [issue.title, issue.body],
'files': [],
'type': 'issue',
}
# for comment in issue.get_comments():
# details['blobs'].append(comment.body)
if issue.pull_request:
details['type'] = 'pull request'
pull = PullRequest(repo._requester, {}, {}, completed=True)
pull._url = _ValuedAttribute('%s/%s/%s' % (repo.url, 'pulls',
issue.number))
for pull_file in pull.get_files():
details['files'].append(pull_file.filename)
# for comment in pull.get_comments():
# details['blobs'].append(comment.body)
# I'm not totally sure this is useful, so disabling as it
# get's us back a lot of API requests
# for commit in pull.get_commits():
# details['blobs'].append(commit.commit.message)
results[repo.full_name].append(details)
def reuse_comment(self, pull: PullRequest, body: str) -> bool:
# get comments of this pull request
comments = self.get_pull_request_comments(pull, order_by_updated=True)
# get all comments that come from this action and are not hidden
comments = self.get_action_comments(comments)
# if there is no such comment, stop here
if len(comments) == 0:
return False
# edit last comment
comment_id = comments[-1].get("databaseId")
logger.info(f'editing comment {comment_id}')
if ':recycle:' not in body:
body = f'{body}\n:recycle: This comment has been updated with latest results.'
try:
pull.get_issue_comment(comment_id).edit(body)
except Exception as e:
self._gha.warning(f'Failed to edit existing comment #{comment_id}')
logger.debug('editing existing comment failed', exc_info=e)
return True
def pullreq_commits_authors(pullreq: PullRequest,
cache: MutableMapping[int, CacheItem]) -> List[AuthorTuple]:
"""Return a list of git authors of all commits contained in the given pull request."""
cached_item = cache.get(pullreq.id)
if cached_item and cached_item.updated_at >= pullreq.updated_at:
LOG.debug("Using cached data for pull request #%s", pullreq.number)
commits_authors = cached_item.value
else:
LOG.debug("Loading commits for pull request #%s", pullreq.number)
commits_authors = [commit_git_author(c) for c in pullreq.get_commits()]
cache[pullreq.id] = CacheItem(commits_authors, pullreq.updated_at)
return commits_authors
def pullreq_commits_authors(
pullreq: PullRequest,
cache: MutableMapping[int, CacheItem]) -> List[AuthorTuple]:
"""Return a list of git authors of all commits contained in the given pull request."""
cached_item = cache.get(pullreq.id)
if cached_item and cached_item.updated_at >= pullreq.updated_at:
LOG.debug("Using cached data for pull request #%s", pullreq.number)
commits_authors = cached_item.value
else:
LOG.debug("Loading commits for pull request #%s", pullreq.number)
commits_authors = [commit_git_author(c) for c in pullreq.get_commits()]
cache[pullreq.id] = CacheItem(commits_authors, pullreq.updated_at)
return commits_authors
def extract_pull_request_review_requests(
pull_request: GithubPullRequest) -> List[str]:
"""Extract features from requested reviews of the PR.
GitHub understands review requests rather as requested reviewers than actual
requests.
Arguments:
pull_request {PullRequest} -- PR of which we can extract review requests.
Returns:
List[str] -- list of logins of the requested reviewers
"""
requested_users = pull_request.get_review_requests()[0]
extracted = []
for user in requested_users:
extracted.append(user.login)
return extracted
def get_referenced_issues(pull_request: GithubPullRequest) -> List[str]:
"""Scan all of the Pull Request comments and get referenced issues.
Arguments:
pull_request {PullRequest} -- Pull request for which the referenced
issues are extracted
Returns:
List[str] -- IDs of referenced issues within the Pull Request.
"""
issues_referenced = []
for comment in pull_request.get_issue_comments():
for id in PullRequest.search_for_references(comment.body):
issues_referenced.append(id)
for id in PullRequest.search_for_references(pull_request.body):
issues_referenced.append(id)
_LOGGER.debug(" referenced issues: %s" % issues_referenced)
return issues_referenced
def get_added_files(pr: PullRequest.PullRequest):
print(pr, pr.number)
for file in pr.get_files():
if file.status == "added":
yield file.filename
def get_files_in_pr(pr: PullRequest) -> List[str]:
prs = get_prs_since(repo_url, since)
return [_f.filename for pr in prs for _f in pr.get_files()]
def close_pullreq_with_comment(pullreq: PullRequest, comment: str) -> None:
pullreq.create_issue_comment(comment)
pullreq.edit(state='closed')
def close_pullreq_with_comment(pullreq: PullRequest, comment: str) -> None:
pullreq.create_issue_comment(comment)
pullreq.edit(state='closed')
def update_pull_with_comments(self, pull: PullRequest, pull_data: Dict,
dry_run: bool) -> None:
meta = pull_data["pull"]
if dry_run:
print(f"Would update pull {pull.number} with {meta}")
return
assert meta["head"] == pull.head.ref
pull.edit(
title=meta["title"],
body=meta["body"],
state=meta["state"],
base=meta["base"],
)
pull.set_labels(*meta["labels"])
pull.remove_from_assignees(*[
x.name for x in pull.assignees if x.name not in meta["assignees"]
])
pull.add_to_assignees(*meta["assignees"])
(reviewers, team_reviewers) = pull.get_review_request()
pull.delete_review_request(
reviewers=[u.name for u in reviewers],
team_reviewers=[u.name for u in team_reviewers])
pull.create_review_request(reviewers=meta["reviewers"])
self.update_pull_comments(pull, pull_data["comments"], dry_run)
def get_release_notes(pr: PullRequest) -> str:
for comment in pr.get_issue_comments():
m = re.search("(?si)Release notes:(.*)", comment.body)
if m:
return m[1].strip()
return ""
def get_old_comment(pr: PullRequest.PullRequest):
for comment in pr.get_issue_comments():
if ("github-actions"
in comment.user.login) and ("No news item is found"
in comment.body):
return comment
