def get_stale_branch_names_with_contrib(
repo: Repository) -> List[str]: # noqa: E999
"""Return the list of branches that have the prefix of "contrib/" without open pull requests
and that have not been updated for 2 months (stale)
Args:
repo (Repository): The repository whose branches will be searched and listed
Returns:
(List[str]): List of branch names that are stale and have the "contrib/" prefix
"""
# set now with GMT timezone
now = datetime.now(timezone.min)
organization = 'demisto'
branch_names = []
for branch in repo.get_branches():
# Make sure the branch is contrib
if branch.name.startswith('contrib/'):
prs_with_branch_as_base = repo.get_pulls(state='OPEN',
base=branch.name)
prs_with_branch_as_head = repo.get_pulls(
state='OPEN', head=f'{organization}:{branch.name})')
# Make sure there are no open prs pointing to/from the branch
if prs_with_branch_as_base.totalCount < 1 and prs_with_branch_as_head.totalCount < 1:
# Make sure HEAD commit is stale
if (last_modified := branch.commit.commit.last_modified) and (
last_commit_datetime := parse(last_modified)):
elapsed_days = (now - last_commit_datetime).days
if elapsed_days >= 60:
branch_names.append(branch.name)
else:
print(f"Couldn't load HEAD for {branch.name}")
def analyse_pull_requests(self,
repository: Repository,
prev_knowledge: Dict[str, Any],
is_local: bool = False) -> Dict[str, Any]:
"""Analyse every closed pull_request in repository.
Arguments:
repository {Repository} -- currently the PyGithub lib is used because of its functionality
ogr unfortunatelly did not provide enough to properly analyze issues
prev_knowledge {Dict[str, Any]} -- previous knowledge stored
"""
_LOGGER.info(
"-------------Pull Requests Analysis (including its Reviews)-------------"
)
current_pulls = repository.get_pulls(state="all")
new_pulls = self.get_only_new_entities(prev_knowledge, current_pulls)
if len(new_pulls) == 0:
return
with KnowledgeAnalysis(
entity_type=EntityTypeEnum.PULL_REQUEST.value,
new_entities=new_pulls,
accumulator=prev_knowledge,
store_method=self.store_pull_request,
) as analysis:
accumulated = analysis.store()
return accumulated
def find_matching_pulls(gh_repo: Repository, commits: Iter[Commit]) -> Generator:
"""Find pull requests that contains commits matching the given ``commits``.
It yields tuple :class:`PullRequest` and list of the matched
:class:`Commit`s (subset of the given ``commits``).
The matching algorithm is based on comparing commits by an *author*
(triplet name, email and date) and set of the affected files (just file
names). The match is found when a pull request contains at least one commit
from the given ``commits`` (i.e. their author triplet is the same), and
an union of filenames affected by all the matching commits is the same as of
all the pull request's commits.
"""
LOG.debug('Fetching commits referenced in payload')
commits_by_author = {commit_git_author(c): c for c in commits}
find_matching_commit = commits_by_author.get
cache = shared_cache()
for pullreq in gh_repo.get_pulls(state='open'):
LOG.debug("Checking pull request #%s", pullreq.number)
merged_commits = list(keep(find_matching_commit, pullreq_commits_authors(pullreq, cache)))
merged_files = (f.filename for c in merged_commits for f in c.files)
pullreq_files = (f.filename for f in pullreq.get_files())
if any(merged_commits) and set(merged_files) == set(pullreq_files):
del cache[pullreq.id]
yield pullreq, merged_commits
LOG.debug("Cached items: %d, max size: %d" % (cache.currsize, cache.maxsize))
def get_existing_pull_request(github_repo: Repository, search_label: str) -> PullRequest:
pull_requests = github_repo.get_pulls()
for pull_request in pull_requests:
for label in pull_request.labels:
if label.name == search_label:
return pull_request
return None
def analyse_pull_requests(project: Repository,
prev_pulls: Dict[str, Any]) -> Dict[str, Any]:
"""Analyse every closed pull_request in repository.
Arguments:
project {Repository} -- currently the PyGithub lib is used because of its functionality
ogr unfortunatelly did not provide enough to properly analyze issues
project_knowledge {Path} -- project directory where the issues knowledge will be stored
"""
_LOGGER.info(
"-------------Pull Requests Analysis (including its Reviews)-------------"
)
current_pulls = project.get_pulls(state="closed")
new_pulls = get_only_new_entities(prev_pulls, current_pulls)
if len(new_pulls) == 0:
return
with Knowledge(entity_type="PullRequest",
new_entities=new_pulls,
accumulator=prev_pulls,
store_method=store_pull_request) as analysis:
accumulated = analysis.store()
return accumulated
def find_matching_pulls(gh_repo: Repository,
commits: Iter[Commit]) -> Generator:
"""Find pull requests that contains commits matching the given ``commits``.
It yields tuple :class:`PullRequest` and list of the matched
:class:`Commit`s (subset of the given ``commits``).
The matching algorithm is based on comparing commits by an *author*
(triplet name, email and date) and set of the affected files (just file
names). The match is found when a pull request contains at least one commit
from the given ``commits`` (i.e. their author triplet is the same), and
an union of filenames affected by all the matching commits is the same as of
all the pull request's commits.
"""
LOG.debug('Fetching commits referenced in payload')
commits_by_author = {commit_git_author(c): c for c in commits}
find_matching_commit = commits_by_author.get
cache = shared_cache()
for pullreq in gh_repo.get_pulls(state='open'):
LOG.debug("Checking pull request #%s", pullreq.number)
merged_commits = list(
keep(find_matching_commit, pullreq_commits_authors(pullreq,
cache)))
merged_files = (f.filename for c in merged_commits for f in c.files)
pullreq_files = (f.filename for f in pullreq.get_files())
if any(merged_commits) and set(merged_files) == set(pullreq_files):
del cache[pullreq.id]
yield pullreq, merged_commits
LOG.debug("Cached items: %d, max size: %d" %
(cache.currsize, cache.maxsize))
def determine_reviewer(potential_reviewers: List[str], repo: Repository) -> str:
"""Checks the number of open 'Contribution' PRs that have either been assigned to a user or a review
was requested from the user for each potential reviewer and returns the user with the smallest amount
Args:
potential_reviewers (List): The github usernames from which a reviewer will be selected
repo (Repository): The relevant repo
Returns:
str: The github username to assign to a PR
"""
label_to_consider = 'contribution'
pulls = repo.get_pulls(state='OPEN')
assigned_prs_per_potential_reviewer = {reviewer: 0 for reviewer in potential_reviewers}
for pull in pulls:
# we only consider 'Contribution' prs when computing who to assign
pr_labels = [label.name.casefold() for label in pull.labels]
if label_to_consider not in pr_labels:
continue
assignees = set([assignee.login for assignee in pull.assignees])
requested_reviewers, _ = pull.get_review_requests()
requested_reviewers = set([requested_reviewer.login for requested_reviewer in requested_reviewers])
combined_list = assignees.union(requested_reviewers)
for reviewer in potential_reviewers:
if reviewer in combined_list:
assigned_prs_per_potential_reviewer[reviewer] = assigned_prs_per_potential_reviewer.get(reviewer) + 1
selected_reviewer = sorted(assigned_prs_per_potential_reviewer, key=assigned_prs_per_potential_reviewer.get)[0]
return selected_reviewer
def determine_reviewer(potential_reviewers: List[str],
repo: Repository) -> str:
"""Checks the number of open PRs that have either been assigned to a user or a review was requested
from the user for each potential reviewer and returns the user with the smallest amount
Args:
potential_reviewers (List): The github usernames from which a reviewer will be selected
repo (Repository): The relevant repo
Returns:
str: The github username to assign to a PR
"""
pulls = repo.get_pulls(state='OPEN')
assigned_prs_per_potential_reviewer = {
reviewer: 0
for reviewer in potential_reviewers
}
for pull in pulls:
assignees = set([assignee.login for assignee in pull.assignees])
requested_reviewers, _ = pull.get_review_requests()
requested_reviewers = set([
requested_reviewer.login
for requested_reviewer in requested_reviewers
])
combined_list = assignees.union(requested_reviewers)
for reviewer in potential_reviewers:
if reviewer in combined_list:
assigned_prs_per_potential_reviewer[
reviewer] = assigned_prs_per_potential_reviewer.get(
reviewer) + 1
selected_reviewer = sorted(assigned_prs_per_potential_reviewer,
key=assigned_prs_per_potential_reviewer.get)[0]
return selected_reviewer
def get_pull_request(repository: Repository, base_branch: Branch,
target_branch: Branch) -> Union[None, PullRequest]:
"""
Check if pull request already exists in forked repository.
:return: Empty list if PR is not found, fetched PR otherwise.
"""
LOGGER.info(
"Checking if Pull Request with base branch <%s> and target branch <%s> exists",
base_branch.name, target_branch.name)
existing_pull_requests = list(repository.get_pulls())
LOGGER.info("Fetched Pull Requests <%s>",
', '.join([pr.title for pr in existing_pull_requests]))
base_branch_check = base_branch.name in [
pr.base.ref for pr in existing_pull_requests
]
target_branch_check = target_branch.name in [
pr.head.ref for pr in existing_pull_requests
]
if base_branch_check and target_branch_check:
return [
pr for pr in existing_pull_requests
if pr.base.ref == base_branch.name
and pr.head.ref == target_branch.name
][0]
else:
return None
def get_branch_names_with_contrib(repo: Repository) -> List[str]: # noqa: E999
'''Return the list of branches that have the prefix of "contrib/" and that are base branches of open PRs
Args:
repo (Repository): The repository whose branches will be searched and listed
Returns:
(List[str]): List of branch names that have the "contrib/" prefix and are base branches of open PRs
'''
branch_names = []
open_prs_head_refs = {open_pr.head.ref for open_pr in repo.get_pulls(state='OPEN')}
for branch in repo.get_branches():
if branch.name.startswith('contrib/'):
prs_with_branch_as_base = repo.get_pulls(state='OPEN', base=branch.name)
if prs_with_branch_as_base.totalCount >= 1 and branch.name not in open_prs_head_refs:
branch_names.append(branch.name)
return branch_names
def pr_dump(repository: Repository,
progress_bar: bool = False,
include_reviewers: bool = True,
include_commit_hashes: bool = True) -> List[dict]:
"""
Dump the Pull Requests for the given repository as a list of dictionaries
:param repository:
:param progress_bar: Include a progress bar. Useful for commandline operation
:param include_reviewers: Include a list of users who reviewed the PR
:param include_commit_hashes: Include commit hashes associated with the PR
:return:
"""
pulls = repository.get_pulls(state="closed")
if progress_bar:
bar = progressbar.ProgressBar(
maxval=pulls.totalCount,
widgets=[progressbar.Percentage(),
progressbar.Bar()]).start()
count = 0
else:
bar = None
pull_entries = []
for p in pulls:
if p.is_merged():
# Handle reviews here
if include_reviewers:
reviews = _get_approved_reviews(p)
else:
reviews = None
# Handle commit hashes here
if include_commit_hashes:
commit_hashes = _get_commit_hashes(p)
else:
commit_hashes = None
# Now create our dictionary entry
entry = {
"target_branch": p.base.label,
"initiator": p.user.login,
"merger": p.merged_by.login,
"title": p.title,
"number": p.number,
}
if reviews is not None:
entry["reviews"] = reviews
if commit_hashes is not None:
entry["commits"] = commit_hashes
pull_entries.append(entry)
if bar:
count += 1
bar.update(count)
if bar:
bar.finish()
return pull_entries
async def list(self, context: commands.Context, repo: Repository):
pulls = list(repo.get_pulls()[:6])
if pulls:
embed = self.as_embed(pulls)
await context.send(embed=embed)
else:
await context.send(
"There's no open pull requests, brother. Never forget to wumbo."
)
def pull_requests(repository: Repository, branch: str) -> PaginatedList:
"""
Wrapper around repository.get_pulls() to add rate limiting check
"""
RateLimiter.check()
return repository.get_pulls(state='closed',
sort='merged_at',
direction='desc',
base=branch)
def repo_pull_requests(
repo: Repository,
since: datetime = None,
until: datetime = None) -> Generator[PullRequest, None, None]:
for pull in repo.get_pulls(sort='created', direction='desc', state='all'):
if since and pull.created_at < since:
break
if until and pull.created_at > until:
continue
yield pull
def get_pull_requests_with_label(repo: Repository.Repository,
label: str) -> List[int]:
pull_requests = repo.get_pulls(state='open')
pull_ids = {}
for pr in pull_requests:
labels = [k for k in pr.get_labels()]
if _find_label_by_name(labels, label):
pull_ids[pr.number] = sorted([
j.created_at for j in pr.get_issue_events()
if j.event == 'labeled' and j.label.name == label
])[-1]
sorted_pulls = sorted(pull_ids.items(), key=itemgetter(1))
return [k[0] for k in sorted_pulls]
def create_or_edit_pr(title: str, body: str, skills_repo: Repository,
user, branch: str):
base = skills_repo.default_branch
head = '{}:{}'.format(user.login, branch)
pulls = list(skills_repo.get_pulls(base=base, head=head))
if pulls:
pull = pulls[0]
if 'mycroft-skills-kit' in pull.body:
pull.edit(title, body)
else:
raise PRModified('Not updating description since it was not autogenerated')
return pull
else:
return skills_repo.create_pull(title, body, base=base, head=head)
def get_pr_from_commit(repo: Repository, sha: str) -> Optional[PullRequest]:
cached = redis.get_int(f'github:head:{sha}')
if cached:
try:
pr = repo.get_pull(cached)
if pr.head.sha == sha and pr.state == 'open':
return pr
except UnknownObjectException:
pass
for pr in repo.get_pulls():
head = pr.head.sha
redis.store(f'github:head:{head}', pr.number, ex=3600)
if head == sha:
return pr
return None
def create_or_edit_pr(title: str, body: str, skills_repo: Repository,
user, branch: str, repo_branch: str):
base = repo_branch
head = '{}:{}'.format(user.login, branch)
pulls = list(skills_repo.get_pulls(base=base, head=head))
if pulls:
pull = pulls[0]
if 'mycroft-skills-kit' in pull.body:
pull.edit(title, body)
else:
raise PRModified('Not updating description since it was not autogenerated')
return pull
else:
try:
return skills_repo.create_pull(title, body, base=base, head=head)
except GithubException as e:
if e.status == 422:
raise SkillNameTaken(title) from e
raise
def _get_repository_pull_requests(repo: Repository) -> List[Dict]:
"""
Get pull request data from repository provided.
:param repo: Github repository object
:returns: array of pull request data objects for given repository
"""
repo_full_name = repo.full_name
prs = repo.get_pulls(state='open', sort='created')
return [{
'repository': repo_full_name,
'pull_request': pr.number,
'url': pr.url,
'user': pr.user.login,
'date': pr.created_at.strftime(DATE_FORMAT),
'branch': pr.head.ref,
'mergeable': pr.mergeable,
'mergeable_state': pr.mergeable_state,
} for pr in prs]
def process_all_prs(report: GprrReport, repo: Repository,
strict_to_teamfilter: bool):
"""
Reads repo's PRs and check, whether PR's author, reviewer or assignee in
accounts provided
:param report: GprrReport instance, please where all data will be saved
:param repo: repo to process, Repository instance
:return:
"""
for pr in repo.get_pulls(state='open'):
marked_pr = False
gprr_pr: GprrPR = convert_pr(pr)
if report.accounts_filter.contains_item_id(gprr_pr.creator.id):
report.by_author.append_item(gprr_pr, gprr_pr.creator.name)
marked_pr = True
for review in gprr_pr.reviews:
if report.accounts_filter.contains_item_id(review.user.id):
report.by_reviewer.append_item(gprr_pr, review.user.name, True)
marked_pr = True
for assignee in gprr_pr.assignees:
if report.accounts_filter.contains_item_id(assignee.id):
report.by_assignee.append_item(gprr_pr, assignee.name)
marked_pr = True
if not strict_to_teamfilter:
report.by_repository.append_item(gprr_pr, repo.name)
report.full_list.append_item(gprr_pr, report.default_section)
else:
if marked_pr:
report.by_repository.append_item(gprr_pr, repo.name)
report.full_list.append_item(gprr_pr, report.default_section)
def pr_already_exists(repo: Repository.Repository):
pull_requests = repo.get_pulls(base=BASE_BRANCH, head=HEAD_BRANCH)
if pull_requests.totalCount != 0:
return True, pull_requests
else:
return False, None
def _get_has_pulls(repo: Repository) -> int:
pulls = repo.get_pulls()
return 1 if (pulls.totalCount > 0) else 0
