async def get_total_open_prs(
gh: GitHubAPI,
installation_id: int,
*,
repository: str,
user_login: Optional[str] = None,
count: Optional[bool] = True,
) -> Any:
"""Return the total number of open pull requests in the repository.
If the `user_login` parameter is given, then return the total number of open
pull request by that user in the repository.
If the `count` parameter is `False`, it returns the list of pull request
numbers instead.
For GitHub's REST API v3, issues and pull requests are the same so
`repository["open_issues_count"]` returns the total number of open
issues and pull requests. As we only want the pull request count,
we can make a search API call for open pull requests.
"""
installation_access_token = await get_access_token(gh, installation_id)
search_url = f"/search/issues?q=type:pr+state:open+repo:{repository}"
if user_login is not None:
search_url += f"+author:{user_login}"
if count is False:
pr_numbers = []
async for pull in gh.getiter(search_url,
oauth_token=installation_access_token):
pr_numbers.append(pull["number"])
return pr_numbers
data = await gh.getitem(search_url, oauth_token=installation_access_token)
return data["total_count"]
async def get_issues(user, repo):
async with aiohttp.ClientSession() as session:
gh = GitHubAPI(session, "octaflop", oauth_token=os.getenv("GH_AUTH"))
url = f'/repos/{user}/{repo}/issues'
print(url)
issues = gh.getiter(url)
return [i async for i in issues]
def search_issues(
gh: GitHubAPI,
token: str,
query_parameters: List[str],
) -> AsyncGenerator[Dict[str, Any], None]:
"""Search github issues and pull requests.
As documented here:
https://developer.github.com/v3/search/#search-issues-and-pull-requests
A common query string is likely "repo:NixOS/nixpkgs". Returns an async
iterator of issues, automatically handling pagination.
"""
query = "+".join(query_parameters)
return gh.getiter(f"https://api.github.com/search/issues?q={query}",
oauth_token=token)
async def get_parsed_commit_range(
start: str, end: str, repo: str, gh: GitHubAPI, edit: bool = False
) -> Tuple[List[Commit], List[Commit]]:
commits_iter = gh.getiter(f"/repos/{repo}/commits?sha={start}")
commits = []
unparsed_commits = []
try:
async for item in commits_iter:
commit_hash = item["sha"]
commit_message = item["commit"]["message"]
if commit_hash == end:
break
invalid_message = False
try:
_default_parser(commit_message)
# if this succeeds, do nothing
except UnknownCommitMessageStyleError as err:
print("Unknown commit message style!")
if not commit_message.startswith("Merge"):
invalid_message = True
if (
(invalid_message or edit)
and sys.stdout.isatty()
and False
and typer.confirm(f"Edit effective message '{commit_message}'?")
):
commit_message = typer.edit(commit_message)
_default_parser(commit_message)
commit = Commit(commit_hash, commit_message, item["author"]["login"])
commits.append(commit)
if invalid_message:
unparsed_commits.append(commit)
print("-", commit)
if len(commits) > 200:
raise RuntimeError(f"{len(commits)} are a lot. Aborting!")
return commits, unparsed_commits
except gidgethub.BadRequest:
print(
"BadRequest for commit retrieval. That is most likely because you forgot to push the merge commit."
)
return
async def get_pr_files(gh: GitHubAPI, installation_id: int, *,
pull_request: Dict[str, Any]) -> List[Dict[str, str]]:
"""Return the list of files data from a given pull request.
The data will include `filename` and `contents_url`. The `contents_url` will be
used to download and parse the Python code and check for tests and type hints.
"""
installation_access_token = await get_access_token(gh, installation_id)
files = []
async for data in gh.getiter(pull_request["url"] + "/files",
oauth_token=installation_access_token):
# No need to do any checks for files which are removed.
if data["status"] != "removed":
files.append({
"filename": data["filename"],
"contents_url": data["contents_url"]
})
return files
class PullRequester:
"""Create GitHub pull requests.
Parameters
----------
path : `pathlib.Path`
Path to the Git repository.
config : `neophile.config.Configuration`
neophile configuration.
session : `aiohttp.ClientSession`
The client session to use for requests.
"""
def __init__(self, path: Path, config: Configuration,
session: ClientSession) -> None:
self._config = config
self._github = GitHubAPI(
session,
config.github_user,
oauth_token=config.github_token.get_secret_value(),
)
self._repo = Repo(str(path))
async def make_pull_request(self, changes: Sequence[Update]) -> None:
"""Create or update a pull request for a list of changes.
Parameters
----------
changes : Sequence[`neophile.update.base.Update`]
The changes.
Raises
------
neophile.exceptions.PushError
Pushing the branch to GitHub failed.
"""
github_repo = self._get_github_repo()
default_branch = await self._get_github_default_branch(github_repo)
pull_number = await self._get_pr(github_repo, default_branch)
message = await self._commit_changes(changes)
self._push_branch()
if pull_number is not None:
await self._update_pr(github_repo, pull_number, message)
else:
await self._create_pr(github_repo, default_branch, message)
def _build_commit_message(self,
changes: Sequence[Update]) -> CommitMessage:
"""Build a commit message from a list of changes.
Parameters
----------
changes : Sequence[`neophile.update.base.Update`]
The changes.
Returns
-------
message : `CommitMessage`
The corresponding commit message.
"""
descriptions = [change.description() for change in changes]
return CommitMessage(changes=descriptions)
async def _commit_changes(self,
changes: Sequence[Update]) -> CommitMessage:
"""Commit a set of changes to the repository.
The changes will be committed on the current branch.
Parameters
----------
changes : Sequence[`neophile.update.base.Update`]
The changes to apply and commit.
Returns
-------
message : `CommitMessage`
The commit message of the commit.
"""
actor = await self._get_github_actor()
for change in changes:
self._repo.index.add(str(change.path))
message = self._build_commit_message(changes)
self._repo.index.commit(str(message), author=actor, committer=actor)
return message
async def _create_pr(
self,
github_repo: GitHubRepository,
base_branch: str,
message: CommitMessage,
) -> None:
"""Create a new PR for the current branch.
Parameters
----------
github_repo : `neophile.config.GitHubRepository`
GitHub repository in which to create the pull request.
base_branch : `str`
The branch of the repository to use as the base for the PR.
message : `CommitMessage`
The commit message to use for the pull request.
"""
branch = self._repo.head.ref.name
data = {
"title": message.title,
"body": message.body,
"head": branch,
"base": base_branch,
"maintainer_can_modify": True,
"draft": False,
}
await self._github.post(
"/repos{/owner}{/repo}/pulls",
url_vars={
"owner": github_repo.owner,
"repo": github_repo.repo
},
data=data,
)
def _get_authenticated_remote(self) -> str:
"""Get the URL with authentication credentials of the origin remote.
Supports an ssh URL, an https URL, or the SSH syntax that Git
understands (user@host:path).
Returns
-------
url : `str`
A URL suitable for an authenticated push of a new branch.
"""
url = self._get_remote_url()
token = self._config.github_token.get_secret_value()
auth = f"{self._config.github_user}:{token}"
host = url.netloc.rsplit("@", 1)[-1]
url = url._replace(scheme="https", netloc=f"{auth}@{host}")
return url.geturl()
async def _get_github_actor(self) -> Actor:
"""Get authorship information for commits.
Using the GitHub API, retrieve the name and email address of the user
for which we have a GitHub token. Use that to construct the Author
information for a GitHub commit.
Returns
-------
author : `git.objects.util.Actor`
The actor to use for commits.
"""
response = await self._github.getitem("/user")
if self._config.github_email:
return Actor(response["name"], self._config.github_email)
else:
return Actor(response["name"], response["email"])
async def _get_github_default_branch(self,
github_repo: GitHubRepository) -> str:
"""Get the main branch of the repository.
Uses ``main`` if that branch exists, else ``master``.
Parameters
----------
github_repo : `neophile.config.GitHubRepository`
GitHub repository in which to create the pull request.
Returns
-------
branch : `str`
The name of the main branch.
"""
repo = await self._github.getitem(
"/repos{/owner}{/repo}",
url_vars={
"owner": github_repo.owner,
"repo": github_repo.repo
},
)
return repo.get("default_branch", "master")
def _get_github_repo(self) -> GitHubRepository:
"""Get the GitHub repository.
Done by parsing the URL of the origin remote.
Returns
-------
repo : `neophile.config.GitHubRepository`
GitHub repository information.
"""
url = self._get_remote_url()
_, owner, repo = url.path.split("/")
if repo.endswith(".git"):
repo = repo[:-len(".git")]
return GitHubRepository(owner=owner, repo=repo)
async def _get_pr(self, github_repo: GitHubRepository,
base_branch: str) -> Optional[str]:
"""Get the pull request number of an existing neophile PR.
Parameters
----------
github_repo : `neophile.config.GitHubRepository`
GitHub repository in which to search for a pull request.
bsae_branch : `str`
The base repository branch used to limit the search.
Returns
-------
pull_number : `str` or `None`
The PR number or `None` if there is no open pull request from
neophile.
Notes
-----
The pull request is found by searching for all PRs in the open state
whose branch is u/neophile.
"""
query = {
"state": "open",
"head": f"{github_repo.owner}:u/neophile",
"base": base_branch,
}
prs = self._github.getiter(
f"/repos{{/owner}}{{/repo}}/pulls?{urlencode(query)}",
url_vars={
"owner": github_repo.owner,
"repo": github_repo.repo
},
)
async for pr in prs:
return str(pr["number"])
return None
def _get_remote_url(self) -> ParseResult:
"""Get the parsed URL of the origin remote.
The URL will be converted to https form. https, ssh, and the SSH
remote syntax used by Git are supported.
Returns
-------
url : `str`
The results of `~urllib.parse.urlparse` on the origin remote URL.
"""
url = next(self._repo.remotes.origin.urls)
if "//" in url:
return urlparse(url)
else:
path = url.rsplit(":", 1)[-1]
return urlparse(f"https://github.com/{path}")
def _push_branch(self) -> None:
"""Push the u/neophile branch to GitHub.
Raises
------
neophile.exceptions.PushError
Pushing the branch to GitHub failed.
"""
branch = self._repo.head.ref.name
remote_url = self._get_authenticated_remote()
remote = Remote.add(self._repo, "tmp-neophile", remote_url)
try:
push_info = remote.push(f"{branch}:{branch}", force=True)
for result in push_info:
if result.flags & PushInfo.ERROR:
msg = f"Pushing {branch} failed: {result.summary}"
raise PushError(msg)
finally:
Remote.remove(self._repo, "tmp-neophile")
async def _update_pr(
self,
github_repo: GitHubRepository,
pull_number: str,
message: CommitMessage,
) -> None:
"""Update an existing PR with a new commit message.
Parameters
----------
github_repo : `neophile.config.GitHubRepository`
GitHub repository in which to create the pull request.
pull_number : `str`
The number of the pull request to update.
message : `CommitMessage`
The commit message to use for the pull request.
"""
data = {
"title": message.title,
"body": message.body,
}
await self._github.patch(
"/repos{/owner}{/repo}/pulls{/pull_number}",
url_vars={
"owner": github_repo.owner,
"repo": github_repo.repo,
"pull_number": pull_number,
},
data=data,
)
class Host(ni_abc.ContribHost):
"""Implement a webhook for GitHub pull requests."""
route = 'POST', '/github'
_useful_actions = {PullRequestEvent.opened.value,
PullRequestEvent.unlabeled.value,
PullRequestEvent.synchronize.value}
def __init__(self, server: ni_abc.ServerHost, client: aiohttp.ClientSession,
event: PullRequestEvent,
request: JSONDict) -> None:
"""Represent a contribution."""
self.server = server
self.event = event
self.request = request
self._gh = GitHubAPI(client, "the-knights-who-say-ni",
oauth_token=server.contrib_auth_token())
@classmethod
async def process(cls, server: ni_abc.ServerHost,
request: web.Request, client: aiohttp.ClientSession) -> "Host":
"""Process the pull request."""
event = sansio.Event.from_http(request.headers,
await request.read(),
secret=server.contrib_secret())
if event.event == "ping":
# A ping event; nothing to do.
# https://developer.github.com/webhooks/#ping-event
raise ni_abc.ResponseExit(status=http.HTTPStatus.OK)
elif event.event != "pull_request":
# Only happens if GitHub is misconfigured to send the wrong events.
raise TypeError(f"don't know how to handle a {event.event!r} event")
elif event.data['action'] not in cls._useful_actions:
raise ni_abc.ResponseExit(status=http.HTTPStatus.NO_CONTENT)
elif event.data['action'] in {PullRequestEvent.opened.value, PullRequestEvent.synchronize.value}:
if event.data['action'] == PullRequestEvent.opened.value:
# GitHub is eventually consistent, so add a delay to wait for
# the API to digest the new pull request.
await asyncio.sleep(1)
return cls(server, client, PullRequestEvent(event.data['action']),
event.data)
elif event.data['action'] == PullRequestEvent.unlabeled.value:
label = event.data['label']['name']
if not label.startswith(LABEL_PREFIX):
raise ni_abc.ResponseExit(status=http.HTTPStatus.NO_CONTENT)
return cls(server, client, PullRequestEvent.unlabeled, event.data)
else: # pragma: no cover
# Should never happen.
raise TypeError(f"don't know how to handle a {event.data['action']!r} action")
async def usernames(self) -> AbstractSet[str]:
"""Return an iterable with all of the contributors' usernames."""
pull_request = self.request['pull_request']
# Start with the author of the pull request.
logins = {pull_request['user']['login']}
# For each commit, get the author and committer.
async for commit in self._gh.getiter(pull_request['commits_url']):
author = commit['author']
# When the author is missing there seems to typically be a
# matching commit that **does** specify the author. (issue #56)
if author:
author_login = author.get('login')
if commit['commit']['author']['email'].lower() == GITHUB_EMAIL:
self.server.log("Ignoring GitHub-managed username: "******"Ignoring GitHub-managed username: "******"""Construct the URL to the label."""
if not hasattr(self, '_labels_url'):
issue_url = self.request['pull_request']['issue_url']
issue_data = await self._gh.getitem(issue_url)
self._labels_url = uritemplate.URITemplate(issue_data['labels_url'])
return self._labels_url.expand(name=label)
async def current_label(self) -> Optional[str]:
"""Return the current CLA-related label."""
labels_url = await self.labels_url()
all_labels = []
async for label in self._gh.getiter(labels_url):
all_labels.append(label['name'])
cla_labels = [x for x in all_labels if x.startswith(LABEL_PREFIX)]
cla_labels.sort()
return cla_labels[0] if len(cla_labels) > 0 else None
async def set_label(self, status: ni_abc.Status) -> str:
"""Set the label on the pull request based on the status of the CLA."""
labels_url = await self.labels_url()
if status == ni_abc.Status.signed:
await self._gh.post(labels_url, data=[CLA_OK])
return CLA_OK
else:
await self._gh.post(labels_url, data=[NO_CLA])
return NO_CLA
async def remove_label(self) -> Optional[str]:
"""Remove any CLA-related labels from the pull request."""
cla_label = await self.current_label()
if cla_label is None:
return None
deletion_url = await self.labels_url(cla_label)
await self._gh.delete(deletion_url)
return cla_label
async def comment(self, status: ni_abc.Status) -> Optional[str]:
"""Add an appropriate comment relating to the CLA status."""
comments_url = self.request['pull_request']['comments_url']
if status == ni_abc.Status.signed:
return None
elif status == ni_abc.Status.not_signed:
if random.random() < EASTEREGG_PROBABILITY: # pragma: no cover
message = NO_CLA_TEMPLATE.format(body=NO_CLA_BODY_EASTEREGG)
else:
message = NO_CLA_TEMPLATE.format(body=NO_CLA_BODY)
elif status == ni_abc.Status.username_not_found:
message = NO_CLA_TEMPLATE.format(body=NO_USERNAME_BODY)
else: # pragma: no cover
# Should never be reached.
raise TypeError("don't know how to handle {}".format(status))
await self._gh.post(comments_url, data={'body': message})
return message
async def update(self, status: ni_abc.Status) -> None:
if self.event == PullRequestEvent.opened:
await self.set_label(status)
await self.comment(status)
elif self.event == PullRequestEvent.unlabeled:
# The assumption is that a PR will almost always go from no CLA to
# being cleared, so don't bug the user with what will probably
# amount to a repeated message about lacking a CLA.
await self.set_label(status)
elif self.event == PullRequestEvent.synchronize:
current_label = await self.current_label()
if status == ni_abc.Status.signed:
if current_label != CLA_OK:
await self.remove_label()
elif current_label != NO_CLA:
await self.remove_label()
# Since there is a chance a new person was added to a PR
# which caused the change in status, a comment on how to
# resolve the CLA issue is probably called for.
await self.comment(status)
else: # pragma: no cover
# Should never be reached.
msg = 'do not know how to update a PR for {}'.format(self.event)
raise RuntimeError(msg)
class GitHubInventory:
"""Inventory available tags of a GitHub repository.
Parameters
----------
config : `neophile.config.Configuration`
neophile configuration.
session : `aiohttp.ClientSession`
The aiohttp client session to use to make requests for GitHub tags.
"""
def __init__(self, config: Configuration, session: ClientSession) -> None:
self._github = GitHubAPI(
session,
config.github_user,
oauth_token=config.github_token.get_secret_value(),
)
async def inventory(
self, owner: str, repo: str, semantic: bool = False
) -> Optional[str]:
"""Return the latest tag of a GitHub repository.
Parameters
----------
owner : `str`
Owner of the repository.
repo : `str`
Name of the repository.
semantic : `bool`, optional
If set to true, only semantic versions will be considered and the
latest version will be determined by semantic version sorting
instead of `packaging.version.Version`.
Returns
-------
result : `str` or `None`
The latest tag in sorted order. Tags that parse as valid versions
sort before tags that do not, which should normally produce the
correct results when version tags are mixed with other tags. If
no valid tags are found or the repository doesn't exist, returns
`None`.
"""
logging.info("Inventorying GitHub repo %s/%s", owner, repo)
cls = SemanticVersion if semantic else PackagingVersion
try:
tags = self._github.getiter(
"/repos{/owner}{/repo}/tags",
url_vars={"owner": owner, "repo": repo},
)
versions = [
cls.from_str(tag["name"])
async for tag in tags
if cls.is_valid(tag["name"])
]
except ClientError as e:
logging.warning(
"Unable to inventory GitHub repo %s/%s: %s",
owner,
repo,
str(e),
)
return None
if versions:
return str(max(versions))
else:
logging.warning(
"No valid versions for GitHub repo %s/%s", owner, repo
)
return None
async def main(draft, dry_run):
token = os.environ["GH_TOKEN"]
async with aiohttp.ClientSession(loop=asyncio.get_event_loop()) as session:
gh = GitHubAPI(session, __name__, oauth_token=token)
version_file = Path("version_number")
current_version = version_file.read_text()
tag_hash = str(
git("rev-list", "-n", "1", f"v{current_version}").strip())
print("current_version:", current_version, "[" + tag_hash[:8] + "]")
sha = git("rev-parse", "HEAD").strip()
print("sha:", sha)
repo = get_repo()
print("repo:", repo)
commits_iter = gh.getiter(f"/repos/{repo}/commits?sha={sha}")
commits = []
try:
async for item in commits_iter:
commit_hash = item["sha"]
commit_message = item["commit"]["message"]
if commit_hash == tag_hash:
break
try:
_default_parser(commit_message)
# if this succeeds, do nothing
except UnknownCommitMessageStyleError as err:
print("Unkown commit message style:")
print(commit_message)
if sys.stdout.isatty() and click.confirm(
"Edit effective message?"):
commit_message = click.edit(commit_message)
_default_parser(commit_message)
commit = Commit(commit_hash, commit_message)
commits.append(commit)
print("-", commit)
except gidgethub.BadRequest:
print(
"BadRequest for commit retrieval. That is most likely because you forgot to push the merge commit."
)
return
if len(commits) > 100:
print(len(commits), "are a lot. Aborting!")
sys.exit(1)
bump = evaluate_version_bump(commits)
print("bump:", bump)
if bump is None:
print("-> nothing to do")
return
next_version = get_new_version(current_version, bump)
print("next version:", next_version)
next_tag = f"v{next_version}"
changes = generate_changelog(commits)
md = markdown_changelog(next_version, changes, header=False)
print(md)
if not dry_run:
version_file.write_text(next_version)
git.add(version_file)
git.commit(m=f"Bump to version {next_tag}")
# git.tag(next_tag)
target_hash = str(git("rev-parse", "HEAD")).strip()
print("target_hash:", target_hash)
git.push()
commit_ok = False
print("Waiting for commit", target_hash[:8], "to be received")
for _ in range(10):
try:
url = f"/repos/{repo}/commits/{target_hash}"
await gh.getitem(url)
commit_ok = True
break
except InvalidField as e:
print("Commit", target_hash[:8], "not received yet")
pass # this is what we want
await asyncio.sleep(0.5)
if not commit_ok:
print("Commit", target_hash[:8], "was not created on remote")
sys.exit(1)
print("Commit", target_hash[:8], "received")
await gh.post(
f"/repos/{repo}/releases",
data={
"body": md,
"tag_name": next_tag,
"name": next_tag,
"draft": draft,
"target_commitish": target_hash,
},
)
async def get_tag(tag: str, repo: str, gh: GitHubAPI):
async for item in gh.getiter(f"repos/{repo}/tags"):
if item["name"] == tag:
return item
return None
async def get_tag_hash(tag: str, repo: str, gh: GitHubAPI) -> str:
async for item in gh.getiter(f"repos/{repo}/tags"):
if item["name"] == tag:
return item["commit"]["sha"]
raise ValueError(f"Tag {tag} not found")