async def get_oauth_token(repository):
if GITHUB_APP_ID is None or GITHUB_APP_PRIVATE_KEY is None:
return None
async with GitHubAPIContext() as github_api:
# Lookup the installation_id from the repository name
if repository in _github_repositories:
installation_id = _github_repositories[repository]
else:
data = await github_api.getitem(
f"/repos/{repository}/installation",
accept=accept_format(version="machine-man-preview"),
jwt=get_jwt())
installation_id = data["id"]
_github_repositories[repository] = installation_id
# Check if we already have a token; otherwise get one
if installation_id in _github_installations:
expires_at, oauth_token = _github_installations[installation_id]
if expires_at > datetime.datetime.utcnow() + datetime.timedelta(
minutes=5):
return oauth_token
data = await github_api.post(
f"/installations/{installation_id}/access_tokens",
data="",
accept=accept_format(version="machine-man-preview"),
jwt=get_jwt())
expires_at = datetime.datetime.strptime(data["expires_at"],
"%Y-%m-%dT%H:%M:%SZ")
_github_installations[installation_id] = (expires_at, data["token"])
return data["token"]
async def opened_pr(event, gh_api, *args, **kwargs):
pr = event.data['pull_request']
repo = pr['base']['repo']['name']
author = pr['user']['login']
comments_url = pr['comments_url']
diff_url = pr['_links']['self']['href'] # does not use the diff_url field
rules_url = f'{config.github_uri}/api/v3/repos/{config.github_owner}/{repo}/contents/barrelman.yml'
futures = [
gh_api.getitem(diff_url, accept=sansio.accept_format(media='diff', json=False)),
gh_api.getitem(rules_url, accept=sansio.accept_format(media='raw', json=True)),
]
diff, rules = await asyncio.gather(*futures)
diff = parser.parse_diff(diff)
# if a barrelman.yml file has changed or been added in this PR, check if in valid format
if 'barrelman.yml' in diff:
ref = pr['head']['ref']
new_rules = await gh_api.getitem(f'{rules_url}?ref={ref}', accept=sansio.accept_format(media='raw', json=True))
parsed = parser.parse_barrel_rules(new_rules)
if type(parsed) is str:
await _create_warning_comment(gh_api, comments_url, parsed, ref)
if rules is None:
return
parsed = parser.parse_barrel_rules(rules)
if type(parsed) is str:
# if barrelman.yml file on master branch is corrupted
await _create_error_comment(gh_api, comments_url, parsed)
return
checker = rule_checker.RuleChecker(parsed)
checker.check_rules(diff)
# Author of PR cannot be added as a reviewer
checker.users_to_notify.discard(author)
if len(checker.triggered_regex_rules) == 0:
return
futures = [
_add_code_reviewers(gh_api, repo, pr['number'], list(checker.users_to_notify),
list(checker.teams_to_notify)),
_create_comment(gh_api, comments_url, checker.triggered_regex_rules)
]
await asyncio.gather(*futures)
async def getiter(self, url, *, accept=sansio.accept_format(), oauth_token=None):
self.getiter_url.append(url)
data = self._getiter_return[url]
if isinstance(data, dict) and "items" in data:
data = data["items"]
for item in data:
yield item
async def token_for(self, installation_id):
cit = self._installation_tokens[installation_id]
while _too_close_for_comfort(cit.expires_at):
print(
f"{installation_id}: Token is uncached or expired or will expire soon"
)
if cit.refresh_event is not None:
print(
f"{installation_id}: Renewal already in progress; waiting")
await cit.refresh_event.wait()
else:
print(f"{installation_id}: Renewing now")
cit.refresh_event = anyio.create_event()
try:
response = await self.app_client.post(
"/app/installations/{installation_id}/access_tokens",
url_vars={"installation_id": installation_id},
accept=accept_format(version="machine-man-preview"),
data={},
)
cit.token = response["token"]
cit.expires_at = pendulum.parse(response["expires_at"])
assert not _too_close_for_comfort(cit.expires_at)
print(f"{installation_id}: Renewed successfully")
finally:
# Make sure that even if we get cancelled, any other tasks
# will still wake up (and can retry the operation)
await cit.refresh_event.set()
cit.refresh_event = None
return cit.token
async def get_oauth_token(installation_id):
"""
Get the oauth_token for a given installation_id.
It first checks if the oauth_token is in the cache and still valid. Otherwise it will retrieve a new
token from the server.
Tokens that are about to expire (give or take 5 minutes) are considered expired, to reduce the amount of
possible 401s given.
"""
if _github_installations[installation_id] is not None:
expires_at, oauth_token = _github_installations[installation_id]
if expires_at > datetime.datetime.now() + datetime.timedelta(
minutes=5):
return oauth_token
async with GitHubAPIContext() as github_api:
data = await github_api.post(
f"/installations/{installation_id}/access_tokens",
data="",
accept=accept_format(version="machine-man-preview"),
jwt=get_jwt())
expires_at = datetime.datetime.strptime(data["expires_at"],
"%Y-%m-%dT%H:%M:%SZ")
_github_installations[installation_id] = (expires_at, data["token"])
async def getitem(self,
url,
*,
accept=sansio.accept_format(),
oauth_token=None):
self.getitem_url = url
return self._getitem_return[url]
def coroutine_wrapper(*args: Tuple[Any],
**kwargs: Dict[str, Any]) -> AsyncGeneratorType:
accept_media = kwargs.pop('accept', None)
preview_api_version = kwargs.pop('preview_api_version', None)
if preview_api_version is not None:
accept_media = accept_format(
version=f'{preview_api_version}-preview', )
if accept_media is not None:
kwargs['accept'] = accept_media
coroutine_instance = wrapped_coroutine(*args, **kwargs)
is_async_generator = isinstance(coroutine_instance, AsyncGeneratorType)
if not is_async_generator:
async def async_function_wrapper():
return await coroutine_instance
return async_function_wrapper()
async def async_generator_wrapper():
async for result_item in coroutine_instance:
yield result_item
return async_generator_wrapper()
async def test_more(self):
"""The 'next' link is returned appropriately."""
headers = MockGitHubAPI.DEFAULT_HEADERS.copy()
headers["link"] = "<https://api.github.com/fake?page=2>; " 'rel="next"'
gh = MockGitHubAPI(headers=headers)
_, more = await gh._make_request("GET", "/fake", {}, "",
sansio.accept_format())
assert more == "https://api.github.com/fake?page=2"
async def delete(self,
url,
*,
data=b"",
accept=sansio.accept_format(),
oauth_token=None):
self.delete_url = url
return self._delete_return
async def test_headers(self):
"""Appropriate headers are created."""
accept = sansio.accept_format()
gh = MockGitHubAPI(oauth_token="oauth token")
await gh._make_request("GET", "/rate_limit", {}, "", accept)
assert gh.headers["user-agent"] == "test_abc"
assert gh.headers["accept"] == accept
assert gh.headers["authorization"] == "token oauth token"
async def getiter(self,
url,
*,
accept=sansio.accept_format(),
oauth_token=None):
self.getiter_url = url
for item in self._getiter_return[url]:
yield item
async def put(self,
url,
*,
data=b"",
accept=sansio.accept_format(),
oauth_token=None):
self.put_url = url
self.put_data = data
return self._put_return
async def patch(self,
url,
*,
data,
accept=sansio.accept_format(),
oauth_token=None):
self.patch_url = url
self.patch_data = data
return self._patch_return
async def handle_ping(command, event_type, payload, gh_client):
assert command == ["ping"]
await gh_client.post(reply_url(event_type, payload),
data={"body": "pong!"})
await gh_client.post(
reaction_url(event_type, payload),
data={"content": "heart"},
accept=accept_format(version="squirrel-girl-preview"),
)
async def get_pr_diff(repo_name: str, pr_number: int, gh, event) -> str:
logger.info(f"Fetching diff in PR:{pr_number} in repo:{repo_name}")
installation_access_token = await get_github_token(gh, event)
response = await gh.getitem(
f"/repos/{repo_name}/pulls/{pr_number}",
accept=sansio.accept_format(media="diff"),
oauth_token=installation_access_token["token"],
)
return PatchSet(response)
async def test_decoding(self):
"""Test that appropriate decoding occurs."""
original_data = {"hello": "world"}
headers = MockGitHubAPI.DEFAULT_HEADERS.copy()
headers["content-type"] = "application/json; charset=utf-8"
gh = MockGitHubAPI(headers=headers,
body=json.dumps(original_data).encode("utf8"))
data, _ = await gh._make_request("GET", "/rate_limit", {}, "",
sansio.accept_format())
assert data == original_data
async def test_rate_limit_set(self):
"""The rate limit is updated after receiving a response."""
rate_headers = {
"x-ratelimit-limit": "42",
"x-ratelimit-remaining": "1",
"x-ratelimit-reset": "0",
}
gh = MockGitHubAPI(headers=rate_headers)
await gh._make_request("GET", "/rate_limit", {}, "",
sansio.accept_format())
assert gh.rate_limit.limit == 42
async def test_url_formatted_with_base_url(self):
"""The URL is appropriately formatted."""
gh = MockGitHubAPI(base_url="https://my.host.com")
await gh._make_request(
"GET",
"/users/octocat/following{/other_user}",
{"other_user": "******"},
"",
sansio.accept_format(),
)
assert gh.url == "https://my.host.com/users/octocat/following/brettcannon"
async def test_auth_headers_with_passed_jwt(self):
"""Test the authorization header with the passed jwt."""
accept = sansio.accept_format()
gh = MockGitHubAPI()
await gh._make_request("GET",
"/rate_limit", {},
"",
accept,
jwt="json web token")
assert gh.headers["user-agent"] == "test_abc"
assert gh.headers["accept"] == accept
assert gh.headers["authorization"] == "bearer json web token"
async def getiter(gh, url, identifier, **url_vars):
data = b""
accept = accept_format()
data, more = await gh._make_request("GET", url, url_vars, data, accept)
if isinstance(data, dict) and identifier in data:
data = data[identifier]
for item in data:
yield item
if more:
async for item in getiter(gh, more, identifier, **url_vars):
yield item
async def test_make_request_passing_token_and_jwt(self):
"""Test that passing both jwt and oauth_token raises ValueError."""
accept = sansio.accept_format()
gh = MockGitHubAPI()
with pytest.raises(ValueError) as exc_info:
await gh._make_request(
"GET",
"/rate_limit",
{},
"",
accept,
jwt="json web token",
oauth_token="oauth token",
)
assert str(exc_info.value) == "Cannot pass both oauth_token and jwt."
async def get_issue_labels(self, gh):
"""Get the issue's labels."""
count = 0
accept = ','.join([sansio.accept_format(), 'application/vnd.github.symmetra-preview+json'])
async for label in gh.getiter(self.issue_labels_url, {'number': self.number}, accept=accept):
# Not sure how many get returned before it must page, so sleep for
# one second on the arbitrary value of 20. That is a lot of labels for
# one issue, so it is probably not going to trigger often.
count += 1
if (count % 20) == 0:
await asyncio.sleep(1)
yield label['name']
async def startup():
"""
On startup do some initial requests to GitHub API.
This has two purposes:
- It lists our installations, allowing us to check if we are still in sync over time (in regards to installations)
- Check if our JWT it valid to communicate with GitHub API.
"""
async with GitHubAPIContext() as github_api:
installations = await github_api.getitem(
"/app/installations",
accept=accept_format(version="machine-man-preview"),
jwt=get_jwt())
for installation in installations:
_github_installations[installation["id"]] = None
log.info("Startup done; found %d installations",
len(_github_installations))
"""Utilities."""
import asyncio
import base64
import yaml
import traceback
import sys
import os
from gidgethub import sansio, InvalidField
try:
from yaml import CLoader as Loader
except ImportError:
from yaml import Loader
LABEL_HEADER = ','.join(
[sansio.accept_format(), 'application/vnd.github.symmetra-preview+json'])
REACTION_HEADER = ','.join([
sansio.accept_format(), 'application/vnd.github.squirrel-girl-preview+json'
])
HTML_HEADER = sansio.accept_format(media="html")
SINGLE_VALUES = {
'brace_expansion', 'extended_glob', 'case_insensitive', 'triage_label',
'review_label', 'delete_labels'
}
LIST_VALUES = {
'labels', 'rules', 'wip', 'lgtm_remove', 'lgtm_add_issue',
'lgtm_add_pull_request', 'triage_skip', 'triage_remove', 'review_skip',
'review_remove'
}
def test_version(self):
expect = "application/vnd.github.cloak-preview+json"
assert sansio.accept_format(version="cloak-preview") == expect
def test_no_json(self):
expect = "application/vnd.github.v3.raw"
assert sansio.accept_format(media="raw", json=False) == expect
def test_format(self):
expect = "application/vnd.github.v3.raw+json"
assert sansio.accept_format(media="raw") == expect
async def from_github(cls,
httpsession,
github_org='lsst-ts',
github_repo='ts_xml',
git_ref='develop',
github_user=None,
github_token=None,
cache_dir='~/.kafkaefd/github'):
"""Load the ``ts_sal`` repository directly from GitHub.
Parameters
----------
httpsession : `aiohttp.ClientSession`
Session from aiohttp.
github_org : `str`
Owner of the GitHub repository.
github_repo : `str`
Name of the repository.
git_ref : `str`
Branch or tag name to check out.
github_user : `str`
Your GitHub username, if providing a ``github_token``.
github_token : `str`
GitHub token (such as a personal access token).
Returns
-------
repo : `SalXmlRepo`
SAL XML repository.
"""
# Cache for files downloaded from GitHub
cache_root = Path(cache_dir).expanduser()
if github_user is None:
github_user = '******'
ghclient = gh_aiohttp.GitHubAPI(httpsession,
github_user,
oauth_token=github_token,
cache=cache)
git_sha = await ghclient.getitem('/repos{/owner}{/repo}/commits{/ref}',
url_vars={
'owner': github_org,
'repo': github_repo,
'ref': git_ref
},
accept=accept_format(media='sha',
json=False))
cache_dir = cache_root / git_sha
if cache_dir.exists():
# Load XML files from the cache
topics = await SalXmlRepo._load_xml_files_from_cache(cache_dir)
else:
# Download XML files from GitHub API, then cache for second
# execution
cache_dir.mkdir(parents=True, exist_ok=True)
data = await ghclient.getitem(
'/repos{/owner}{/repo}/git/trees{/sha}?recursive=1',
url_vars={
'owner': github_org,
'repo': github_repo,
'sha': git_sha
})
if data['truncated']:
raise RuntimeError('/git/trees result is truncated')
paths = []
for blob in data['tree']:
if blob['type'] != 'blob':
continue
if blob['path'].startswith('sal_interfaces/') \
and blob['path'].endswith('.xml'):
paths.append((blob['path'], blob['url']))
tasks = []
sem = asyncio.Semaphore(10)
for path, url in paths:
tasks.append(
asyncio.ensure_future(
cls._get_blob_gh(ghclient, sem, path, url, cache_dir)))
filedata = await asyncio.gather(*tasks)
topics = {}
for path, data in filedata:
topics.update(SalXmlRepo._parse_xml_topics(path, data))
print('GitHub rate limit: {}'.format(ghclient.rate_limit))
return cls(topics)
async def test_client_part_of_app():
app = GithubApp(
user_agent=TEST_USER_AGENT,
app_id=TEST_APP_ID,
private_key=TEST_PRIVATE_KEY,
webhook_secret=TEST_WEBHOOK_SECRET,
)
assert app.app_client.app is app
# github actually won't let any app client access the /rate_limit
# endpoint, because app credentials are so locked down. But we can look up
# information about ourself!
data = await app.app_client.getitem(
"/app", accept=accept_format(version="machine-man-preview"))
assert glom(data, "name") == "snekomatic-test"
# We can get an installation token
token = await app.token_for(TEST_INSTALLATION_ID)
# They're cached
token2 = await app.token_for(TEST_INSTALLATION_ID)
assert token == token2
# And the client works too:
i_client = app.client_for(TEST_INSTALLATION_ID)
assert i_client.app is app
assert i_client.installation_id == TEST_INSTALLATION_ID
data = await i_client.getitem("/rate_limit")
assert "rate" in data
# Now we'll cheat and trick the app into thinking that the token is
# expiring, and check that the client automatically renews it.
soon = pendulum.now().add(seconds=10)
app._installation_tokens[TEST_INSTALLATION_ID].expires_at = soon
# The client still works...
i_client = app.client_for(TEST_INSTALLATION_ID)
data = await i_client.getitem("/rate_limit")
assert "rate" in data
# ...but the token has changed.
assert token != await app.token_for(TEST_INSTALLATION_ID)
# And let's do that again, but this time we'll have two tasks try to fetch
# the token at the same time.
soon = pendulum.now().add(seconds=10)
app._installation_tokens[TEST_INSTALLATION_ID].expires_at = soon
tokens = []
async def get_token():
tokens.append(await app.token_for(TEST_INSTALLATION_ID))
async with trio.open_nursery() as nursery:
nursery.start_soon(get_token)
nursery.start_soon(get_token)
# They both end up with the same token, demonstrating that they didn't do
# two independent fetches
assert len(tokens) == 2
assert len(set(tokens)) == 1
def test_defaults(self):
assert sansio.accept_format() == "application/vnd.github.v3+json"
