async def get_oauth_token(repository): if GITHUB_APP_ID is None or GITHUB_APP_PRIVATE_KEY is None: return None async with GitHubAPIContext() as github_api: # Lookup the installation_id from the repository name if repository in _github_repositories: installation_id = _github_repositories[repository] else: data = await github_api.getitem( f"/repos/{repository}/installation", accept=accept_format(version="machine-man-preview"), jwt=get_jwt()) installation_id = data["id"] _github_repositories[repository] = installation_id # Check if we already have a token; otherwise get one if installation_id in _github_installations: expires_at, oauth_token = _github_installations[installation_id] if expires_at > datetime.datetime.utcnow() + datetime.timedelta( minutes=5): return oauth_token data = await github_api.post( f"/installations/{installation_id}/access_tokens", data="", accept=accept_format(version="machine-man-preview"), jwt=get_jwt()) expires_at = datetime.datetime.strptime(data["expires_at"], "%Y-%m-%dT%H:%M:%SZ") _github_installations[installation_id] = (expires_at, data["token"]) return data["token"]
async def opened_pr(event, gh_api, *args, **kwargs): pr = event.data['pull_request'] repo = pr['base']['repo']['name'] author = pr['user']['login'] comments_url = pr['comments_url'] diff_url = pr['_links']['self']['href'] # does not use the diff_url field rules_url = f'{config.github_uri}/api/v3/repos/{config.github_owner}/{repo}/contents/barrelman.yml' futures = [ gh_api.getitem(diff_url, accept=sansio.accept_format(media='diff', json=False)), gh_api.getitem(rules_url, accept=sansio.accept_format(media='raw', json=True)), ] diff, rules = await asyncio.gather(*futures) diff = parser.parse_diff(diff) # if a barrelman.yml file has changed or been added in this PR, check if in valid format if 'barrelman.yml' in diff: ref = pr['head']['ref'] new_rules = await gh_api.getitem(f'{rules_url}?ref={ref}', accept=sansio.accept_format(media='raw', json=True)) parsed = parser.parse_barrel_rules(new_rules) if type(parsed) is str: await _create_warning_comment(gh_api, comments_url, parsed, ref) if rules is None: return parsed = parser.parse_barrel_rules(rules) if type(parsed) is str: # if barrelman.yml file on master branch is corrupted await _create_error_comment(gh_api, comments_url, parsed) return checker = rule_checker.RuleChecker(parsed) checker.check_rules(diff) # Author of PR cannot be added as a reviewer checker.users_to_notify.discard(author) if len(checker.triggered_regex_rules) == 0: return futures = [ _add_code_reviewers(gh_api, repo, pr['number'], list(checker.users_to_notify), list(checker.teams_to_notify)), _create_comment(gh_api, comments_url, checker.triggered_regex_rules) ] await asyncio.gather(*futures)
async def getiter(self, url, *, accept=sansio.accept_format(), oauth_token=None): self.getiter_url.append(url) data = self._getiter_return[url] if isinstance(data, dict) and "items" in data: data = data["items"] for item in data: yield item
async def token_for(self, installation_id): cit = self._installation_tokens[installation_id] while _too_close_for_comfort(cit.expires_at): print( f"{installation_id}: Token is uncached or expired or will expire soon" ) if cit.refresh_event is not None: print( f"{installation_id}: Renewal already in progress; waiting") await cit.refresh_event.wait() else: print(f"{installation_id}: Renewing now") cit.refresh_event = anyio.create_event() try: response = await self.app_client.post( "/app/installations/{installation_id}/access_tokens", url_vars={"installation_id": installation_id}, accept=accept_format(version="machine-man-preview"), data={}, ) cit.token = response["token"] cit.expires_at = pendulum.parse(response["expires_at"]) assert not _too_close_for_comfort(cit.expires_at) print(f"{installation_id}: Renewed successfully") finally: # Make sure that even if we get cancelled, any other tasks # will still wake up (and can retry the operation) await cit.refresh_event.set() cit.refresh_event = None return cit.token
async def get_oauth_token(installation_id): """ Get the oauth_token for a given installation_id. It first checks if the oauth_token is in the cache and still valid. Otherwise it will retrieve a new token from the server. Tokens that are about to expire (give or take 5 minutes) are considered expired, to reduce the amount of possible 401s given. """ if _github_installations[installation_id] is not None: expires_at, oauth_token = _github_installations[installation_id] if expires_at > datetime.datetime.now() + datetime.timedelta( minutes=5): return oauth_token async with GitHubAPIContext() as github_api: data = await github_api.post( f"/installations/{installation_id}/access_tokens", data="", accept=accept_format(version="machine-man-preview"), jwt=get_jwt()) expires_at = datetime.datetime.strptime(data["expires_at"], "%Y-%m-%dT%H:%M:%SZ") _github_installations[installation_id] = (expires_at, data["token"])
async def getitem(self, url, *, accept=sansio.accept_format(), oauth_token=None): self.getitem_url = url return self._getitem_return[url]
def coroutine_wrapper(*args: Tuple[Any], **kwargs: Dict[str, Any]) -> AsyncGeneratorType: accept_media = kwargs.pop('accept', None) preview_api_version = kwargs.pop('preview_api_version', None) if preview_api_version is not None: accept_media = accept_format( version=f'{preview_api_version}-preview', ) if accept_media is not None: kwargs['accept'] = accept_media coroutine_instance = wrapped_coroutine(*args, **kwargs) is_async_generator = isinstance(coroutine_instance, AsyncGeneratorType) if not is_async_generator: async def async_function_wrapper(): return await coroutine_instance return async_function_wrapper() async def async_generator_wrapper(): async for result_item in coroutine_instance: yield result_item return async_generator_wrapper()
async def test_more(self): """The 'next' link is returned appropriately.""" headers = MockGitHubAPI.DEFAULT_HEADERS.copy() headers["link"] = "<https://api.github.com/fake?page=2>; " 'rel="next"' gh = MockGitHubAPI(headers=headers) _, more = await gh._make_request("GET", "/fake", {}, "", sansio.accept_format()) assert more == "https://api.github.com/fake?page=2"
async def delete(self, url, *, data=b"", accept=sansio.accept_format(), oauth_token=None): self.delete_url = url return self._delete_return
async def test_headers(self): """Appropriate headers are created.""" accept = sansio.accept_format() gh = MockGitHubAPI(oauth_token="oauth token") await gh._make_request("GET", "/rate_limit", {}, "", accept) assert gh.headers["user-agent"] == "test_abc" assert gh.headers["accept"] == accept assert gh.headers["authorization"] == "token oauth token"
async def getiter(self, url, *, accept=sansio.accept_format(), oauth_token=None): self.getiter_url = url for item in self._getiter_return[url]: yield item
async def put(self, url, *, data=b"", accept=sansio.accept_format(), oauth_token=None): self.put_url = url self.put_data = data return self._put_return
async def patch(self, url, *, data, accept=sansio.accept_format(), oauth_token=None): self.patch_url = url self.patch_data = data return self._patch_return
async def handle_ping(command, event_type, payload, gh_client): assert command == ["ping"] await gh_client.post(reply_url(event_type, payload), data={"body": "pong!"}) await gh_client.post( reaction_url(event_type, payload), data={"content": "heart"}, accept=accept_format(version="squirrel-girl-preview"), )
async def get_pr_diff(repo_name: str, pr_number: int, gh, event) -> str: logger.info(f"Fetching diff in PR:{pr_number} in repo:{repo_name}") installation_access_token = await get_github_token(gh, event) response = await gh.getitem( f"/repos/{repo_name}/pulls/{pr_number}", accept=sansio.accept_format(media="diff"), oauth_token=installation_access_token["token"], ) return PatchSet(response)
async def test_decoding(self): """Test that appropriate decoding occurs.""" original_data = {"hello": "world"} headers = MockGitHubAPI.DEFAULT_HEADERS.copy() headers["content-type"] = "application/json; charset=utf-8" gh = MockGitHubAPI(headers=headers, body=json.dumps(original_data).encode("utf8")) data, _ = await gh._make_request("GET", "/rate_limit", {}, "", sansio.accept_format()) assert data == original_data
async def test_rate_limit_set(self): """The rate limit is updated after receiving a response.""" rate_headers = { "x-ratelimit-limit": "42", "x-ratelimit-remaining": "1", "x-ratelimit-reset": "0", } gh = MockGitHubAPI(headers=rate_headers) await gh._make_request("GET", "/rate_limit", {}, "", sansio.accept_format()) assert gh.rate_limit.limit == 42
async def test_url_formatted_with_base_url(self): """The URL is appropriately formatted.""" gh = MockGitHubAPI(base_url="https://my.host.com") await gh._make_request( "GET", "/users/octocat/following{/other_user}", {"other_user": "******"}, "", sansio.accept_format(), ) assert gh.url == "https://my.host.com/users/octocat/following/brettcannon"
async def test_auth_headers_with_passed_jwt(self): """Test the authorization header with the passed jwt.""" accept = sansio.accept_format() gh = MockGitHubAPI() await gh._make_request("GET", "/rate_limit", {}, "", accept, jwt="json web token") assert gh.headers["user-agent"] == "test_abc" assert gh.headers["accept"] == accept assert gh.headers["authorization"] == "bearer json web token"
async def getiter(gh, url, identifier, **url_vars): data = b"" accept = accept_format() data, more = await gh._make_request("GET", url, url_vars, data, accept) if isinstance(data, dict) and identifier in data: data = data[identifier] for item in data: yield item if more: async for item in getiter(gh, more, identifier, **url_vars): yield item
async def test_make_request_passing_token_and_jwt(self): """Test that passing both jwt and oauth_token raises ValueError.""" accept = sansio.accept_format() gh = MockGitHubAPI() with pytest.raises(ValueError) as exc_info: await gh._make_request( "GET", "/rate_limit", {}, "", accept, jwt="json web token", oauth_token="oauth token", ) assert str(exc_info.value) == "Cannot pass both oauth_token and jwt."
async def get_issue_labels(self, gh): """Get the issue's labels.""" count = 0 accept = ','.join([sansio.accept_format(), 'application/vnd.github.symmetra-preview+json']) async for label in gh.getiter(self.issue_labels_url, {'number': self.number}, accept=accept): # Not sure how many get returned before it must page, so sleep for # one second on the arbitrary value of 20. That is a lot of labels for # one issue, so it is probably not going to trigger often. count += 1 if (count % 20) == 0: await asyncio.sleep(1) yield label['name']
async def startup(): """ On startup do some initial requests to GitHub API. This has two purposes: - It lists our installations, allowing us to check if we are still in sync over time (in regards to installations) - Check if our JWT it valid to communicate with GitHub API. """ async with GitHubAPIContext() as github_api: installations = await github_api.getitem( "/app/installations", accept=accept_format(version="machine-man-preview"), jwt=get_jwt()) for installation in installations: _github_installations[installation["id"]] = None log.info("Startup done; found %d installations", len(_github_installations))
"""Utilities.""" import asyncio import base64 import yaml import traceback import sys import os from gidgethub import sansio, InvalidField try: from yaml import CLoader as Loader except ImportError: from yaml import Loader LABEL_HEADER = ','.join( [sansio.accept_format(), 'application/vnd.github.symmetra-preview+json']) REACTION_HEADER = ','.join([ sansio.accept_format(), 'application/vnd.github.squirrel-girl-preview+json' ]) HTML_HEADER = sansio.accept_format(media="html") SINGLE_VALUES = { 'brace_expansion', 'extended_glob', 'case_insensitive', 'triage_label', 'review_label', 'delete_labels' } LIST_VALUES = { 'labels', 'rules', 'wip', 'lgtm_remove', 'lgtm_add_issue', 'lgtm_add_pull_request', 'triage_skip', 'triage_remove', 'review_skip', 'review_remove' }
def test_version(self): expect = "application/vnd.github.cloak-preview+json" assert sansio.accept_format(version="cloak-preview") == expect
def test_no_json(self): expect = "application/vnd.github.v3.raw" assert sansio.accept_format(media="raw", json=False) == expect
def test_format(self): expect = "application/vnd.github.v3.raw+json" assert sansio.accept_format(media="raw") == expect
async def from_github(cls, httpsession, github_org='lsst-ts', github_repo='ts_xml', git_ref='develop', github_user=None, github_token=None, cache_dir='~/.kafkaefd/github'): """Load the ``ts_sal`` repository directly from GitHub. Parameters ---------- httpsession : `aiohttp.ClientSession` Session from aiohttp. github_org : `str` Owner of the GitHub repository. github_repo : `str` Name of the repository. git_ref : `str` Branch or tag name to check out. github_user : `str` Your GitHub username, if providing a ``github_token``. github_token : `str` GitHub token (such as a personal access token). Returns ------- repo : `SalXmlRepo` SAL XML repository. """ # Cache for files downloaded from GitHub cache_root = Path(cache_dir).expanduser() if github_user is None: github_user = '******' ghclient = gh_aiohttp.GitHubAPI(httpsession, github_user, oauth_token=github_token, cache=cache) git_sha = await ghclient.getitem('/repos{/owner}{/repo}/commits{/ref}', url_vars={ 'owner': github_org, 'repo': github_repo, 'ref': git_ref }, accept=accept_format(media='sha', json=False)) cache_dir = cache_root / git_sha if cache_dir.exists(): # Load XML files from the cache topics = await SalXmlRepo._load_xml_files_from_cache(cache_dir) else: # Download XML files from GitHub API, then cache for second # execution cache_dir.mkdir(parents=True, exist_ok=True) data = await ghclient.getitem( '/repos{/owner}{/repo}/git/trees{/sha}?recursive=1', url_vars={ 'owner': github_org, 'repo': github_repo, 'sha': git_sha }) if data['truncated']: raise RuntimeError('/git/trees result is truncated') paths = [] for blob in data['tree']: if blob['type'] != 'blob': continue if blob['path'].startswith('sal_interfaces/') \ and blob['path'].endswith('.xml'): paths.append((blob['path'], blob['url'])) tasks = [] sem = asyncio.Semaphore(10) for path, url in paths: tasks.append( asyncio.ensure_future( cls._get_blob_gh(ghclient, sem, path, url, cache_dir))) filedata = await asyncio.gather(*tasks) topics = {} for path, data in filedata: topics.update(SalXmlRepo._parse_xml_topics(path, data)) print('GitHub rate limit: {}'.format(ghclient.rate_limit)) return cls(topics)
async def test_client_part_of_app(): app = GithubApp( user_agent=TEST_USER_AGENT, app_id=TEST_APP_ID, private_key=TEST_PRIVATE_KEY, webhook_secret=TEST_WEBHOOK_SECRET, ) assert app.app_client.app is app # github actually won't let any app client access the /rate_limit # endpoint, because app credentials are so locked down. But we can look up # information about ourself! data = await app.app_client.getitem( "/app", accept=accept_format(version="machine-man-preview")) assert glom(data, "name") == "snekomatic-test" # We can get an installation token token = await app.token_for(TEST_INSTALLATION_ID) # They're cached token2 = await app.token_for(TEST_INSTALLATION_ID) assert token == token2 # And the client works too: i_client = app.client_for(TEST_INSTALLATION_ID) assert i_client.app is app assert i_client.installation_id == TEST_INSTALLATION_ID data = await i_client.getitem("/rate_limit") assert "rate" in data # Now we'll cheat and trick the app into thinking that the token is # expiring, and check that the client automatically renews it. soon = pendulum.now().add(seconds=10) app._installation_tokens[TEST_INSTALLATION_ID].expires_at = soon # The client still works... i_client = app.client_for(TEST_INSTALLATION_ID) data = await i_client.getitem("/rate_limit") assert "rate" in data # ...but the token has changed. assert token != await app.token_for(TEST_INSTALLATION_ID) # And let's do that again, but this time we'll have two tasks try to fetch # the token at the same time. soon = pendulum.now().add(seconds=10) app._installation_tokens[TEST_INSTALLATION_ID].expires_at = soon tokens = [] async def get_token(): tokens.append(await app.token_for(TEST_INSTALLATION_ID)) async with trio.open_nursery() as nursery: nursery.start_soon(get_token) nursery.start_soon(get_token) # They both end up with the same token, demonstrating that they didn't do # two independent fetches assert len(tokens) == 2 assert len(set(tokens)) == 1
def test_defaults(self): assert sansio.accept_format() == "application/vnd.github.v3+json"