async def _retrieve_subscription_from_db(
cls: typing.Type[SubscriptionT], redis: utils.RedisCache, owner_id: int
) -> SubscriptionT:
print(config.SUBSCRIPTION_BASE_URL)
print(f"{config.SUBSCRIPTION_BASE_URL}/on-premise/subscription/{owner_id}")
async with http.AsyncClient() as client:
try:
resp = await client.get(
f"{config.SUBSCRIPTION_BASE_URL}/on-premise/subscription/{owner_id}",
headers={"Authorization": f"token {config.SUBSCRIPTION_TOKEN}"},
)
except http.HTTPUnauthorized:
LOG.critical(
"The SUBSCRIPTION_TOKEN is invalid, the subscription can't be checked"
)
raise exceptions.MergifyNotInstalled()
except http.HTTPForbidden:
LOG.critical(
"The subscription attached to SUBSCRIPTION_TOKEN is not valid"
)
raise exceptions.MergifyNotInstalled()
else:
sub = resp.json()
if not sub["subscription_active"]:
LOG.critical(
"The subscription attached to SUBSCRIPTION_TOKEN is not active"
)
raise exceptions.MergifyNotInstalled()
return cls.from_dict(redis, owner_id, sub)
def __init__(self, app: httpx.AsyncClient,
repository_id: github_types.GitHubRepositoryIdType) -> None:
self._app = app
self._session = http.AsyncClient()
self._handled_events: asyncio.Queue[ForwardedEvent] = asyncio.Queue()
self._counter = 0
hostname = parse.urlparse(config.GITHUB_URL).hostname
self._namespace_endpoint = f"{config.TESTING_FORWARDER_ENDPOINT}/{hostname}/{config.INTEGRATION_ID}/{repository_id}"
async def api_call(url, method="post"):
data = os.urandom(250)
hmac = utils.compute_hmac(data)
async with http.AsyncClient() as client:
r = await client.request(
method, url, headers={"X-Hub-Signature": "sha1=" + hmac}, data=data
)
r.raise_for_status()
print(r.text)
async def test_client_temporary_HTTP_500(respx_mock: respx.MockRouter) -> None:
respx_mock.get("/").mock(side_effect=[
httpx.Response(500, text="This is a 5XX error"),
httpx.Response(500, text="This is a 5XX error"),
httpx.Response(500, text="This is a 5XX error"),
httpx.Response(200, text="It works now !"),
])
async with http.AsyncClient() as client:
await client.get("https://foobar/")
async def main():
parser = argparse.ArgumentParser()
parser.add_argument("--clean", action="store_true")
parser.add_argument("--dest", default="http://localhost:8802/event")
args = parser.parse_args()
logs.setup_logging()
payload_data = os.urandom(250)
payload_hmac = utils.compute_hmac(payload_data)
async with http.AsyncClient(
base_url="https://test-forwarder.mergify.io",
headers={"X-Hub-Signature": "sha1=" + payload_hmac},
) as session:
if args.clean:
r = await session.request("DELETE",
"/events-testing",
data=payload_data)
r.raise_for_status()
while True:
try:
resp = await session.request("GET",
"/events-testing",
data=payload_data)
events = resp.json()
for event in reversed(events):
LOG.info("")
LOG.info(
"==================================================")
LOG.info(
">>> GOT EVENT: %s %s/%s",
event["id"],
event["type"],
event["payload"].get("state",
event["payload"].get("action")),
)
data = json.dumps(event["payload"])
hmac = utils.compute_hmac(data.encode("utf8"))
await session.post(
args.dest,
headers={
"X-GitHub-Event": event["type"],
"X-GitHub-Delivery": event["id"],
"X-Hub-Signature": f"sha1={hmac}",
"Content-type": "application/json",
},
data=data,
)
except Exception:
LOG.error("event handling failure", exc_info=True)
time.sleep(1)
async def test_client_HTTP_500(respx_mock: respx.MockRouter) -> None:
respx_mock.get("https://foobar/").respond(500, text="This is a 5XX error")
async with http.AsyncClient() as client:
with pytest.raises(http.HTTPServerSideError) as exc_info:
await client.get("https://foobar/")
assert exc_info.value.message == "This is a 5XX error"
assert exc_info.value.status_code == 500
assert exc_info.value.response.status_code == 500
assert str(exc_info.value.request.url) == "https://foobar/"
async def _retrieve_subscription_from_db(cls, owner_id: int) -> "Subscription":
async with http.AsyncClient() as client:
try:
resp = await client.get(
f"{config.SUBSCRIPTION_BASE_URL}/engine/github-account/{owner_id}",
auth=(config.OAUTH_CLIENT_ID, config.OAUTH_CLIENT_SECRET),
)
except http.HTTPNotFound as e:
return cls(owner_id, False, e.message, {}, frozenset())
else:
sub = resp.json()
return cls.from_dict(owner_id, sub)
async def event_handler(
request: requests.Request,
redis_cache: utils.RedisCache = fastapi.Depends( # noqa: B008
redis.get_redis_cache
),
redis_stream: utils.RedisStream = fastapi.Depends( # noqa: B008
redis.get_redis_stream
),
) -> responses.Response:
event_type = request.headers.get("X-GitHub-Event")
event_id = request.headers.get("X-GitHub-Delivery")
data = await request.json()
try:
await github_events.filter_and_dispatch(
redis_cache, redis_stream, event_type, event_id, data
)
except github_events.IgnoredEvent as ie:
status_code = 200
reason = f"Event ignored: {ie.reason}"
else:
status_code = 202
reason = "Event queued"
if (
config.WEBHOOK_APP_FORWARD_URL
and config.WEBHOOK_FORWARD_EVENT_TYPES is not None
and event_type in config.WEBHOOK_FORWARD_EVENT_TYPES
):
raw = await request.body()
try:
async with http.AsyncClient(timeout=EVENT_FORWARD_TIMEOUT) as client:
await client.post(
config.WEBHOOK_APP_FORWARD_URL,
content=raw.decode(),
headers={
"X-GitHub-Event": event_type,
"X-GitHub-Delivery": event_id,
"X-Hub-Signature": request.headers.get("X-Hub-Signature"),
"User-Agent": request.headers.get("User-Agent"),
"Content-Type": request.headers.get("Content-Type"),
},
)
except httpx.TimeoutException:
LOG.warning(
"Fail to forward GitHub event",
event_type=event_type,
event_id=event_id,
sender=data["sender"]["login"],
)
return responses.Response(reason, status_code=status_code)
async def _retrieve_from_db(cls, redis: utils.RedisCache,
owner_id: int) -> "UserTokens":
async with http.AsyncClient() as client:
try:
resp = await client.get(
f"{config.SUBSCRIPTION_BASE_URL}/engine/user_tokens/{owner_id}",
auth=(config.OAUTH_CLIENT_ID, config.OAUTH_CLIENT_SECRET),
)
except http.HTTPNotFound:
return cls(redis, owner_id, {})
else:
tokens = resp.json()
return cls(redis, owner_id, tokens["tokens"])
async def get_installation_from_id(installation_id):
url = f"{config.GITHUB_API_URL}/app/installations/{installation_id}"
async with http.AsyncClient(auth=GithubBearerAuth(),
**http.DEFAULT_CLIENT_OPTIONS) as client:
try:
installation = (await client.get(url)).json()
permissions_need_to_be_updated(installation)
return installation
except http.HTTPNotFound as e:
LOG.debug(
"Mergify not installed",
error_message=e.message,
)
raise exceptions.MergifyNotInstalled()
async def test_client_HTTP_400(httpserver: httpserver.HTTPServer) -> None:
httpserver.expect_oneshot_request("/").respond_with_json(
{"message": "This is an 4XX error"}, status=400)
async with http.AsyncClient() as client:
with pytest.raises(http.HTTPClientSideError) as exc_info:
await client.get(httpserver.url_for("/"))
assert exc_info.value.message == "This is an 4XX error"
assert exc_info.value.status_code == 400
assert exc_info.value.response.status_code == 400
assert str(exc_info.value.request.url) == httpserver.url_for("/")
httpserver.check_assertions()
async def token(request: requests.Request):
authorization = request.headers.get("Authorization")
if authorization:
if authorization.startswith("token "):
try:
options = http.DEFAULT_CLIENT_OPTIONS.copy()
options["headers"]["Authorization"] = authorization
async with http.AsyncClient(base_url=config.GITHUB_API_URL,
**options) as client:
await client.get("/user")
return
except http.HTTPStatusError as e:
raise fastapi.HTTPException(status_code=e.response.status_code)
raise fastapi.HTTPException(status_code=403)
async def _retrieve_subscription_from_db(
cls: typing.Type[SubscriptionT], redis: utils.RedisCache, owner_id: int
) -> SubscriptionT:
async with http.AsyncClient() as client:
try:
resp = await client.get(
f"{config.SUBSCRIPTION_BASE_URL}/engine/subscription/{owner_id}",
auth=(config.OAUTH_CLIENT_ID, config.OAUTH_CLIENT_SECRET),
)
except http.HTTPNotFound as e:
return cls(redis, owner_id, False, e.message, frozenset())
else:
sub = resp.json()
return cls.from_dict(redis, owner_id, sub)
async def _retrieve_subscription_from_db(cls, owner_id):
LOG.info("Subscription not cached, retrieving it...",
gh_owner=owner_id)
async with http.AsyncClient() as client:
try:
resp = await client.get(
f"{config.SUBSCRIPTION_BASE_URL}/engine/github-account/{owner_id}",
auth=(config.OAUTH_CLIENT_ID, config.OAUTH_CLIENT_SECRET),
)
except http.HTTPNotFound as e:
return cls(owner_id, False, e.message, {}, frozenset())
else:
sub = resp.json()
sub["tokens"] = dict((login, token["access_token"])
for login, token in sub["tokens"].items())
return cls.from_dict(owner_id, sub)
async def test_client_HTTP_500(httpserver: httpserver.HTTPServer) -> None:
httpserver.expect_request("/").respond_with_data("This is an 5XX error", status=500)
async with http.AsyncClient() as client:
with pytest.raises(http.HTTPServerSideError) as exc_info:
await client.get(httpserver.url_for("/"))
# 5 retries
assert len(httpserver.log) == 5
assert exc_info.value.message == "This is an 5XX error"
assert exc_info.value.status_code == 500
assert exc_info.value.response.status_code == 500
assert str(exc_info.value.request.url) == httpserver.url_for("/")
httpserver.check_assertions()
async def marketplace_handler(
request: requests.Request,
redis_links: redis_utils.RedisLinks = fastapi.Depends( # noqa: B008
redis.get_redis_links),
) -> responses.Response:
event_type = request.headers.get("X-GitHub-Event")
event_id = request.headers.get("X-GitHub-Delivery")
data = await request.json()
LOG.info(
"Marketplace event",
event_type=event_type,
event_id=event_id,
sender=data["sender"]["login"],
gh_owner=data["marketplace_purchase"]["account"]["login"],
)
await subscription.Subscription.delete_subscription(
redis_links.cache, data["marketplace_purchase"]["account"]["id"])
if config.WEBHOOK_MARKETPLACE_FORWARD_URL:
raw = await request.body()
try:
async with http.AsyncClient(
timeout=EVENT_FORWARD_TIMEOUT) as client:
await client.post(
config.WEBHOOK_MARKETPLACE_FORWARD_URL,
content=raw.decode(),
headers={
"X-GitHub-Event": event_type,
"X-GitHub-Delivery": event_id,
"X-Hub-Signature":
request.headers.get("X-Hub-Signature"),
"User-Agent": request.headers.get("User-Agent"),
"Content-Type": request.headers.get("Content-Type"),
},
)
except httpx.TimeoutException:
LOG.warning(
"Fail to forward Marketplace event",
event_type=event_type,
event_id=event_id,
sender=data["sender"]["login"],
gh_owner=data["marketplace_purchase"]["account"]["login"],
)
return responses.Response("Event queued", status_code=202)
async def get_installation(account):
owner = account["login"]
account_type = "users" if account["type"].lower() == "user" else "orgs"
url = f"{config.GITHUB_API_URL}/{account_type}/{owner}/installation"
async with http.AsyncClient(auth=GithubBearerAuth(),
**http.DEFAULT_CLIENT_OPTIONS) as client:
try:
installation = (await client.get(url)).json()
permissions_need_to_be_updated(installation)
return installation
except http.HTTPNotFound as e:
LOG.debug(
"Mergify not installed",
gh_owner=owner,
error_message=e.message,
)
raise exceptions.MergifyNotInstalled()
async def test_client_temporary_HTTP_500(
httpserver: httpserver.HTTPServer) -> None:
httpserver.expect_oneshot_request("/").respond_with_data(
"This is an 5XX error", status=500)
httpserver.expect_oneshot_request("/").respond_with_data(
"This is an 5XX error", status=500)
httpserver.expect_oneshot_request("/").respond_with_data(
"This is an 5XX error", status=500)
httpserver.expect_request("/").respond_with_data("It works now !",
status=200)
async with http.AsyncClient() as client:
await client.get(httpserver.url_for("/"))
# 4 retries
assert len(httpserver.log) == 4
httpserver.check_assertions()
async def _do_test_client_retry_429(respx_mock: respx.MockRouter,
retry_after: str) -> datetime.datetime:
records: typing.List[datetime.datetime] = []
def record_date(_):
if records:
records.append(date.utcnow())
return httpx.Response(200, text="It works now !")
else:
records.append(date.utcnow())
return httpx.Response(
429,
text="This is a 429 error",
headers={"Retry-After": retry_after},
)
respx_mock.get("/").mock(side_effect=record_date)
async with http.AsyncClient() as client:
await client.get("https://foobar/")
return records[1]
async def get_repositories_setuped(
token: str, install_id: int
) -> typing.List[github_types.GitHubRepository]: # pragma: no cover
repositories = []
url = f"{config.GITHUB_API_URL}/user/installations/{install_id}/repositories"
token = f"token {token}"
async with http.AsyncClient(headers={
"Authorization": token,
"Accept": "application/vnd.github.machine-man-preview+json",
"User-Agent": "PyGithub/Python",
}, ) as session:
while True:
response = await session.get(url)
if response.status_code == 200:
repositories.extend(response.json()["repositories"])
if "next" in response.links:
url = response.links["next"]["url"]
continue
else:
return repositories
else:
response.raise_for_status()
async def _do_test_client_retry_429(
httpserver: httpserver.HTTPServer, retry_after: str, expected_seconds: int
) -> None:
records = []
def record_date(_):
records.append(datetime.datetime.utcnow())
return Response("It works now !", 200)
httpserver.expect_oneshot_request("/").respond_with_data(
"This is an 429 error", status=429, headers={"Retry-After": retry_after}
)
httpserver.expect_request("/").respond_with_handler(record_date)
async with http.AsyncClient() as client:
now = datetime.datetime.utcnow()
await client.get(httpserver.url_for("/"))
assert len(httpserver.log) == 2
elapsed_seconds = (records[0] - now).total_seconds()
assert expected_seconds - 1 < elapsed_seconds <= expected_seconds + 1
httpserver.check_assertions()
async def send_seats(seats: SeatsCountResultT) -> None:
async with http.AsyncClient() as client:
try:
await client.post(
f"{config.SUBSCRIPTION_BASE_URL}/on-premise/report",
headers={
"Authorization": f"token {config.SUBSCRIPTION_TOKEN}"
},
json={
"write_users": seats.write_users,
"active_users": seats.active_users,
"engine_version": config.VERSION,
# Deprecated version
"seats": seats.write_users,
},
)
except Exception as exc:
if exceptions.should_be_ignored(exc):
return
elif exceptions.need_retry(exc):
raise tenacity.TryAgain
else:
raise
async def test_message_format_client_HTTP_400(
respx_mock: respx.MockRouter) -> None:
respx_mock.get("https://foobar/").respond(400,
json={
"message":
"This is a 4XX error",
"documentation_url":
"fake_url"
})
async with http.AsyncClient() as client:
with pytest.raises(http.HTTPClientSideError) as exc_info:
await client.get("https://foobar/")
assert exc_info.value.message == "This is a 4XX error"
respx_mock.get("https://foobar/").respond(
400,
json={
"message": "error message",
"errors": ["This is a 4XX error"],
"documentation_url": "fake_url",
},
)
async with http.AsyncClient() as client:
with pytest.raises(http.HTTPClientSideError) as exc_info:
await client.get("https://foobar/")
assert exc_info.value.message == "This is a 4XX error"
respx_mock.get("https://foobar/").respond(
400,
json={
"message": "This is a 4XX error",
"errors": [{
"resource": "test",
"field": "test",
"code": "test"
}],
"documentation_url": "fake_url",
},
)
async with http.AsyncClient() as client:
with pytest.raises(http.HTTPClientSideError) as exc_info:
await client.get("https://foobar/")
assert exc_info.value.message == "This is a 4XX error"
respx_mock.get("https://foobar/").respond(
400,
json={
"message":
"error message",
"errors": [{
"resource": "test",
"code": "test",
"field": "test",
"message": "This is a 4XX error",
}],
"documentation_url":
"fake_url",
},
)
async with http.AsyncClient() as client:
with pytest.raises(http.HTTPClientSideError) as exc_info:
await client.get("https://foobar/")
assert exc_info.value.message == "This is a 4XX error"
respx_mock.get("https://foobar/").respond(
400,
json={
"not_message_key": "false_key",
"documentation_url": "fake_url",
},
)
async with http.AsyncClient() as client:
with pytest.raises(http.HTTPClientSideError) as exc_info:
await client.get("https://foobar/")
assert exc_info.value.message == "No error message provided by GitHub"
async def http_post(*args, **kwargs):
# Set the maximum timeout to 3 seconds: GitHub is not going to wait for
# more than 10 seconds for us to accept an event, so if we're unable to
# forward an event in 3 seconds, just drop it.
async with http.AsyncClient(timeout=5) as client:
await client.post(*args, **kwargs)
async def test_client_connection_error() -> None:
async with http.AsyncClient() as client:
with pytest.raises(http.RequestError):
await client.get("http://localhost:12345")
def __init__(self, app):
self._app = app
self._session = http.AsyncClient()
self._handled_events = asyncio.Queue()
self._counter = 0
async def api_call(*args: typing.Any, **kwargs: typing.Any) -> None:
async with http.AsyncClient() as client:
r = await client.request(*args, **kwargs)
r.raise_for_status()
print(r.text)