async def refresh_branch(
owner: github_types.GitHubLogin,
repo_name: github_types.GitHubRepositoryName,
branch: str,
redis_cache: utils.RedisCache = fastapi.Depends( # noqa: B008
redis.get_redis_cache
),
redis_stream: utils.RedisStream = fastapi.Depends( # noqa: B008
redis.get_redis_stream
),
) -> responses.Response:
async with github.aget_client(owner_name=owner) as client:
try:
repository = await client.item(f"/repos/{owner}/{repo_name}")
except http.HTTPNotFound:
return responses.JSONResponse(
status_code=404, content="repository not found"
)
await github_events.send_refresh(
redis_cache,
redis_stream,
repository,
ref=github_types.GitHubRefType(f"refs/heads/{branch}"),
)
return responses.Response("Refresh queued", status_code=202)
async def test_run_command_with_rerun_and_without_user(redis_cache,
monkeypatch):
client = github.aget_client(owner_name="Mergifyio", owner_id=123)
ctxt = await _create_context(redis_cache, client)
http_calls = []
async def mock_post(*args, **kwargs):
http_calls.append((args, kwargs))
return
monkeypatch.setattr(client, "post", mock_post)
await handle(
ctxt=ctxt,
mergify_config={},
comment="@mergifyio something",
user=None,
rerun=True,
)
assert ("Sorry but I didn't understand the command."
in http_calls[0][1]["json"]["body"])
async def refresh_pull(
owner_login: github_types.GitHubLogin,
repo_name: github_types.GitHubRepositoryName,
pull_request_number: github_types.GitHubPullRequestNumber,
action: github_types.GitHubEventRefreshActionType = "user",
redis_links: redis_utils.RedisLinks = fastapi.Depends( # noqa: B008
redis.get_redis_links),
) -> responses.Response:
action = RefreshActionSchema(action)
installation_json = await github.get_installation_from_login(owner_login)
async with github.aget_client(installation_json) as client:
try:
repository = await client.item(f"/repos/{owner_login}/{repo_name}")
except http.HTTPNotFound:
return responses.JSONResponse(status_code=404,
content="repository not found")
await utils.send_pull_refresh(
redis_links.stream,
repository,
action=action,
pull_request_number=pull_request_number,
source="API",
)
return responses.Response("Refresh queued", status_code=202)
async def refresh_pull(
owner: github_types.GitHubLogin,
repo_name: github_types.GitHubRepositoryName,
pull_request_number: github_types.GitHubPullRequestNumber,
action: github_types.GitHubEventRefreshActionType = "user",
redis_cache: utils.RedisCache = fastapi.Depends( # noqa: B008
redis.get_redis_cache),
redis_stream: utils.RedisStream = fastapi.Depends( # noqa: B008
redis.get_redis_stream),
) -> responses.Response:
action = RefreshActionSchema(action)
async with github.aget_client(owner_name=owner) as client:
try:
repository = await client.item(f"/repos/{owner}/{repo_name}")
except http.HTTPNotFound:
return responses.JSONResponse(status_code=404,
content="repository not found")
await utils.send_refresh(
redis_cache,
redis_stream,
repository,
pull_request_number=pull_request_number,
action=action,
)
return responses.Response("Refresh queued", status_code=202)
async def _simulator(redis_cache, pull_request_rules, owner, repo, pull_number, token):
try:
if token:
auth = github.GithubTokenAuth(token)
else:
auth = github.get_auth(owner)
async with github.aget_client(auth=auth) as client:
try:
data = await client.item(f"/repos/{owner}/{repo}/pulls/{pull_number}")
except http.HTTPNotFound:
raise PullRequestUrlInvalid(
message=f"Pull request {owner}/{repo}/pulls/{pull_number} not found"
)
sub = await subscription.Subscription.get_subscription(
redis_cache, data["base"]["user"]["id"]
)
installation = context.Installation(
data["base"]["user"]["id"],
owner,
sub,
client,
redis_cache,
)
repository = context.Repository(installation, data["base"]["repo"])
ctxt = await repository.get_pull_request_context(data["number"], data)
ctxt.sources = [{"event_type": "mergify-simulator", "data": []}]
match = await pull_request_rules.get_pull_request_rule(ctxt)
return await actions_runner.gen_summary(ctxt, pull_request_rules, match)
except exceptions.MergifyNotInstalled:
raise PullRequestUrlInvalid(
message=f"Mergify not installed on repository '{owner}/{repo}'"
)
async def refresh_branch(
owner_login: github_types.GitHubLogin,
repo_name: github_types.GitHubRepositoryName,
branch: str,
redis_cache: utils.RedisCache = fastapi.Depends( # noqa: B008
redis.get_redis_cache),
redis_stream: utils.RedisStream = fastapi.Depends( # noqa: B008
redis.get_redis_stream),
) -> responses.Response:
installation_json = await github.get_installation_from_login(owner_login)
async with github.aget_client(installation_json) as client:
try:
repository = await client.item(f"/repos/{owner_login}/{repo_name}")
except http.HTTPNotFound:
return responses.JSONResponse(status_code=404,
content="repository not found")
await utils.send_branch_refresh(
redis_cache,
redis_stream,
repository,
action="user",
source="API",
ref=github_types.GitHubRefType(f"refs/heads/{branch}"),
)
return responses.Response("Refresh queued", status_code=202)
async def test_owner(
installation: github_types.GitHubInstallation = fastapi.
Depends( # noqa: B008
security.get_installation),
) -> ResponseTest:
async with github.aget_client(installation) as client:
org = await client.item(f"/user/{installation['account']['id']}")
return ResponseTest(org["login"])
async def on_each_event(event: github_types.GitHubEventIssueComment) -> None:
action = load_action(event["comment"]["body"])
if action:
owner = event["repository"]["owner"]["login"]
repo = event["repository"]["name"]
async with github.aget_client(owner) as client:
await client.post(
f"/repos/{owner}/{repo}/issues/comments/{event['comment']['id']}/reactions",
json={"content": "+1"},
api_version="squirrel-girl",
) # type: ignore[call-arg]
async def on_each_event(event: github_types.GitHubEventIssueComment) -> None:
action_classes = actions.get_commands()
match = COMMAND_MATCHER.search(event["comment"]["body"])
if match and match[1] in action_classes:
owner = event["repository"]["owner"]["login"]
repo = event["repository"]["name"]
async with github.aget_client(owner) as client:
await client.post(
f"/repos/{owner}/{repo}/issues/comments/{event['comment']['id']}/reactions",
json={"content": "+1"},
api_version="squirrel-girl",
)
async def refresh_repo(
owner: github_types.GitHubLogin,
repo_name: github_types.GitHubRepositoryName) -> responses.Response:
global _AREDIS_STREAM, _AREDIS_CACHE
async with github.aget_client(owner_name=owner) as client:
try:
repository = await client.item(f"/repos/{owner}/{repo_name}")
except http.HTTPNotFound:
return responses.JSONResponse(status_code=404,
content="repository not found")
await github_events.send_refresh(_AREDIS_CACHE, _AREDIS_STREAM, repository)
return responses.Response("Refresh queued", status_code=202)
async def test_client_401_raise_ratelimit(
httpserver: httpserver.HTTPServer) -> None:
owner = github_types.GitHubLogin("owner")
repo = "repo"
httpserver.expect_request("/users/owner/installation").respond_with_json({
"id":
12345,
"target_type":
"User",
"permissions": {
"checks": "write",
"contents": "write",
"pull_requests": "write",
},
"account": {
"login": "******",
"id": 12345
},
})
httpserver.expect_request(
"/app/installations/12345/access_tokens").respond_with_json(
{
"token": "<token>",
"expires_at": "2100-12-31T23:59:59Z"
},
headers={
"X-RateLimit-Remaining": 5000,
"X-RateLimit-Reset": 1234567890
},
)
httpserver.expect_oneshot_request(
"/repos/owner/repo/pull/1").respond_with_json(
{"message": "quota !"},
status=403,
headers={
"X-RateLimit-Remaining": 0,
"X-RateLimit-Reset": 1234567890
},
)
with mock.patch(
"mergify_engine.config.GITHUB_API_URL",
httpserver.url_for("/")[:-1],
):
async with github.aget_client(owner) as client:
with pytest.raises(exceptions.RateLimited):
await client.item(f"/repos/{owner}/{repo}/pull/1")
httpserver.check_assertions()
async def create_initial_summary(
redis: utils.RedisCache,
event: github_types.GitHubEventPullRequest) -> None:
owner = event["repository"]["owner"]
repo = event["pull_request"]["base"]["repo"]
if not await redis.exists(
context.Repository.get_config_location_cache_key(
owner["login"],
repo["name"],
)):
# Mergify is probably not activated on this repo
return
# NOTE(sileht): It's possible that a "push" event creates a summary before we
# received the pull_request/opened event.
# So we check first if a summary does not already exists, to not post
# the summary twice. Since this method can ran in parallel of the worker
# this is not a 100% reliable solution, but if we post a duplicate summary
# check_api.set_check_run() handle this case and update both to not confuse users.
summary_exists = await context.Context.summary_exists(
redis, owner["id"], repo["id"], event["pull_request"])
if summary_exists:
return
installation_json = await github.get_installation_from_account_id(
owner["id"])
async with github.aget_client(installation_json) as client:
post_parameters = {
"name": constants.SUMMARY_NAME,
"head_sha": event["pull_request"]["head"]["sha"],
"status": check_api.Status.IN_PROGRESS.value,
"started_at": date.utcnow().isoformat(),
"details_url": f"{event['pull_request']['html_url']}/checks",
"output": {
"title": "Your rules are under evaluation",
"summary": "Be patient, the page will be updated soon.",
},
"external_id": str(event["pull_request"]["number"]),
}
try:
await client.post(
f"/repos/{event['pull_request']['base']['user']['login']}/{event['pull_request']['base']['repo']['name']}/check-runs",
api_version="antiope",
json=post_parameters,
)
except http.HTTPClientSideError as e:
if e.status_code == 422 and "No commit found for SHA" in e.message:
return
raise
async def test_github_async_client(self):
rules = {
"pull_request_rules": [{
"name":
"simulator",
"conditions": [f"base!={self.master_branch_name}"],
"actions": {
"merge": {}
},
}]
}
other_branch = self.get_full_branch_name("other")
await self.setup_repo(yaml.dump(rules), test_branches=[other_branch])
p1, _ = await self.create_pr()
p2, _ = await self.create_pr()
await self.create_pr(base=other_branch)
client = github.aget_client("mergifyio-testing")
url = f"/repos/mergifyio-testing/{self.REPO_NAME}/pulls"
pulls = [p async for p in client.items(url)]
self.assertEqual(3, len(pulls))
pulls = [p async for p in client.items(url, per_page=1)]
self.assertEqual(3, len(pulls))
pulls = [p async for p in client.items(url, per_page=1, page=2)]
self.assertEqual(2, len(pulls))
pulls = [
p async for p in client.items(url, base=other_branch, state="all")
]
self.assertEqual(1, len(pulls))
pulls = [p async for p in client.items(url, base="unknown")]
self.assertEqual(0, len(pulls))
pull = await client.item(f"{url}/{p1['number']}")
self.assertEqual(p1["number"], pull["number"])
pull = await client.item(f"{url}/{p2['number']}")
self.assertEqual(p2["number"], pull["number"])
with self.assertRaises(http.HTTPStatusError) as ctxt:
await client.item(f"{url}/10000000000")
self.assertEqual(404, ctxt.exception.response.status_code)
async def test_client_401_raise_ratelimit(
respx_mock: respx.MockRouter) -> None:
owner_id = github_types.GitHubAccountIdType(12345)
owner_login = github_types.GitHubLogin("owner")
repo = "repo"
respx_mock.get("/user/12345/installation").respond(
200,
json={
"id": 12345,
"target_type": "User",
"permissions": {
"checks": "write",
"contents": "write",
"pull_requests": "write",
},
"account": {
"login": "******",
"id": 12345
},
},
)
respx_mock.post("/app/installations/12345/access_tokens").respond(
200,
json={
"token": "<token>",
"expires_at": "2100-12-31T23:59:59Z"
},
headers={
"X-RateLimit-Remaining": "5000",
"X-RateLimit-Reset": "1234567890",
},
)
respx_mock.get("/repos/owner/repo/pull/1").respond(
403,
json={"message": "quota !"},
headers={
"X-RateLimit-Remaining": "0",
"X-RateLimit-Reset": "1234567890"
},
)
installation_json = await github.get_installation_from_account_id(owner_id)
async with github.aget_client(installation_json) as client:
with pytest.raises(exceptions.RateLimited):
await client.item(f"/repos/{owner_login}/{repo}/pull/1")
async def populate_with_collaborators_with_write_users_access(
self) -> None:
async with github.AsyncGithubClient(
auth=github_app.GithubBearerAuth(), ) as app_client:
async for installation in app_client.items(
"/app/installations",
resource_name="installations",
page_limit=100,
):
installation = typing.cast(github_types.GitHubInstallation,
installation)
org = SeatAccount(installation["account"]["id"],
installation["account"]["login"])
async with github.aget_client(installation) as client:
try:
async for repository in client.items(
"/installation/repositories",
list_items="repositories",
resource_name="repositories",
page_limit=100,
):
repository = typing.cast(
github_types.GitHubRepository, repository)
repo = SeatRepository(repository["id"],
repository["name"])
async for collaborator in client.items(
f"{repository['url']}/collaborators",
resource_name="collaborators",
page_limit=100,
):
if collaborator["permissions"]["push"]:
seat = SeatAccount(collaborator["id"],
collaborator["login"])
repo_seats = self.seats[org][repo]
if repo_seats["write_users"] is None:
repo_seats["write_users"] = {seat}
else:
repo_seats["write_users"].add(seat)
except exceptions.MergifyNotInstalled:
LOG.warning(
"can't retrieve collaborators with write users access",
account_id=installation["account"]["id"],
account_login=installation["account"]["login"],
suspended_at=installation["suspended_at"],
suspended_by=installation["suspended_by"],
)
async def test_run_command_with_user(user_id, permission, result, redis_cache,
monkeypatch):
client = github.aget_client(owner_name="Mergifyio", owner_id=123)
ctxt = await _create_context(redis_cache, client)
user = github_types.GitHubAccount(
{
"id": user_id,
"login": "******",
"type": "Bot",
"avatar_url": "https://avatars.githubusercontent.com/u/583231?v=4",
}, )
class MockResponse:
@staticmethod
def json():
return {
"permission": permission,
"user": {
"login": "******",
},
}
async def mock_get(*args, **kwargs):
return MockResponse()
monkeypatch.setattr(client, "get", mock_get)
http_calls = []
async def mock_post(*args, **kwargs):
http_calls.append((args, kwargs))
return
monkeypatch.setattr(client, "post", mock_post)
await handle(ctxt=ctxt,
mergify_config={},
comment="@mergifyio something",
user=user)
assert len(http_calls) == 1
assert result in http_calls[0][1]["json"]["body"]
async def create_initial_summary(
redis: utils.RedisCache,
event: github_types.GitHubEventPullRequest) -> None:
owner = event["repository"]["owner"]["login"]
if not await redis.exists(
context.Repository.get_config_location_cache_key(
event["pull_request"]["base"]["repo"]["owner"]["login"],
event["pull_request"]["base"]["repo"]["name"],
)):
# Mergify is probably not activated on this repo
return
# NOTE(sileht): It's possible that a "push" event creates a summary before we
# received the pull_request/opened event.
# So we check first if a summary does not already exists, to not post
# the summary twice. Since this method can ran in parallel of the worker
# this is not a 100% reliable solution, but if we post a duplicate summary
# check_api.set_check_run() handle this case and update both to not confuse users.
sha = await context.Context.get_cached_last_summary_head_sha_from_pull(
redis, event["pull_request"])
if sha is not None or sha == event["pull_request"]["head"]["sha"]:
return
async with github.aget_client(owner) as client:
post_parameters = {
"name": context.Context.SUMMARY_NAME,
"head_sha": event["pull_request"]["head"]["sha"],
"status": check_api.Status.IN_PROGRESS.value,
"started_at": utils.utcnow().isoformat(),
"details_url": f"{event['pull_request']['html_url']}/checks",
"output": {
"title": "Your rules are under evaluation",
"summary": "Be patient, the page will be updated soon.",
},
}
await client.post(
f"/repos/{event['pull_request']['base']['user']['login']}/{event['pull_request']['base']['repo']['name']}/check-runs",
api_version="antiope", # type: ignore[call-arg]
json=post_parameters,
)
async def test_github_async_client_with_owner_id(self):
rules = {
"pull_request_rules": [
{
"name": "fake PR",
"conditions": ["base=master"],
"actions": {"merge": {}},
}
]
}
await self.setup_repo(yaml.dump(rules))
p, _ = await self.create_pr()
client = github.aget_client(owner_id=self.o_integration.id)
url = f"/repos/{self.o_integration.login}/{self.r_o_integration.name}/pulls"
pulls = [p async for p in client.items(url)]
self.assertEqual(1, len(pulls))
async def queues_by_owner_id(owner_id):
global _AREDIS_CACHE
queues = collections.defaultdict(dict)
async for queue in _AREDIS_CACHE.scan_iter(
match=f"merge-queue~{owner_id}~*"):
_, _, repo_id, branch = queue.split("~")
async with github.aget_client(owner_id=owner_id) as client:
try:
repo = await client.item(f"/repositories/{repo_id}")
except exceptions.RateLimited:
return responses.JSONResponse(
status_code=403,
content={
"message":
f"{client.auth.owner} account with {client.auth.owner_id} ID, rate limited by GitHub"
},
)
queues[client.auth.owner + "/" + repo["name"]][branch] = [
int(pull) async for pull, _ in _AREDIS_CACHE.zscan_iter(queue)
]
return responses.JSONResponse(status_code=200, content=queues)
async def get_repository_context(
owner: github_types.GitHubLogin = fastapi.Path( # noqa: B008
..., description="The owner of the repository"),
repository: github_types.GitHubRepositoryName = fastapi.Path( # noqa: B008
..., description="The name of the repository"),
redis_links: redis_utils.RedisLinks = fastapi.Depends( # noqa: B008
redis.get_redis_links),
installation_json: github_types.GitHubInstallation = fastapi.
Depends( # noqa: B008
get_installation),
) -> typing.AsyncGenerator[context.Repository, None]:
async with github.aget_client(installation_json) as client:
try:
# Check this token has access to this repository
repo = typing.cast(
github_types.GitHubRepository,
await client.item(f"/repos/{owner}/{repository}"),
)
except (http.HTTPNotFound, http.HTTPForbidden, http.HTTPUnauthorized):
raise fastapi.HTTPException(status_code=404)
sub = await subscription.Subscription.get_subscription(
redis_links.cache, installation_json["account"]["id"])
installation = context.Installation(installation_json, sub, client,
redis_links)
repository_ctxt = installation.get_repository_from_github_data(repo)
# NOTE(sileht): Since this method is used as fastapi Depends only, it's safe to set this
# for the ongoing http request
sentry_sdk.set_user(
{"username": repository_ctxt.installation.owner_login})
sentry_sdk.set_tag("gh_owner",
repository_ctxt.installation.owner_login)
sentry_sdk.set_tag("gh_repo", repository_ctxt.repo["name"])
yield repository_ctxt
async def refresh_pull(
owner: github_types.GitHubLogin,
repo_name: github_types.GitHubRepositoryName,
pull_request_number: github_types.GitHubPullRequestNumber,
action: github_types.GitHubEventRefreshActionType = "user",
) -> responses.Response:
action = RefreshActionSchema(action)
async with github.aget_client(owner_name=owner) as client:
try:
repository = await client.item(f"/repos/{owner}/{repo_name}")
except http.HTTPNotFound:
return responses.JSONResponse(status_code=404,
content="repository not found")
global _AREDIS_STREAM, _AREDIS_CACHE
await github_events.send_refresh(
_AREDIS_CACHE,
_AREDIS_STREAM,
repository,
pull_request_number=pull_request_number,
action=action,
)
return responses.Response("Refresh queued", status_code=202)
async def test_github_async_client(self) -> None:
rules = {
"pull_request_rules": [{
"name":
"simulator",
"conditions": [f"base!={self.main_branch_name}"],
"actions": {
"merge": {}
},
}]
}
other_branch = self.get_full_branch_name("other")
await self.setup_repo(yaml.dump(rules), test_branches=[other_branch])
p1 = await self.create_pr()
p2 = await self.create_pr()
await self.create_pr(base=other_branch)
installation_json = await github.get_installation_from_login(
github_types.GitHubLogin("mergifyio-testing"))
client = github.aget_client(installation_json)
url = f"/repos/mergifyio-testing/{self.RECORD_CONFIG['repository_name']}/pulls"
pulls = [
p async for p in client.items(
url, resource_name="pull", page_limit=5)
]
self.assertEqual(3, len(pulls))
pulls = [
p async for p in client.items(url,
params={"per_page": "1"},
resource_name="pull",
page_limit=5)
]
self.assertEqual(3, len(pulls))
pulls = [
p async for p in client.items(
url,
params={
"per_page": "1",
"page": "2"
},
resource_name="pull",
page_limit=5,
)
]
self.assertEqual(2, len(pulls))
pulls = [
p async for p in client.items(
url,
params={
"base": other_branch,
"state": "all"
},
resource_name="pull",
page_limit=5,
)
]
self.assertEqual(1, len(pulls))
pulls = [
p async for p in client.items(url,
params={"base": "unknown"},
resource_name="pull",
page_limit=5)
]
self.assertEqual(0, len(pulls))
pull = await client.item(f"{url}/{p1['number']}")
self.assertEqual(p1["number"], pull["number"])
pull = await client.item(f"{url}/{p2['number']}")
self.assertEqual(p2["number"], pull["number"])
with self.assertRaises(http.HTTPStatusError) as ctxt:
await client.item(f"{url}/10000000000")
self.assertEqual(404, ctxt.exception.response.status_code)
async def asyncSetUp(self) -> None:
super(FunctionalTestBase, self).setUp()
# NOTE(sileht): don't preempted bucket consumption
# Otherwise preemption doesn't occur at the same moment during record
# and replay. Making some tests working during record and failing
# during replay.
config.BUCKET_PROCESSING_MAX_SECONDS = 100000
config.API_ENABLE = True
self.existing_labels: typing.List[str] = []
self.pr_counter: int = 0
self.git_counter: int = 0
mock.patch.object(branch_updater.gitter, "Gitter",
self.get_gitter).start()
mock.patch.object(duplicate_pull.gitter, "Gitter",
self.get_gitter).start()
# Web authentification always pass
mock.patch("hmac.compare_digest", return_value=True).start()
self.main_branch_name = self.get_full_branch_name("main")
self.git = self.get_gitter(LOG)
await self.git.init()
self.addAsyncCleanup(self.git.cleanup)
self.redis_links = redis_utils.RedisLinks(max_idle_time=0)
await self.clear_redis()
installation_json = await github.get_installation_from_account_id(
config.TESTING_ORGANIZATION_ID)
self.client_integration = github.aget_client(installation_json)
self.client_admin = github.AsyncGithubInstallationClient(
auth=github.GithubTokenAuth(token=config.ORG_ADMIN_PERSONAL_TOKEN,
))
self.client_fork = github.AsyncGithubInstallationClient(
auth=github.GithubTokenAuth(token=self.FORK_PERSONAL_TOKEN, ))
self.addAsyncCleanup(self.client_integration.aclose)
self.addAsyncCleanup(self.client_admin.aclose)
self.addAsyncCleanup(self.client_fork.aclose)
await self.client_admin.item("/user")
await self.client_fork.item("/user")
self.url_origin = (
f"/repos/mergifyio-testing/{self.RECORD_CONFIG['repository_name']}"
)
self.url_fork = f"/repos/mergify-test2/{self.RECORD_CONFIG['repository_name']}"
self.git_origin = f"{config.GITHUB_URL}/mergifyio-testing/{self.RECORD_CONFIG['repository_name']}"
self.git_fork = (
f"{config.GITHUB_URL}/mergify-test2/{self.RECORD_CONFIG['repository_name']}"
)
self.installation_ctxt = context.Installation(
installation_json,
self.subscription,
self.client_integration,
self.redis_links,
)
self.repository_ctxt = await self.installation_ctxt.get_repository_by_id(
github_types.GitHubRepositoryIdType(
self.RECORD_CONFIG["repository_id"]))
# NOTE(sileht): We mock this method because when we replay test, the
# timing maybe not the same as when we record it, making the formatted
# elapsed time different in the merge queue summary.
def fake_pretty_datetime(dt: datetime.datetime) -> str:
return "<fake_pretty_datetime()>"
mock.patch(
"mergify_engine.date.pretty_datetime",
side_effect=fake_pretty_datetime,
).start()
self._event_reader = EventReader(self.app,
self.RECORD_CONFIG["repository_id"])
await self._event_reader.drain()
# Track when worker work
real_consume_method = worker.StreamProcessor.consume
self.worker_concurrency_works = 0
async def tracked_consume(
inner_self: worker.StreamProcessor,
bucket_org_key: worker_lua.BucketOrgKeyType,
owner_id: github_types.GitHubAccountIdType,
owner_login_for_tracing: github_types.GitHubLoginForTracing,
) -> None:
self.worker_concurrency_works += 1
try:
await real_consume_method(inner_self, bucket_org_key, owner_id,
owner_login_for_tracing)
finally:
self.worker_concurrency_works -= 1
worker.StreamProcessor.consume = tracked_consume # type: ignore[assignment]
def cleanup_consume() -> None:
worker.StreamProcessor.consume = real_consume_method # type: ignore[assignment]
self.addCleanup(cleanup_consume)
async def report(
url: str,
) -> typing.Union[context.Context, github.AsyncGithubInstallationClient, None]:
redis_links = redis_utils.RedisLinks(max_idle_time=0)
try:
owner_login, repo, pull_number = _url_parser(url)
except ValueError:
print(f"{url} is not valid")
return None
try:
installation_json = await github.get_installation_from_login(
owner_login)
client = github.aget_client(installation_json)
except exceptions.MergifyNotInstalled:
print(f"* Mergify is not installed on account {owner_login}")
return None
# Do a dumb request just to authenticate
await client.get("/")
print(f"* INSTALLATION ID: {installation_json['id']}")
if repo is None:
slug = None
else:
slug = owner_login + "/" + repo
owner_id = installation_json["account"]["id"]
cached_sub = await subscription.Subscription.get_subscription(
redis_links.cache, owner_id)
db_sub = await subscription.Subscription._retrieve_subscription_from_db(
redis_links.cache, owner_id)
cached_tokens = await user_tokens.UserTokens.get(redis_links.cache,
owner_id)
if config.SAAS_MODE:
db_tokens = typing.cast(
user_tokens.UserTokens,
(await user_tokens.UserTokensSaas._retrieve_from_db(
redis_links.cache, owner_id)),
)
else:
db_tokens = cached_tokens
print("* Features (db):")
for v in sorted(f.value for f in db_sub.features):
print(f" - {v}")
print("* Features (cache):")
for v in sorted(f.value for f in cached_sub.features):
print(f" - {v}")
installation = context.Installation(installation_json, cached_sub, client,
redis_links)
await report_dashboard_synchro(installation.installation["id"], cached_sub,
cached_tokens, "ENGINE-CACHE", slug)
await report_dashboard_synchro(installation.installation["id"], db_sub,
db_tokens, "DASHBOARD", slug)
await report_worker_status(owner_login)
if repo is not None:
repository = await installation.get_repository_by_name(repo)
print(
f"* REPOSITORY IS {'PRIVATE' if repository.repo['private'] else 'PUBLIC'}"
)
print(f"* DEFAULT BRANCH: {repository.repo['default_branch']}")
print("* CONFIGURATION:")
mergify_config = None
config_file = await repository.get_mergify_config_file()
if not config_file:
print(".mergify.yml is missing")
else:
print(f"Config filename: {config_file['path']}")
print(config_file["decoded_content"])
try:
mergify_config = await repository.get_mergify_config()
except rules.InvalidRules as e: # pragma: no cover
print(f"configuration is invalid {str(e)}")
if pull_number is None:
async for branch in typing.cast(
typing.AsyncGenerator[github_types.GitHubBranch, None],
client.items(
f"/repos/{owner_login}/{repo}/branches",
resource_name="branches",
page_limit=100,
),
):
q = merge_train.Train(repository, branch["name"])
await q.load()
await report_queue("TRAIN", q)
else:
repository = await installation.get_repository_by_name(
github_types.GitHubRepositoryName(repo))
try:
ctxt = await repository.get_pull_request_context(
github_types.GitHubPullRequestNumber(int(pull_number)))
except http.HTTPNotFound:
print(f"Pull request `{url}` does not exist")
return client
# FIXME queues could also be printed if no pull number given
# TODO(sileht): display train if any
q = await merge_train.Train.from_context(ctxt)
print(
f"* TRAIN: {', '.join([f'#{p}' for p in await q.get_pulls()])}"
)
print("* PULL REQUEST:")
pr_data = await ctxt.pull_request.items()
pprint.pprint(pr_data, width=160)
is_behind = await ctxt.is_behind
print(f"is_behind: {is_behind}")
print(f"mergeable_state: {ctxt.pull['mergeable_state']}")
print("* MERGIFY LAST CHECKS:")
for c in await ctxt.pull_engine_check_runs:
print(
f"[{c['name']}]: {c['conclusion']} | {c['output'].get('title')} | {c['html_url']}"
)
print("> " + "\n> ".join(
("No Summary", ) if c["output"]["summary"] is None else
c["output"]["summary"].split("\n")))
if mergify_config is not None:
print("* MERGIFY LIVE MATCHES:")
pull_request_rules = mergify_config["pull_request_rules"]
match = await pull_request_rules.get_pull_request_rule(ctxt)
summary_title, summary = await actions_runner.gen_summary(
ctxt, pull_request_rules, match)
print(f"[Summary]: success | {summary_title}")
print("> " + "\n> ".join(summary.strip().split("\n")))
return ctxt
return client
async def asyncSetUp(self):
super(FunctionalTestBase, self).setUp()
self.existing_labels: typing.List[str] = []
self.protected_branches: typing.Set[str] = set()
self.pr_counter: int = 0
self.git_counter: int = 0
self.cassette_library_dir = os.path.join(CASSETTE_LIBRARY_DIR_BASE,
self.__class__.__name__,
self._testMethodName)
# Recording stuffs
if RECORD:
if os.path.exists(self.cassette_library_dir):
shutil.rmtree(self.cassette_library_dir)
os.makedirs(self.cassette_library_dir)
self.recorder = vcr.VCR(
cassette_library_dir=self.cassette_library_dir,
record_mode="all" if RECORD else "none",
match_on=["method", "uri"],
ignore_localhost=True,
filter_headers=[
("Authorization", "<TOKEN>"),
("X-Hub-Signature", "<SIGNATURE>"),
("User-Agent", None),
("Accept-Encoding", None),
("Connection", None),
],
before_record_response=self.response_filter,
)
if RECORD:
github.CachedToken.STORAGE = {}
else:
# Never expire token during replay
mock.patch.object(github_app,
"get_or_create_jwt",
return_value="<TOKEN>").start()
mock.patch.object(
github.GithubAppInstallationAuth,
"get_access_token",
return_value="<TOKEN>",
).start()
# NOTE(sileht): httpx pyvcr stubs does not replay auth_flow as it directly patch client.send()
# So anything occurring during auth_flow have to be mocked during replay
def get_auth(owner_name=None, owner_id=None, auth=None):
if auth is None:
auth = github.get_auth(owner_name, owner_id)
auth.installation = {
"id": config.INSTALLATION_ID,
}
auth.permissions_need_to_be_updated = False
auth.owner_id = config.TESTING_ORGANIZATION_ID
auth.owner = config.TESTING_ORGANIZATION
return auth
def github_aclient(owner_name=None, owner_id=None, auth=None):
return github.AsyncGithubInstallationClient(
get_auth(owner_name, owner_id, auth))
mock.patch.object(github, "aget_client", github_aclient).start()
mock.patch.object(branch_updater.gitter, "Gitter",
self.get_gitter).start()
mock.patch.object(duplicate_pull.gitter, "Gitter",
self.get_gitter).start()
if not RECORD:
# NOTE(sileht): Don't wait exponentialy during replay
mock.patch.object(context.Context._ensure_complete.retry, "wait",
None).start()
# Web authentification always pass
mock.patch("hmac.compare_digest", return_value=True).start()
branch_prefix_path = os.path.join(self.cassette_library_dir,
"branch_prefix")
if RECORD:
self.BRANCH_PREFIX = datetime.datetime.utcnow().strftime(
"%Y%m%d%H%M%S")
with open(branch_prefix_path, "w") as f:
f.write(self.BRANCH_PREFIX)
else:
with open(branch_prefix_path, "r") as f:
self.BRANCH_PREFIX = f.read()
self.master_branch_name = self.get_full_branch_name("master")
self.git = self.get_gitter(LOG)
await self.git.init()
self.addAsyncCleanup(self.git.cleanup)
await root.startup()
self.app = httpx.AsyncClient(app=root.app, base_url="http://localhost")
await self.clear_redis_cache()
self.redis_cache = utils.create_aredis_for_cache(max_idle_time=0)
self.subscription = subscription.Subscription(
self.redis_cache,
config.TESTING_ORGANIZATION_ID,
self.SUBSCRIPTION_ACTIVE,
"You're not nice",
frozenset(
getattr(subscription.Features, f)
for f in subscription.Features.__members__)
if self.SUBSCRIPTION_ACTIVE else frozenset(),
)
await self.subscription._save_subscription_to_cache()
self.user_tokens = user_tokens.UserTokens(
self.redis_cache,
config.TESTING_ORGANIZATION_ID,
{
"mergify-test1": config.ORG_ADMIN_GITHUB_APP_OAUTH_TOKEN,
"mergify-test3": config.ORG_USER_PERSONAL_TOKEN,
},
)
await self.user_tokens.save_to_cache()
# Let's start recording
cassette = self.recorder.use_cassette("http.json")
cassette.__enter__()
self.addCleanup(cassette.__exit__)
self.client_integration = github.aget_client(
config.TESTING_ORGANIZATION, config.TESTING_ORGANIZATION_ID)
self.client_admin = github.AsyncGithubInstallationClient(
auth=github.GithubTokenAuth(token=config.ORG_ADMIN_PERSONAL_TOKEN))
self.client_fork = github.AsyncGithubInstallationClient(
auth=github.GithubTokenAuth(token=self.FORK_PERSONAL_TOKEN))
self.addAsyncCleanup(self.client_integration.aclose)
self.addAsyncCleanup(self.client_admin.aclose)
self.addAsyncCleanup(self.client_fork.aclose)
await self.client_admin.item("/user")
await self.client_fork.item("/user")
if RECORD:
assert self.client_admin.auth.owner == "mergify-test1"
assert self.client_fork.auth.owner == "mergify-test2"
else:
self.client_admin.auth.owner = "mergify-test1"
self.client_fork.auth.owner = "mergify-test2"
self.url_main = f"/repos/mergifyio-testing/{self.REPO_NAME}"
self.url_fork = f"/repos/{self.client_fork.auth.owner}/{self.REPO_NAME}"
self.git_main = f"{config.GITHUB_URL}/mergifyio-testing/{self.REPO_NAME}"
self.git_fork = (
f"{config.GITHUB_URL}/{self.client_fork.auth.owner}/{self.REPO_NAME}"
)
self.installation_ctxt = context.Installation(
config.TESTING_ORGANIZATION_ID,
config.TESTING_ORGANIZATION,
self.subscription,
self.client_integration,
self.redis_cache,
)
self.repository_ctxt = context.Repository(self.installation_ctxt,
self.REPO_NAME, self.REPO_ID)
real_get_subscription = subscription.Subscription.get_subscription
async def fake_retrieve_subscription_from_db(redis_cache, owner_id):
if owner_id == config.TESTING_ORGANIZATION_ID:
return self.subscription
return subscription.Subscription(
redis_cache,
owner_id,
False,
"We're just testing",
set(),
)
async def fake_subscription(redis_cache, owner_id):
if owner_id == config.TESTING_ORGANIZATION_ID:
return await real_get_subscription(redis_cache, owner_id)
return subscription.Subscription(
redis_cache,
owner_id,
False,
"We're just testing",
set(),
)
mock.patch(
"mergify_engine.subscription.Subscription._retrieve_subscription_from_db",
side_effect=fake_retrieve_subscription_from_db,
).start()
mock.patch(
"mergify_engine.subscription.Subscription.get_subscription",
side_effect=fake_subscription,
).start()
async def fake_retrieve_user_tokens_from_db(redis_cache, owner_id):
if owner_id == config.TESTING_ORGANIZATION_ID:
return self.user_tokens
return user_tokens.UserTokens(redis_cache, owner_id, {})
real_get_user_tokens = user_tokens.UserTokens.get
async def fake_user_tokens(redis_cache, owner_id):
if owner_id == config.TESTING_ORGANIZATION_ID:
return await real_get_user_tokens(redis_cache, owner_id)
return user_tokens.UserTokens(redis_cache, owner_id, {})
mock.patch(
"mergify_engine.user_tokens.UserTokens._retrieve_from_db",
side_effect=fake_retrieve_user_tokens_from_db,
).start()
mock.patch(
"mergify_engine.user_tokens.UserTokens.get",
side_effect=fake_user_tokens,
).start()
self._event_reader = EventReader(self.app)
await self._event_reader.drain()
# NOTE(sileht): Prepare a fresh redis
await self.clear_redis_stream()
async def consume(self, stream_name: StreamNameType) -> None:
owner_login, owner_id = self._extract_owner(stream_name)
LOG.debug("consoming stream", gh_owner=owner_login)
try:
async with self._translate_exception_to_retries(stream_name):
sub = await subscription.Subscription.get_subscription(
self.redis_cache, owner_id
)
async with github.aget_client(owner_login) as client:
installation = context.Installation(
owner_id, owner_login, sub, client, self.redis_cache
)
async with self._translate_exception_to_retries(stream_name):
pulls = await self._extract_pulls_from_stream(installation)
if pulls:
client.set_requests_ratio(len(pulls))
await self._consume_pulls(installation, pulls)
await self._refresh_merge_trains(installation)
except StreamUnused:
LOG.info("unused stream, dropping it", gh_owner=owner_login, exc_info=True)
try:
await self.redis_stream.delete(stream_name)
except aredis.exceptions.ConnectionError:
LOG.warning(
"fail to drop stream, it will be retried", stream_name=stream_name
)
except StreamRetry as e:
log_method = (
LOG.error
if e.attempts >= STREAM_ATTEMPTS_LOGGING_THRESHOLD
else LOG.info
)
log_method(
"failed to process stream, retrying",
attempts=e.attempts,
retry_at=e.retry_at,
gh_owner=owner_login,
exc_info=True,
)
return
except vcr_errors_CannotOverwriteExistingCassetteException:
messages = await self.redis_stream.xrange(
stream_name, count=config.STREAM_MAX_BATCH
)
for message_id, message in messages:
LOG.info(msgpack.unpackb(message[b"event"], raw=False))
await self.redis_stream.execute_command("XDEL", stream_name, message_id)
except Exception:
# Ignore it, it will retried later
LOG.error("failed to process stream", gh_owner=owner_login, exc_info=True)
LOG.debug("cleanup stream start", stream_name=stream_name)
try:
await self.redis_stream.eval(
self.ATOMIC_CLEAN_STREAM_SCRIPT, 1, stream_name.encode(), time.time()
)
except aredis.exceptions.ConnectionError:
LOG.warning(
"fail to cleanup stream, it maybe partially replayed",
stream_name=stream_name,
)
LOG.debug("cleanup stream end", stream_name=stream_name)
async def report(
url: str,
) -> typing.Union[context.Context, github.AsyncGithubInstallationClient, None]:
redis_cache = utils.create_aredis_for_cache(max_idle_time=0)
try:
owner, repo, pull_number = _url_parser(url)
except ValueError:
print(f"{url} is not valid")
return None
try:
client = github.aget_client(owner)
except exceptions.MergifyNotInstalled:
print(f"* Mergify is not installed on account {owner}")
return None
# Do a dumb request just to authenticate
await client.get("/")
if client.auth.installation is None:
print("No installation detected")
return None
print(f"* INSTALLATION ID: {client.auth.installation['id']}")
if client.auth.owner_id is None:
raise RuntimeError("Unable to get owner_id")
if repo is None:
slug = None
else:
slug = owner + "/" + repo
cached_sub = await subscription.Subscription.get_subscription(
redis_cache, client.auth.owner_id)
db_sub = await subscription.Subscription._retrieve_subscription_from_db(
redis_cache, client.auth.owner_id)
cached_tokens = await user_tokens.UserTokens.get(redis_cache,
client.auth.owner_id)
db_tokens = await user_tokens.UserTokens._retrieve_from_db(
redis_cache, client.auth.owner_id)
print(f"* SUBSCRIBED (cache/db): {cached_sub.active} / {db_sub.active}")
print("* Features (cache):")
for f in db_sub.features:
print(f" - {f.value}")
print("* Features (db):")
for f in cached_sub.features:
print(f" - {f.value}")
await report_dashboard_synchro(client.auth.installation["id"], cached_sub,
cached_tokens, "ENGINE-CACHE", slug)
await report_dashboard_synchro(client.auth.installation["id"], db_sub,
db_tokens, "DASHBOARD", slug)
await report_worker_status(owner)
installation = context.Installation(client.auth.owner_id, owner,
cached_sub, client, redis_cache)
if repo is not None:
repo_info: github_types.GitHubRepository = await client.item(
f"/repos/{owner}/{repo}")
repository = context.Repository(installation, repo_info["name"],
repo_info["id"])
print(
f"* REPOSITORY IS {'PRIVATE' if repo_info['private'] else 'PUBLIC'}"
)
print("* CONFIGURATION:")
mergify_config = None
config_file = await repository.get_mergify_config_file()
if config_file is None:
print(".mergify.yml is missing")
else:
print(f"Config filename: {config_file['path']}")
print(config_file["decoded_content"].decode())
try:
mergify_config = rules.get_mergify_config(config_file)
except rules.InvalidRules as e: # pragma: no cover
print(f"configuration is invalid {str(e)}")
else:
mergify_config["pull_request_rules"].rules.extend(
engine.MERGIFY_BUILTIN_CONFIG["pull_request_rules"].rules)
if pull_number is None:
async for branch in typing.cast(
typing.AsyncGenerator[github_types.GitHubBranch, None],
client.items(f"/repos/{owner}/{repo}/branches"),
):
# TODO(sileht): Add some informations on the train
q: queue.QueueBase = naive.Queue(repository, branch["name"])
await report_queue("QUEUES", q)
q = merge_train.Train(repository, branch["name"])
await q.load()
await report_queue("TRAIN", q)
else:
repository = context.Repository(
installation, github_types.GitHubRepositoryName(repo))
ctxt = await repository.get_pull_request_context(
github_types.GitHubPullRequestNumber(int(pull_number)))
# FIXME queues could also be printed if no pull number given
# TODO(sileht): display train if any
q = await naive.Queue.from_context(ctxt)
print(
f"* QUEUES: {', '.join([f'#{p}' for p in await q.get_pulls()])}"
)
q = await merge_train.Train.from_context(ctxt)
print(
f"* TRAIN: {', '.join([f'#{p}' for p in await q.get_pulls()])}"
)
print("* PULL REQUEST:")
pr_data = await ctxt.pull_request.items()
pprint.pprint(pr_data, width=160)
is_behind = await ctxt.is_behind
print(f"is_behind: {is_behind}")
print(f"mergeable_state: {ctxt.pull['mergeable_state']}")
print("* MERGIFY LAST CHECKS:")
for c in await ctxt.pull_engine_check_runs:
print(
f"[{c['name']}]: {c['conclusion']} | {c['output'].get('title')}"
)
print("> " + "\n> ".join(
("No Summary", ) if c["output"]["summary"] is None else
c["output"]["summary"].split("\n")))
if mergify_config is not None:
print("* MERGIFY LIVE MATCHES:")
match = await mergify_config["pull_request_rules"
].get_pull_request_rule(ctxt)
summary_title, summary = await actions_runner.gen_summary(
ctxt, match)
print(f"[Summary]: success | {summary_title}")
print("> " + "\n> ".join(summary.strip().split("\n")))
return ctxt
return client