async def test_get_mergify_config_location_from_cache(
redis_cache: utils.RedisCache, ) -> None:
client = mock.AsyncMock()
client.auth.owner = "foo"
client.item.side_effect = [
http.HTTPNotFound("Not Found",
request=mock.Mock(),
response=mock.Mock()),
http.HTTPNotFound("Not Found",
request=mock.Mock(),
response=mock.Mock()),
github_types.GitHubContentFile({
"content":
encodebytes("whatever".encode()).decode(),
"type":
"file",
"path":
".github/mergify.yml",
"sha":
github_types.SHAType("zeazeaze"),
}),
]
installation = context.Installation(
github_types.GitHubAccountIdType(0),
github_types.GitHubLogin("foo"),
subscription.Subscription(redis_cache, 0, False, "", frozenset()),
client,
redis_cache,
)
repository = context.Repository(installation,
github_types.GitHubRepositoryName("bar"))
await repository.get_mergify_config_file()
assert client.item.call_count == 3
client.item.assert_has_calls([
mock.call("/repos/foo/bar/contents/.mergify.yml"),
mock.call("/repos/foo/bar/contents/.mergify/config.yml"),
mock.call("/repos/foo/bar/contents/.github/mergify.yml"),
])
client.item.reset_mock()
client.item.side_effect = [
github_types.GitHubContentFile({
"content":
encodebytes("whatever".encode()).decode(),
"type":
"file",
"path":
".github/mergify.yml",
"sha":
github_types.SHAType("zeazeaze"),
}),
]
repository._cache = context.RepositoryCache()
await repository.get_mergify_config_file()
assert client.item.call_count == 1
client.item.assert_has_calls([
mock.call("/repos/foo/bar/contents/.github/mergify.yml"),
])
async def test_get_mergify_config_invalid(
invalid: str, redis_cache: utils.RedisCache) -> None:
with pytest.raises(InvalidRules):
async def item(*args, **kwargs):
return github_types.GitHubContentFile({
"content":
encodebytes(invalid.encode()).decode(),
"path":
".mergify.yml",
"type":
"file",
"sha":
"azertyu",
})
client = mock.Mock()
client.item.return_value = item()
installation = context.Installation(
github_types.GitHubAccountIdType(0),
github_types.GitHubLogin("foobar"),
subscription.Subscription(redis_cache, 0, False, "", frozenset()),
client,
redis_cache,
)
repository = context.Repository(
installation,
github_types.GitHubRepositoryName("xyz"),
github_types.GitHubRepositoryIdType(0),
)
config_file = await repository.get_mergify_config_file()
assert config_file is not None
get_mergify_config(config_file)
async def test_disabled(redis_cache):
action = request_reviews.RequestReviewsAction.get_schema()(
{
"random_count": 2,
"teams": {
"foobar": 2,
"foobaz": 1,
},
"users": {
"jd": 2,
"sileht": 1,
},
},
)
client = mock.MagicMock()
client.auth.installation.__getitem__.return_value = 123
sub = subscription.Subscription(
redis_cache,
123,
False,
"No sub",
{},
frozenset({}),
)
installation = context.Installation(123, "Mergifyio", sub, client, redis_cache)
repository = context.Repository(installation, "demo")
ctxt = await context.Context.create(
repository,
{
"number": 123,
"state": None,
"mergeable_state": "ok",
"merged_by": None,
"merged": None,
"merged_at": None,
"base": {
"sha": "sha",
"ref": "main",
"user": {
"login": {
"Mergifyio",
},
},
"repo": {
"name": "demo",
"private": False,
"owner": {
"login": "******",
},
},
},
},
)
result = await action.run(ctxt, None)
assert result.conclusion == check_api.Conclusion.ACTION_REQUIRED
assert result.title == "Random request reviews are disabled"
assert result.summary == (
"⚠ The [subscription](https://dashboard.mergify.io/github/Mergifyio/subscription) "
"needs to be updated to enable this feature."
)
async def _simulator(redis_cache, pull_request_rules, owner, repo, pull_number, token):
try:
if token:
auth = github.GithubTokenAuth(token)
else:
auth = github.get_auth(owner)
async with github.aget_client(auth=auth) as client:
try:
data = await client.item(f"/repos/{owner}/{repo}/pulls/{pull_number}")
except http.HTTPNotFound:
raise PullRequestUrlInvalid(
message=f"Pull request {owner}/{repo}/pulls/{pull_number} not found"
)
sub = await subscription.Subscription.get_subscription(
redis_cache, data["base"]["user"]["id"]
)
installation = context.Installation(
data["base"]["user"]["id"],
owner,
sub,
client,
redis_cache,
)
repository = context.Repository(installation, data["base"]["repo"])
ctxt = await repository.get_pull_request_context(data["number"], data)
ctxt.sources = [{"event_type": "mergify-simulator", "data": []}]
match = await pull_request_rules.get_pull_request_rule(ctxt)
return await actions_runner.gen_summary(ctxt, pull_request_rules, match)
except exceptions.MergifyNotInstalled:
raise PullRequestUrlInvalid(
message=f"Mergify not installed on repository '{owner}/{repo}'"
)
async def prepare_context(client, redis_cache, subscribed=True):
sub = subscription.Subscription(
redis_cache,
123,
subscribed,
"sub or not to sub",
frozenset(
getattr(subscription.Features, f)
for f in subscription.Features.__members__)
if subscribed else frozenset(),
)
installation = context.Installation(123, "Mergifyio", sub, client,
redis_cache)
repository = context.Repository(installation, "demo", 123)
return await context.Context.create(
repository,
{
"id": 12345,
"number": 123,
"state": None,
"mergeable_state": "ok",
"merged_by": None,
"merged": None,
"merged_at": None,
"user": {
"login": "******"
},
"requested_reviewers": [{
"login": "******"
}, {
"login": "******"
}],
"requested_teams": [{
"slug": "foobar"
}, {
"slug": "foobaz"
}],
"base": {
"sha": "sha",
"ref": "main",
"user": {
"login": {
"Mergifyio",
},
},
"repo": {
"full_name": "Mergifyio/demo",
"name": "demo",
"private": False,
"owner": {
"login": "******",
},
},
},
},
)
async def test_merge_commit_message_syntax_error(body, error, redis_cache):
pull = PR.copy()
pull["body"] = body
client = mock.MagicMock()
installation = context.Installation(123, "whatever", {}, client, redis_cache)
repository = context.Repository(installation, "whatever")
pr = await context.Context.create(repository=repository, pull=pull)
with pytest.raises(context.RenderTemplateFailure) as rmf:
await merge.MergeAction._get_commit_message(pr.pull_request)
assert str(rmf) == error
async def test_merge_commit_message_undefined(body):
pull = PR.copy()
pull["body"] = body
client = mock.MagicMock()
installation = context.Installation(123, "whatever", {}, client, None)
repository = context.Repository(installation, "whatever", 123)
pr = await context.Context.create(repository=repository, pull=pull)
with pytest.raises(context.RenderTemplateFailure) as x:
await merge.MergeAction._get_commit_message(pr.pull_request)
assert str(x) == "foobar"
async def _create_context(redis_cache, client):
sub = subscription.Subscription(
redis_cache,
123,
True,
"",
{},
frozenset({}),
)
installation = context.Installation(123, "Mergifyio", sub, client,
redis_cache)
repository = context.Repository(installation, "demo", 123)
return await context.Context.create(
repository,
{
"number": 789,
"state": "open",
"title": "Amazing new feature",
"user": {
"login": "******",
"id": 1,
},
"mergeable_state": "ok",
"merged_by": None,
"merged": None,
"merged_at": None,
"base": {
"sha": "sha",
"ref": "main",
"user": {
"login": {
"Mergifyio",
},
},
"repo": {
"name": "demo",
"private": False,
"owner": {
"login": "******",
"id": 123,
},
"permissions": {
"admin": False,
"push": False,
"pull": True,
},
},
},
},
[],
)
def repository(redis_cache, fake_client):
installation = context.Installation(
github_types.GitHubAccountIdType(123),
github_types.GitHubLogin("user"),
subscription.Subscription(redis_cache, 0, False, "", frozenset()),
fake_client,
redis_cache,
)
return context.Repository(
installation,
github_types.GitHubRepositoryName("name"),
github_types.GitHubRepositoryIdType(123),
)
def fake_repository(
redis_links: redis_utils.RedisLinks,
fake_subscription: subscription.Subscription,
) -> context.Repository:
gh_owner = github_types.GitHubAccount({
"login":
github_types.GitHubLogin("Mergifyio"),
"id":
github_types.GitHubAccountIdType(0),
"type":
"User",
"avatar_url":
"",
})
gh_repo = github_types.GitHubRepository({
"full_name":
"Mergifyio/mergify-engine",
"name":
github_types.GitHubRepositoryName("mergify-engine"),
"private":
False,
"id":
github_types.GitHubRepositoryIdType(0),
"owner":
gh_owner,
"archived":
False,
"url":
"",
"html_url":
"",
"default_branch":
github_types.GitHubRefType("main"),
})
installation_json = github_types.GitHubInstallation({
"id":
github_types.GitHubInstallationIdType(12345),
"target_type":
gh_owner["type"],
"permissions": {},
"account":
gh_owner,
})
fake_client = mock.Mock()
installation = context.Installation(installation_json, fake_subscription,
fake_client, redis_links)
return context.Repository(installation, gh_repo)
async def test_stream_processor_retrying_after_read_error(
run_engine, _, redis_stream, redis_cache):
response = mock.Mock()
response.json.return_value = {"message": "boom"}
response.status_code = 503
run_engine.side_effect = httpx.ReadError(
"Server disconnected while attempting read",
request=mock.Mock(),
)
p = worker.StreamProcessor(redis_stream, redis_cache)
installation = context.Installation(123, "owner", {}, None, None)
with pytest.raises(worker.StreamRetry):
async with p._translate_exception_to_retries(installation.stream_name):
await worker.run_engine(installation, "repo", 1234, [])
async def _do_test_event_to_pull_check_run(redis_cache, filename, expected_pulls):
owner = "CytopiaTeam"
repo = "Cytopia"
event_type = "check_run"
with open(
os.path.join(os.path.dirname(__file__), "events", filename),
"rb",
) as f:
data = json.load(f)
installation = context.Installation(123, owner, {}, mock.Mock(), redis_cache)
pulls = await github_events.extract_pull_numbers_from_event(
installation, repo, event_type, data, []
)
assert pulls == expected_pulls
async def test_pull_behind(commits_tree_generator, redis_cache):
expected, commits = commits_tree_generator
async def get_commits(*args, **kwargs):
for c in commits:
yield c
async def item(*args, **kwargs):
return {"commit": {"sha": "base"}}
client = mock.Mock()
client.items.return_value = get_commits() # /pulls/X/commits
client.item.return_value = item() # /branch/#foo
installation = context.Installation(123, "user", {}, client, redis_cache)
repository = context.Repository(installation, "name", 0)
ctxt = await context.Context.create(
repository,
{
"number": 1,
"mergeable_state": "clean",
"state": "open",
"merged": False,
"merged_at": None,
"merged_by": None,
"base": {
"ref": "#foo",
"repo": {
"name": "foobar",
"private": False
},
"sha": "miaou",
"user": {
"login": "******"
},
},
},
{},
)
assert expected == await ctxt.is_behind
def fake_repository(
redis_cache: utils.RedisCache,
fake_subscription: subscription.Subscription,
) -> context.Repository:
gh_owner = github_types.GitHubAccount(
{
"login": github_types.GitHubLogin("Mergifyio"),
"id": github_types.GitHubAccountIdType(0),
"type": "User",
"avatar_url": "",
}
)
gh_repo = github_types.GitHubRepository(
{
"full_name": "Mergifyio/mergify-engine",
"name": github_types.GitHubRepositoryName("mergify-engine"),
"private": False,
"id": github_types.GitHubRepositoryIdType(0),
"owner": gh_owner,
"archived": False,
"url": "",
"html_url": "",
"default_branch": github_types.GitHubRefType("main"),
}
)
installation_json = github_types.GitHubInstallation(
{
"id": github_types.GitHubInstallationIdType(12345),
"target_type": gh_owner["type"],
"permissions": {},
"account": gh_owner,
}
)
fake_client = redis_queue = mock.Mock()
# NOTE(Syffe): Since redis_queue is not used in fake_repository, we simply mock it,
# otherwise a fixture is needed for it. This might change with future use of redis_queue.
installation = context.Installation(
installation_json, fake_subscription, fake_client, redis_cache, redis_queue
)
return context.Repository(installation, gh_repo)
async def _do_test_event_to_pull_check_run(
redis_cache: utils.RedisCache, filename: str,
expected_pulls: typing.List[int]) -> None:
with open(
os.path.join(os.path.dirname(__file__), "events", filename),
"r",
) as f:
data = json.loads(f.read().replace("https://github.com",
config.GITHUB_URL).replace(
"https://api.github.com",
config.GITHUB_REST_API_URL))
gh_owner = github_types.GitHubAccount({
"type":
"User",
"id":
github_types.GitHubAccountIdType(12345),
"login":
github_types.GitHubLogin("CytopiaTeam"),
"avatar_url":
"",
})
installation_json = github_types.GitHubInstallation({
"id":
github_types.GitHubInstallationIdType(12345),
"target_type":
gh_owner["type"],
"permissions": {},
"account":
gh_owner,
})
installation = context.Installation(installation_json, mock.Mock(),
mock.Mock(), redis_cache, mock.Mock())
pulls = await github_events.extract_pull_numbers_from_event(
installation,
"check_run",
data,
[],
)
assert pulls == expected_pulls
def repository(redis_cache, fake_client):
gh_owner = github_types.GitHubAccount({
"login":
github_types.GitHubLogin("user"),
"id":
github_types.GitHubAccountIdType(0),
"type":
"User",
"avatar_url":
"",
})
gh_repo = github_types.GitHubRepository({
"full_name":
"user/name",
"name":
github_types.GitHubRepositoryName("name"),
"private":
False,
"id":
github_types.GitHubRepositoryIdType(0),
"owner":
gh_owner,
"archived":
False,
"url":
"",
"html_url":
"",
"default_branch":
github_types.GitHubRefType("ref"),
})
installation = context.Installation(
github_types.GitHubAccountIdType(123),
github_types.GitHubLogin("user"),
subscription.Subscription(redis_cache, 0, False, "", frozenset()),
fake_client,
redis_cache,
)
return context.Repository(installation, gh_repo)
async def test_merge_commit_message(body, title, message, mode):
pull = PR.copy()
pull["body"] = body
client = mock.MagicMock()
installation = context.Installation(123, "whatever", {}, client, None)
repository = context.Repository(installation, "whatever", 123)
ctxt = await context.Context.create(repository=repository, pull=pull)
ctxt._cache["pull_statuses"] = [
github_types.GitHubStatus({
"target_url": "http://example.com",
"context": "my CI",
"state": "success",
"description": "foobar",
"avatar_url": "",
})
]
ctxt._cache["pull_check_runs"] = []
pr = ctxt.pull_request
assert await merge.MergeAction._get_commit_message(pr, mode=mode) == (
title,
message,
)
async def get_repository_context(
owner: github_types.GitHubLogin = fastapi.Path( # noqa: B008
..., description="The owner of the repository"),
repository: github_types.GitHubRepositoryName = fastapi.Path( # noqa: B008
..., description="The name of the repository"),
redis_links: redis_utils.RedisLinks = fastapi.Depends( # noqa: B008
redis.get_redis_links),
installation_json: github_types.GitHubInstallation = fastapi.
Depends( # noqa: B008
get_installation),
) -> typing.AsyncGenerator[context.Repository, None]:
async with github.aget_client(installation_json) as client:
try:
# Check this token has access to this repository
repo = typing.cast(
github_types.GitHubRepository,
await client.item(f"/repos/{owner}/{repository}"),
)
except (http.HTTPNotFound, http.HTTPForbidden, http.HTTPUnauthorized):
raise fastapi.HTTPException(status_code=404)
sub = await subscription.Subscription.get_subscription(
redis_links.cache, installation_json["account"]["id"])
installation = context.Installation(installation_json, sub, client,
redis_links)
repository_ctxt = installation.get_repository_from_github_data(repo)
# NOTE(sileht): Since this method is used as fastapi Depends only, it's safe to set this
# for the ongoing http request
sentry_sdk.set_user(
{"username": repository_ctxt.installation.owner_login})
sentry_sdk.set_tag("gh_owner",
repository_ctxt.installation.owner_login)
sentry_sdk.set_tag("gh_repo", repository_ctxt.repo["name"])
yield repository_ctxt
async def report(
url: str,
) -> typing.Union[context.Context, github.AsyncGithubInstallationClient, None]:
redis_links = redis_utils.RedisLinks(max_idle_time=0)
try:
owner_login, repo, pull_number = _url_parser(url)
except ValueError:
print(f"{url} is not valid")
return None
try:
installation_json = await github.get_installation_from_login(
owner_login)
client = github.aget_client(installation_json)
except exceptions.MergifyNotInstalled:
print(f"* Mergify is not installed on account {owner_login}")
return None
# Do a dumb request just to authenticate
await client.get("/")
print(f"* INSTALLATION ID: {installation_json['id']}")
if repo is None:
slug = None
else:
slug = owner_login + "/" + repo
owner_id = installation_json["account"]["id"]
cached_sub = await subscription.Subscription.get_subscription(
redis_links.cache, owner_id)
db_sub = await subscription.Subscription._retrieve_subscription_from_db(
redis_links.cache, owner_id)
cached_tokens = await user_tokens.UserTokens.get(redis_links.cache,
owner_id)
if config.SAAS_MODE:
db_tokens = typing.cast(
user_tokens.UserTokens,
(await user_tokens.UserTokensSaas._retrieve_from_db(
redis_links.cache, owner_id)),
)
else:
db_tokens = cached_tokens
print("* Features (db):")
for v in sorted(f.value for f in db_sub.features):
print(f" - {v}")
print("* Features (cache):")
for v in sorted(f.value for f in cached_sub.features):
print(f" - {v}")
installation = context.Installation(installation_json, cached_sub, client,
redis_links)
await report_dashboard_synchro(installation.installation["id"], cached_sub,
cached_tokens, "ENGINE-CACHE", slug)
await report_dashboard_synchro(installation.installation["id"], db_sub,
db_tokens, "DASHBOARD", slug)
await report_worker_status(owner_login)
if repo is not None:
repository = await installation.get_repository_by_name(repo)
print(
f"* REPOSITORY IS {'PRIVATE' if repository.repo['private'] else 'PUBLIC'}"
)
print(f"* DEFAULT BRANCH: {repository.repo['default_branch']}")
print("* CONFIGURATION:")
mergify_config = None
config_file = await repository.get_mergify_config_file()
if not config_file:
print(".mergify.yml is missing")
else:
print(f"Config filename: {config_file['path']}")
print(config_file["decoded_content"])
try:
mergify_config = await repository.get_mergify_config()
except rules.InvalidRules as e: # pragma: no cover
print(f"configuration is invalid {str(e)}")
if pull_number is None:
async for branch in typing.cast(
typing.AsyncGenerator[github_types.GitHubBranch, None],
client.items(
f"/repos/{owner_login}/{repo}/branches",
resource_name="branches",
page_limit=100,
),
):
q = merge_train.Train(repository, branch["name"])
await q.load()
await report_queue("TRAIN", q)
else:
repository = await installation.get_repository_by_name(
github_types.GitHubRepositoryName(repo))
try:
ctxt = await repository.get_pull_request_context(
github_types.GitHubPullRequestNumber(int(pull_number)))
except http.HTTPNotFound:
print(f"Pull request `{url}` does not exist")
return client
# FIXME queues could also be printed if no pull number given
# TODO(sileht): display train if any
q = await merge_train.Train.from_context(ctxt)
print(
f"* TRAIN: {', '.join([f'#{p}' for p in await q.get_pulls()])}"
)
print("* PULL REQUEST:")
pr_data = await ctxt.pull_request.items()
pprint.pprint(pr_data, width=160)
is_behind = await ctxt.is_behind
print(f"is_behind: {is_behind}")
print(f"mergeable_state: {ctxt.pull['mergeable_state']}")
print("* MERGIFY LAST CHECKS:")
for c in await ctxt.pull_engine_check_runs:
print(
f"[{c['name']}]: {c['conclusion']} | {c['output'].get('title')} | {c['html_url']}"
)
print("> " + "\n> ".join(
("No Summary", ) if c["output"]["summary"] is None else
c["output"]["summary"].split("\n")))
if mergify_config is not None:
print("* MERGIFY LIVE MATCHES:")
pull_request_rules = mergify_config["pull_request_rules"]
match = await pull_request_rules.get_pull_request_rule(ctxt)
summary_title, summary = await actions_runner.gen_summary(
ctxt, pull_request_rules, match)
print(f"[Summary]: success | {summary_title}")
print("> " + "\n> ".join(summary.strip().split("\n")))
return ctxt
return client
async def test_get_commits_to_cherry_pick_rebase(
commits: mock.PropertyMock,
redis_cache: utils.RedisCache,
) -> None:
gh_owner = github_types.GitHubAccount({
"login":
github_types.GitHubLogin("user"),
"id":
github_types.GitHubAccountIdType(0),
"type":
"User",
"avatar_url":
"",
})
gh_repo = github_types.GitHubRepository({
"full_name":
"user/name",
"name":
github_types.GitHubRepositoryName("name"),
"private":
False,
"id":
github_types.GitHubRepositoryIdType(0),
"owner":
gh_owner,
"archived":
False,
"url":
"",
"html_url":
"",
"default_branch":
github_types.GitHubRefType("ref"),
})
c1 = github_types.GitHubBranchCommit({
"sha": github_types.SHAType("c1f"),
"parents": [],
"commit": {
"message": "foobar"
},
})
c2 = github_types.GitHubBranchCommit({
"sha": github_types.SHAType("c2"),
"parents": [c1],
"commit": {
"message": "foobar"
},
})
commits.return_value = [c1, c2]
client = mock.Mock()
client.auth.get_access_token.return_value = "<token>"
client.items.side_effect = fake_get_github_pulls_from_sha
installation = context.Installation(
github_types.GitHubAccountIdType(123),
github_types.GitHubLogin("user"),
subscription.Subscription(redis_cache, 0, False, "", frozenset()),
client,
redis_cache,
)
repository = context.Repository(installation, gh_repo)
ctxt = await context.Context.create(
repository,
{
"labels": [],
"draft": False,
"merge_commit_sha": github_types.SHAType(""),
"title": "",
"commits": 1,
"rebaseable": False,
"maintainer_can_modify": False,
"id": github_types.GitHubPullRequestId(0),
"number": github_types.GitHubPullRequestNumber(6),
"merged": True,
"state": "closed",
"html_url": "<html_url>",
"changed_files": 1,
"base": {
"label": "",
"sha": github_types.SHAType("sha"),
"user": {
"login": github_types.GitHubLogin("user"),
"id": github_types.GitHubAccountIdType(0),
"type": "User",
"avatar_url": "",
},
"ref": github_types.GitHubRefType("ref"),
"repo": {
"full_name": "user/ref",
"name": github_types.GitHubRepositoryName("name"),
"private": False,
"id": github_types.GitHubRepositoryIdType(0),
"owner": {
"login": github_types.GitHubLogin("user"),
"id": github_types.GitHubAccountIdType(0),
"type": "User",
"avatar_url": "",
},
"archived": False,
"url": "",
"html_url": "",
"default_branch": github_types.GitHubRefType(""),
},
},
"head": {
"label": "",
"sha": github_types.SHAType("sha"),
"ref": github_types.GitHubRefType("fork"),
"user": {
"login": github_types.GitHubLogin("user"),
"id": github_types.GitHubAccountIdType(0),
"type": "User",
"avatar_url": "",
},
"repo": {
"full_name": "fork/other",
"name": github_types.GitHubRepositoryName("other"),
"private": False,
"archived": False,
"url": "",
"html_url": "",
"default_branch": github_types.GitHubRefType(""),
"id": github_types.GitHubRepositoryIdType(0),
"owner": {
"login": github_types.GitHubLogin("user"),
"id": github_types.GitHubAccountIdType(0),
"type": "User",
"avatar_url": "",
},
},
},
"user": {
"login": github_types.GitHubLogin("user"),
"id": github_types.GitHubAccountIdType(0),
"type": "User",
"avatar_url": "",
},
"merged_by": None,
"merged_at": None,
"mergeable_state": "clean",
},
)
base_branch = github_types.GitHubBranchCommitParent(
{"sha": github_types.SHAType("base_branch")})
rebased_c1 = github_types.GitHubBranchCommit({
"sha":
github_types.SHAType("rebased_c1"),
"parents": [base_branch],
"commit": {
"message": "hello c1"
},
})
rebased_c2 = github_types.GitHubBranchCommit({
"sha":
github_types.SHAType("rebased_c2"),
"parents": [rebased_c1],
"commit": {
"message": "hello c2"
},
})
async def fake_get_github_commit_from_sha(url, api_version=None):
if url.endswith("/commits/rebased_c1"):
return rebased_c1
if url.endswith("/commits/rebased_c2"):
return rebased_c2
raise RuntimeError(f"Unknown URL {url}")
client.item.side_effect = fake_get_github_commit_from_sha
assert await duplicate_pull._get_commits_to_cherrypick(ctxt,
rebased_c2) == [
rebased_c1,
rebased_c2,
]
async def test_get_commits_to_cherry_pick_merge(
commits: mock.PropertyMock,
redis_cache: utils.RedisCache,
) -> None:
c1 = github_types.GitHubBranchCommit({
"sha": github_types.SHAType("c1f"),
"parents": [],
"commit": {
"message": "foobar"
},
})
c2 = github_types.GitHubBranchCommit({
"sha": github_types.SHAType("c2"),
"parents": [c1],
"commit": {
"message": "foobar"
},
})
async def fake_commits():
return [c1, c2]
commits.return_value = fake_commits()
client = mock.Mock()
client.auth.get_access_token.return_value = "<token>"
gh_owner = github_types.GitHubAccount({
"login":
github_types.GitHubLogin("user"),
"id":
github_types.GitHubAccountIdType(0),
"type":
"User",
"avatar_url":
"",
})
gh_repo = github_types.GitHubRepository({
"full_name":
"user/name",
"name":
github_types.GitHubRepositoryName("name"),
"private":
False,
"id":
github_types.GitHubRepositoryIdType(0),
"owner":
gh_owner,
"archived":
False,
"url":
"",
"html_url":
"",
"default_branch":
github_types.GitHubRefType("ref"),
})
installation = context.Installation(
github_types.GitHubAccountIdType(123),
github_types.GitHubLogin("user"),
subscription.Subscription(redis_cache, 0, False, "", frozenset()),
client,
redis_cache,
)
repository = context.Repository(installation, gh_repo)
ctxt = await context.Context.create(
repository,
{
"number": github_types.GitHubPullRequestNumber(6),
"commits": 1,
"merged": True,
"state": "closed",
"html_url": "<html_url>",
"id": github_types.GitHubPullRequestId(0),
"maintainer_can_modify": True,
"labels": [],
"rebaseable": True,
"draft": True,
"merge_commit_sha": None,
"title": "foobar",
"changed_files": 1,
"base": {
"label":
"user:ref",
"sha":
github_types.SHAType("sha"),
"ref":
github_types.GitHubRefType("ref"),
"user":
gh_owner,
"repo":
github_types.GitHubRepository(
{
"full_name": "user/ref",
"name": github_types.GitHubRepositoryName("name"),
"private": False,
"id": github_types.GitHubRepositoryIdType(0),
"owner": gh_owner,
"archived": False,
"url": "",
"html_url": "",
"default_branch": github_types.GitHubRefType("ref"),
}),
},
"head": {
"label":
"user:ref",
"sha":
github_types.SHAType("sha"),
"user":
gh_owner,
"ref":
github_types.GitHubRefType("fork"),
"repo":
github_types.GitHubRepository(
{
"full_name": "fork/other",
"name": github_types.GitHubRepositoryName("name"),
"private": False,
"id": github_types.GitHubRepositoryIdType(0),
"owner": gh_owner,
"archived": False,
"url": "",
"html_url": "",
"default_branch": github_types.GitHubRefType("ref"),
}),
},
"user": gh_owner,
"merged_at": None,
"merged_by": None,
"mergeable_state": "clean",
},
)
base_branch = github_types.GitHubBranchCommit({
"sha":
github_types.SHAType("base_branch"),
"parents": [],
"commit": {
"message": "foobar"
},
})
merge_commit = github_types.GitHubBranchCommit({
"sha":
github_types.SHAType("merge_commit"),
"parents": [base_branch, c2],
"commit": {
"message": "foobar"
},
})
assert await duplicate_pull._get_commits_to_cherrypick(ctxt,
merge_commit) == [
c1,
c2,
]
async def asyncSetUp(self):
super(FunctionalTestBase, self).setUp()
self.existing_labels: typing.List[str] = []
self.protected_branches: typing.Set[str] = set()
self.pr_counter: int = 0
self.git_counter: int = 0
self.cassette_library_dir = os.path.join(CASSETTE_LIBRARY_DIR_BASE,
self.__class__.__name__,
self._testMethodName)
# Recording stuffs
if RECORD:
if os.path.exists(self.cassette_library_dir):
shutil.rmtree(self.cassette_library_dir)
os.makedirs(self.cassette_library_dir)
self.recorder = vcr.VCR(
cassette_library_dir=self.cassette_library_dir,
record_mode="all" if RECORD else "none",
match_on=["method", "uri"],
ignore_localhost=True,
filter_headers=[
("Authorization", "<TOKEN>"),
("X-Hub-Signature", "<SIGNATURE>"),
("User-Agent", None),
("Accept-Encoding", None),
("Connection", None),
],
before_record_response=self.response_filter,
)
if RECORD:
github.CachedToken.STORAGE = {}
else:
# Never expire token during replay
mock.patch.object(github_app,
"get_or_create_jwt",
return_value="<TOKEN>").start()
mock.patch.object(
github.GithubAppInstallationAuth,
"get_access_token",
return_value="<TOKEN>",
).start()
# NOTE(sileht): httpx pyvcr stubs does not replay auth_flow as it directly patch client.send()
# So anything occurring during auth_flow have to be mocked during replay
def get_auth(owner_name=None, owner_id=None, auth=None):
if auth is None:
auth = github.get_auth(owner_name, owner_id)
auth.installation = {
"id": config.INSTALLATION_ID,
}
auth.permissions_need_to_be_updated = False
auth.owner_id = config.TESTING_ORGANIZATION_ID
auth.owner = config.TESTING_ORGANIZATION
return auth
def github_aclient(owner_name=None, owner_id=None, auth=None):
return github.AsyncGithubInstallationClient(
get_auth(owner_name, owner_id, auth))
mock.patch.object(github, "aget_client", github_aclient).start()
mock.patch.object(branch_updater.gitter, "Gitter",
self.get_gitter).start()
mock.patch.object(duplicate_pull.gitter, "Gitter",
self.get_gitter).start()
if not RECORD:
# NOTE(sileht): Don't wait exponentialy during replay
mock.patch.object(context.Context._ensure_complete.retry, "wait",
None).start()
# Web authentification always pass
mock.patch("hmac.compare_digest", return_value=True).start()
branch_prefix_path = os.path.join(self.cassette_library_dir,
"branch_prefix")
if RECORD:
self.BRANCH_PREFIX = datetime.datetime.utcnow().strftime(
"%Y%m%d%H%M%S")
with open(branch_prefix_path, "w") as f:
f.write(self.BRANCH_PREFIX)
else:
with open(branch_prefix_path, "r") as f:
self.BRANCH_PREFIX = f.read()
self.master_branch_name = self.get_full_branch_name("master")
self.git = self.get_gitter(LOG)
await self.git.init()
self.addAsyncCleanup(self.git.cleanup)
await root.startup()
self.app = httpx.AsyncClient(app=root.app, base_url="http://localhost")
await self.clear_redis_cache()
self.redis_cache = utils.create_aredis_for_cache(max_idle_time=0)
self.subscription = subscription.Subscription(
self.redis_cache,
config.TESTING_ORGANIZATION_ID,
self.SUBSCRIPTION_ACTIVE,
"You're not nice",
frozenset(
getattr(subscription.Features, f)
for f in subscription.Features.__members__)
if self.SUBSCRIPTION_ACTIVE else frozenset(),
)
await self.subscription._save_subscription_to_cache()
self.user_tokens = user_tokens.UserTokens(
self.redis_cache,
config.TESTING_ORGANIZATION_ID,
{
"mergify-test1": config.ORG_ADMIN_GITHUB_APP_OAUTH_TOKEN,
"mergify-test3": config.ORG_USER_PERSONAL_TOKEN,
},
)
await self.user_tokens.save_to_cache()
# Let's start recording
cassette = self.recorder.use_cassette("http.json")
cassette.__enter__()
self.addCleanup(cassette.__exit__)
self.client_integration = github.aget_client(
config.TESTING_ORGANIZATION, config.TESTING_ORGANIZATION_ID)
self.client_admin = github.AsyncGithubInstallationClient(
auth=github.GithubTokenAuth(token=config.ORG_ADMIN_PERSONAL_TOKEN))
self.client_fork = github.AsyncGithubInstallationClient(
auth=github.GithubTokenAuth(token=self.FORK_PERSONAL_TOKEN))
self.addAsyncCleanup(self.client_integration.aclose)
self.addAsyncCleanup(self.client_admin.aclose)
self.addAsyncCleanup(self.client_fork.aclose)
await self.client_admin.item("/user")
await self.client_fork.item("/user")
if RECORD:
assert self.client_admin.auth.owner == "mergify-test1"
assert self.client_fork.auth.owner == "mergify-test2"
else:
self.client_admin.auth.owner = "mergify-test1"
self.client_fork.auth.owner = "mergify-test2"
self.url_main = f"/repos/mergifyio-testing/{self.REPO_NAME}"
self.url_fork = f"/repos/{self.client_fork.auth.owner}/{self.REPO_NAME}"
self.git_main = f"{config.GITHUB_URL}/mergifyio-testing/{self.REPO_NAME}"
self.git_fork = (
f"{config.GITHUB_URL}/{self.client_fork.auth.owner}/{self.REPO_NAME}"
)
self.installation_ctxt = context.Installation(
config.TESTING_ORGANIZATION_ID,
config.TESTING_ORGANIZATION,
self.subscription,
self.client_integration,
self.redis_cache,
)
self.repository_ctxt = context.Repository(self.installation_ctxt,
self.REPO_NAME, self.REPO_ID)
real_get_subscription = subscription.Subscription.get_subscription
async def fake_retrieve_subscription_from_db(redis_cache, owner_id):
if owner_id == config.TESTING_ORGANIZATION_ID:
return self.subscription
return subscription.Subscription(
redis_cache,
owner_id,
False,
"We're just testing",
set(),
)
async def fake_subscription(redis_cache, owner_id):
if owner_id == config.TESTING_ORGANIZATION_ID:
return await real_get_subscription(redis_cache, owner_id)
return subscription.Subscription(
redis_cache,
owner_id,
False,
"We're just testing",
set(),
)
mock.patch(
"mergify_engine.subscription.Subscription._retrieve_subscription_from_db",
side_effect=fake_retrieve_subscription_from_db,
).start()
mock.patch(
"mergify_engine.subscription.Subscription.get_subscription",
side_effect=fake_subscription,
).start()
async def fake_retrieve_user_tokens_from_db(redis_cache, owner_id):
if owner_id == config.TESTING_ORGANIZATION_ID:
return self.user_tokens
return user_tokens.UserTokens(redis_cache, owner_id, {})
real_get_user_tokens = user_tokens.UserTokens.get
async def fake_user_tokens(redis_cache, owner_id):
if owner_id == config.TESTING_ORGANIZATION_ID:
return await real_get_user_tokens(redis_cache, owner_id)
return user_tokens.UserTokens(redis_cache, owner_id, {})
mock.patch(
"mergify_engine.user_tokens.UserTokens._retrieve_from_db",
side_effect=fake_retrieve_user_tokens_from_db,
).start()
mock.patch(
"mergify_engine.user_tokens.UserTokens.get",
side_effect=fake_user_tokens,
).start()
self._event_reader = EventReader(self.app)
await self._event_reader.drain()
# NOTE(sileht): Prepare a fresh redis
await self.clear_redis_stream()
async def test_signals(redis_cache):
gh_owner = github_types.GitHubAccount({
"login":
github_types.GitHubLogin("user"),
"id":
github_types.GitHubAccountIdType(0),
"type":
"User",
"avatar_url":
"",
})
gh_repo = github_types.GitHubRepository({
"archived":
False,
"url":
"",
"default_branch":
github_types.GitHubRefType(""),
"id":
github_types.GitHubRepositoryIdType(456),
"full_name":
"user/ref",
"name":
github_types.GitHubRepositoryName("name"),
"private":
False,
"owner":
gh_owner,
})
client = mock.AsyncMock()
client.auth.get_access_token.return_value = "<token>"
sub = subscription.Subscription(redis_cache, 0, False, "", frozenset())
installation = context.Installation(
gh_owner["id"],
gh_owner["login"],
sub,
client,
redis_cache,
)
repository = context.Repository(installation, gh_repo["name"],
gh_repo["id"])
ctxt = await context.Context.create(
repository,
{
"title": "",
"id": github_types.GitHubPullRequestId(0),
"maintainer_can_modify": False,
"rebaseable": False,
"draft": False,
"merge_commit_sha": None,
"labels": [],
"number": github_types.GitHubPullRequestNumber(6),
"commits": 1,
"merged": True,
"state": "closed",
"changed_files": 1,
"html_url": "<html_url>",
"base": {
"label": "",
"sha": github_types.SHAType("sha"),
"user": {
"login": github_types.GitHubLogin("user"),
"id": github_types.GitHubAccountIdType(0),
"type": "User",
"avatar_url": "",
},
"ref": github_types.GitHubRefType("ref"),
"label": "",
"repo": gh_repo,
},
"head": {
"label": "",
"sha": github_types.SHAType("old-sha-one"),
"ref": github_types.GitHubRefType("fork"),
"user": {
"login": github_types.GitHubLogin("user"),
"id": github_types.GitHubAccountIdType(0),
"type": "User",
"avatar_url": "",
},
"repo": {
"archived": False,
"url": "",
"default_branch": github_types.GitHubRefType(""),
"id": github_types.GitHubRepositoryIdType(123),
"full_name": "fork/other",
"name": github_types.GitHubRepositoryName("other"),
"private": False,
"owner": {
"login": github_types.GitHubLogin("user"),
"id": github_types.GitHubAccountIdType(0),
"type": "User",
"avatar_url": "",
},
},
},
"user": {
"login": github_types.GitHubLogin("user"),
"id": github_types.GitHubAccountIdType(0),
"type": "User",
"avatar_url": "",
},
"merged_by": None,
"merged_at": None,
"mergeable_state": "clean",
},
)
assert len(signals.SIGNALS) == 0
signals.setup()
assert len(signals.SIGNALS) == 1
with mock.patch(
"mergify_engine_signals.noop.Signal.__call__") as signal_method:
await signals.send(ctxt, "action.update")
signal_method.assert_called_once_with(ctxt, "action.update")
async def test_configuration_initial(
github_server: httpserver.HTTPServer,
redis_cache: utils.RedisCache,
) -> None:
github_server.expect_oneshot_request(
f"{BASE_URL}/pulls/1", ).respond_with_json(
GH_PULL,
status=200,
)
github_server.expect_oneshot_request(
f"{BASE_URL}/contents/.mergify.yml", ).respond_with_data(status=404)
github_server.expect_oneshot_request(
f"{BASE_URL}/contents/.mergify/config.yml", ).respond_with_data(
status=404)
github_server.expect_oneshot_request(
f"{BASE_URL}/contents/.github/mergify.yml", ).respond_with_data(
status=404)
github_server.expect_oneshot_request(
f"{BASE_URL}/contents/.mergify.yml",
query_string={
"ref": GH_PULL["head"]["sha"]
},
).respond_with_json(
github_types.GitHubContentFile({
"type":
"file",
"content":
FAKE_MERGIFY_CONTENT,
"path":
".mergify.yml",
"sha":
github_types.SHAType("739e5ec79e358bae7a150941a148b4131233ce2c"),
}),
status=200,
)
github_server.expect_oneshot_request(
f"{BASE_URL}/commits/{GH_PULL['head']['sha']}/check-runs"
).respond_with_json({"check_runs": []}, status=200)
github_server.expect_oneshot_request(f"{BASE_URL}/check-runs",
method="POST").respond_with_json(
{}, status=200)
async with github.AsyncGithubInstallationClient(
github.get_auth(GH_OWNER["login"])) as client:
installation = context.Installation(
GH_OWNER["id"],
GH_OWNER["login"],
subscription.Subscription(redis_cache, 0, False, "", frozenset(),
0),
client,
redis_cache,
)
repository = context.Repository(installation, GH_REPO)
ctxt = await repository.get_pull_request_context(
github_types.GitHubPullRequestNumber(1))
main_config_file = await repository.get_mergify_config_file()
assert main_config_file is None
changed = await engine._check_configuration_changes(
ctxt, main_config_file)
assert changed
github_server.check_assertions()
async def test_team_permission_cache(redis_cache: utils.RedisCache) -> None:
class FakeClient(github.AsyncGithubInstallationClient):
called: int
def __init__(self, owner, repo):
super().__init__(auth=None)
self.owner = owner
self.repo = repo
self.called = 0
async def get(self, url, *args, **kwargs):
self.called += 1
if (url ==
f"/orgs/{self.owner}/teams/team-ok/repos/{self.owner}/{self.repo}"
):
return {}
elif (url ==
f"/orgs/{self.owner}/teams/team-nok/repos/{self.owner}/{self.repo}"
):
raise http.HTTPNotFound(message="Not found",
request=mock.ANY,
response=mock.ANY)
elif (url ==
f"/orgs/{self.owner}/teams/team-also-nok/repos/{self.owner}/{self.repo}"
):
raise http.HTTPNotFound(message="Not found",
request=mock.ANY,
response=mock.ANY)
raise ValueError(f"Unknown test URL `{url}`")
gh_owner = github_types.GitHubAccount({
"id":
github_types.GitHubAccountIdType(123),
"login":
github_types.GitHubLogin("jd"),
"type":
"User",
"avatar_url":
"",
})
gh_repo = github_types.GitHubRepository({
"id":
github_types.GitHubRepositoryIdType(0),
"owner":
gh_owner,
"full_name":
"",
"archived":
False,
"url":
"",
"default_branch":
github_types.GitHubRefType(""),
"name":
github_types.GitHubRepositoryName("test"),
"private":
False,
})
team_slug1 = github_types.GitHubTeamSlug("team-ok")
team_slug2 = github_types.GitHubTeamSlug("team-nok")
team_slug3 = github_types.GitHubTeamSlug("team-also-nok")
sub = subscription.Subscription(redis_cache, 0, False, "", frozenset())
client = FakeClient(gh_owner["login"], gh_repo["name"])
installation = context.Installation(gh_owner["id"], gh_owner["login"], sub,
client, redis_cache)
repository = context.Repository(installation, gh_repo["name"],
gh_repo["id"])
assert client.called == 0
assert await repository.team_has_read_permission(team_slug1)
assert client.called == 1
assert await repository.team_has_read_permission(team_slug1)
assert client.called == 1
assert not await repository.team_has_read_permission(team_slug2)
assert client.called == 2
assert not await repository.team_has_read_permission(team_slug2)
assert client.called == 2
assert not await repository.team_has_read_permission(team_slug3)
assert client.called == 3
gh_repo = github_types.GitHubRepository({
"id":
github_types.GitHubRepositoryIdType(1),
"owner":
gh_owner,
"full_name":
"",
"archived":
False,
"url":
"",
"default_branch":
github_types.GitHubRefType(""),
"name":
github_types.GitHubRepositoryName("test2"),
"private":
False,
})
client = FakeClient(gh_owner["login"], gh_repo["name"])
installation = context.Installation(gh_owner["id"], gh_owner["login"], sub,
client, redis_cache)
repository = context.Repository(installation, gh_repo["name"],
gh_repo["id"])
assert client.called == 0
assert not await repository.team_has_read_permission(team_slug2)
assert client.called == 1
assert not await repository.team_has_read_permission(team_slug2)
assert client.called == 1
assert await repository.team_has_read_permission(team_slug1)
assert client.called == 2
await context.Repository.clear_team_permission_cache_for_repo(
redis_cache, gh_owner, gh_repo)
assert await repository.team_has_read_permission(team_slug1)
assert client.called == 3
assert not await repository.team_has_read_permission(team_slug3)
assert client.called == 4
await context.Repository.clear_team_permission_cache_for_org(
redis_cache, gh_owner)
assert not await repository.team_has_read_permission(team_slug3)
assert client.called == 5
assert not await repository.team_has_read_permission(team_slug2)
assert client.called == 6
assert not await repository.team_has_read_permission(team_slug2)
assert client.called == 6
await context.Repository.clear_team_permission_cache_for_team(
redis_cache, gh_owner, team_slug2)
assert not await repository.team_has_read_permission(team_slug2)
assert client.called == 7
async def test_user_permission_cache(redis_cache: utils.RedisCache) -> None:
class FakeClient(github.AsyncGithubInstallationClient):
called: int
def __init__(self, owner, repo):
super().__init__(auth=None)
self.owner = owner
self.repo = repo
self.called = 0
async def item(self, url, *args, **kwargs):
self.called += 1
if self.repo == "test":
if (url ==
f"/repos/{self.owner}/{self.repo}/collaborators/foo/permission"
):
return {"permission": "admin"}
elif url.startswith(
f"/repos/{self.owner}/{self.repo}/collaborators/"):
return {"permission": "loser"}
elif self.repo == "test2":
if (url ==
f"/repos/{self.owner}/{self.repo}/collaborators/bar/permission"
):
return {"permission": "admin"}
elif url.startswith(
f"/repos/{self.owner}/{self.repo}/collaborators/"):
return {"permission": "loser"}
raise ValueError(f"Unknown test URL `{url}` for repo {self.repo}")
gh_owner = github_types.GitHubAccount({
"id":
github_types.GitHubAccountIdType(123),
"login":
github_types.GitHubLogin("jd"),
"type":
"User",
"avatar_url":
"",
})
gh_repo = github_types.GitHubRepository({
"id":
github_types.GitHubRepositoryIdType(0),
"owner":
gh_owner,
"full_name":
"",
"archived":
False,
"url":
"",
"default_branch":
github_types.GitHubRefType(""),
"name":
github_types.GitHubRepositoryName("test"),
"private":
False,
})
user_1 = github_types.GitHubAccount({
"id":
github_types.GitHubAccountIdType(1),
"login":
github_types.GitHubLogin("foo"),
"type":
"User",
"avatar_url":
"",
})
user_2 = github_types.GitHubAccount({
"id":
github_types.GitHubAccountIdType(2),
"login":
github_types.GitHubLogin("bar"),
"type":
"User",
"avatar_url":
"",
})
user_3 = github_types.GitHubAccount({
"id":
github_types.GitHubAccountIdType(3),
"login":
github_types.GitHubLogin("baz"),
"type":
"User",
"avatar_url":
"",
})
sub = subscription.Subscription(redis_cache, 0, False, "", frozenset())
client = FakeClient(gh_owner["login"], gh_repo["name"])
installation = context.Installation(gh_owner["id"], gh_owner["login"], sub,
client, redis_cache)
repository = context.Repository(installation, gh_repo["name"],
gh_repo["id"])
assert client.called == 0
assert await repository.has_write_permission(user_1)
assert client.called == 1
assert await repository.has_write_permission(user_1)
assert client.called == 1
assert not await repository.has_write_permission(user_2)
assert client.called == 2
assert not await repository.has_write_permission(user_2)
assert client.called == 2
assert not await repository.has_write_permission(user_3)
assert client.called == 3
gh_repo = github_types.GitHubRepository({
"id":
github_types.GitHubRepositoryIdType(1),
"owner":
gh_owner,
"full_name":
"",
"archived":
False,
"url":
"",
"default_branch":
github_types.GitHubRefType(""),
"name":
github_types.GitHubRepositoryName("test2"),
"private":
False,
})
client = FakeClient(gh_owner["login"], gh_repo["name"])
installation = context.Installation(gh_owner["id"], gh_owner["login"], sub,
client, redis_cache)
repository = context.Repository(installation, gh_repo["name"],
gh_repo["id"])
assert client.called == 0
assert await repository.has_write_permission(user_2)
assert client.called == 1
assert await repository.has_write_permission(user_2)
assert client.called == 1
assert not await repository.has_write_permission(user_1)
assert client.called == 2
await context.Repository.clear_user_permission_cache_for_repo(
redis_cache, gh_owner, gh_repo)
assert not await repository.has_write_permission(user_1)
assert client.called == 3
assert not await repository.has_write_permission(user_3)
assert client.called == 4
await context.Repository.clear_user_permission_cache_for_org(
redis_cache, gh_owner)
assert not await repository.has_write_permission(user_3)
assert client.called == 5
assert await repository.has_write_permission(user_2)
assert client.called == 6
assert await repository.has_write_permission(user_2)
assert client.called == 6
await context.Repository.clear_user_permission_cache_for_user(
redis_cache, gh_owner, gh_repo, user_2)
assert await repository.has_write_permission(user_2)
assert client.called == 7
async def test_team_members_cache(redis_cache: utils.RedisCache) -> None:
class FakeClient(github.AsyncGithubInstallationClient):
called: int
def __init__(self, owner):
super().__init__(auth=None)
self.owner = owner
self.called = 0
async def items(self, url, *args, **kwargs):
self.called += 1
if url == f"/orgs/{self.owner}/teams/team1/members":
yield {"login": "******"}
yield {"login": "******"}
elif url == f"/orgs/{self.owner}/teams/team2/members":
yield {"login": "******"}
yield {"login": "******"}
elif url == f"/orgs/{self.owner}/teams/team3/members":
return
else:
raise ValueError(
f"Unknown test URL `{url}` for repo {self.repo}")
gh_owner = github_types.GitHubAccount({
"id":
github_types.GitHubAccountIdType(123),
"login":
github_types.GitHubLogin("jd"),
"type":
"User",
"avatar_url":
"",
})
team_slug1 = github_types.GitHubTeamSlug("team1")
team_slug2 = github_types.GitHubTeamSlug("team2")
team_slug3 = github_types.GitHubTeamSlug("team3")
sub = subscription.Subscription(redis_cache, 0, False, "", frozenset())
client = FakeClient(gh_owner["login"])
installation = context.Installation(gh_owner["id"], gh_owner["login"], sub,
client, redis_cache)
assert client.called == 0
assert (await installation.get_team_members(team_slug1)) == [
"member1", "member2"
]
assert client.called == 1
assert (await installation.get_team_members(team_slug1)) == [
"member1", "member2"
]
assert client.called == 1
assert (await installation.get_team_members(team_slug2)) == [
"member3", "member4"
]
assert client.called == 2
assert (await installation.get_team_members(team_slug2)) == [
"member3", "member4"
]
assert client.called == 2
assert (await installation.get_team_members(team_slug3)) == []
assert client.called == 3
assert (await installation.get_team_members(team_slug3)) == []
assert client.called == 3
await installation.clear_team_members_cache_for_team(
redis_cache, gh_owner, github_types.GitHubTeamSlug(team_slug2))
assert (await installation.get_team_members(team_slug2)) == [
"member3", "member4"
]
assert client.called == 4
assert (await installation.get_team_members(team_slug2)) == [
"member3", "member4"
]
assert client.called == 4
await installation.clear_team_members_cache_for_org(redis_cache, gh_owner)
assert (await installation.get_team_members(team_slug1)) == [
"member1", "member2"
]
assert client.called == 5
assert (await installation.get_team_members(team_slug1)) == [
"member1", "member2"
]
assert client.called == 5
assert (await installation.get_team_members(team_slug2)) == [
"member3", "member4"
]
assert client.called == 6
assert (await installation.get_team_members(team_slug2)) == [
"member3", "member4"
]
assert client.called == 6
assert (await installation.get_team_members(team_slug3)) == []
assert client.called == 7
assert (await installation.get_team_members(team_slug3)) == []
assert client.called == 7
async def test_team_permission_cache(redis_cache: utils.RedisCache) -> None:
class FakeClient(github.AsyncGithubInstallationClient):
called: int
def __init__(self, owner: str, repo: str) -> None:
super().__init__(auth=None) # type: ignore[arg-type]
self.owner = owner
self.repo = repo
self.called = 0
async def get(self, url: str, *args: typing.Any, **kwargs: typing.Any) -> typing.Any: # type: ignore[override]
self.called += 1
if (
url
== f"/orgs/{self.owner}/teams/team-ok/repos/{self.owner}/{self.repo}"
):
return {}
elif (
url
== f"/orgs/{self.owner}/teams/team-nok/repos/{self.owner}/{self.repo}"
):
raise http.HTTPNotFound(
message="Not found", request=mock.ANY, response=mock.ANY
)
elif (
url
== f"/orgs/{self.owner}/teams/team-also-nok/repos/{self.owner}/{self.repo}"
):
raise http.HTTPNotFound(
message="Not found", request=mock.ANY, response=mock.ANY
)
raise ValueError(f"Unknown test URL `{url}`")
gh_owner = github_types.GitHubAccount(
{
"id": github_types.GitHubAccountIdType(123),
"login": github_types.GitHubLogin("jd"),
"type": "User",
"avatar_url": "",
}
)
gh_repo = github_types.GitHubRepository(
{
"id": github_types.GitHubRepositoryIdType(0),
"owner": gh_owner,
"full_name": "",
"archived": False,
"url": "",
"html_url": "",
"default_branch": github_types.GitHubRefType(""),
"name": github_types.GitHubRepositoryName("test"),
"private": False,
}
)
installation_json = github_types.GitHubInstallation(
{
"id": github_types.GitHubInstallationIdType(12345),
"target_type": gh_owner["type"],
"permissions": {},
"account": gh_owner,
}
)
team_slug1 = github_types.GitHubTeamSlug("team-ok")
team_slug2 = github_types.GitHubTeamSlug("team-nok")
team_slug3 = github_types.GitHubTeamSlug("team-also-nok")
sub = subscription.Subscription(
redis_cache, 0, "", frozenset([subscription.Features.PUBLIC_REPOSITORY])
)
client = FakeClient(gh_owner["login"], gh_repo["name"])
installation = context.Installation(
installation_json, sub, client, redis_cache, mock.Mock()
)
repository = context.Repository(installation, gh_repo)
assert client.called == 0
assert await repository.team_has_read_permission(team_slug1)
assert client.called == 1
assert await repository.team_has_read_permission(team_slug1)
assert client.called == 1
assert not await repository.team_has_read_permission(team_slug2)
assert client.called == 2
assert not await repository.team_has_read_permission(team_slug2)
assert client.called == 2
assert not await repository.team_has_read_permission(team_slug3)
assert client.called == 3
gh_repo = github_types.GitHubRepository(
{
"id": github_types.GitHubRepositoryIdType(1),
"owner": gh_owner,
"full_name": "",
"archived": False,
"url": "",
"html_url": "",
"default_branch": github_types.GitHubRefType(""),
"name": github_types.GitHubRepositoryName("test2"),
"private": False,
}
)
client = FakeClient(gh_owner["login"], gh_repo["name"])
installation = context.Installation(
installation_json, sub, client, redis_cache, mock.Mock()
)
repository = context.Repository(installation, gh_repo)
assert client.called == 0
assert not await repository.team_has_read_permission(team_slug2)
assert client.called == 1
# From local cache
assert not await repository.team_has_read_permission(team_slug2)
assert client.called == 1
# From redis
repository._caches.team_has_read_permission.clear()
assert not await repository.team_has_read_permission(team_slug2)
assert client.called == 1
assert await repository.team_has_read_permission(team_slug1)
assert client.called == 2
await context.Repository.clear_team_permission_cache_for_repo(
redis_cache, gh_owner, gh_repo
)
repository._caches.team_has_read_permission.clear()
assert await repository.team_has_read_permission(team_slug1)
assert client.called == 3
assert not await repository.team_has_read_permission(team_slug3)
assert client.called == 4
await context.Repository.clear_team_permission_cache_for_org(redis_cache, gh_owner)
repository._caches.team_has_read_permission.clear()
assert not await repository.team_has_read_permission(team_slug3)
assert client.called == 5
assert not await repository.team_has_read_permission(team_slug2)
assert client.called == 6
# From local cache
assert not await repository.team_has_read_permission(team_slug2)
assert client.called == 6
# From redis
repository._caches.team_has_read_permission.clear()
assert not await repository.team_has_read_permission(team_slug2)
assert client.called == 6
repository._caches.team_has_read_permission.clear()
await context.Repository.clear_team_permission_cache_for_team(
redis_cache, gh_owner, team_slug2
)
repository._caches.team_has_read_permission.clear()
assert not await repository.team_has_read_permission(team_slug2)
assert client.called == 7
async def test_team_members_cache(redis_cache: utils.RedisCache) -> None:
class FakeClient(github.AsyncGithubInstallationClient):
called: int
def __init__(self, owner: str) -> None:
super().__init__(auth=None) # type: ignore[arg-type]
self.owner = owner
self.called = 0
async def items(self, url, *args, **kwargs):
self.called += 1
if url == f"/orgs/{self.owner}/teams/team1/members":
yield {"login": "******"}
yield {"login": "******"}
elif url == f"/orgs/{self.owner}/teams/team2/members":
yield {"login": "******"}
yield {"login": "******"}
elif url == f"/orgs/{self.owner}/teams/team3/members":
return
else:
raise ValueError(f"Unknown test URL `{url}` for repo {self.repo}")
gh_owner = github_types.GitHubAccount(
{
"id": github_types.GitHubAccountIdType(123),
"login": github_types.GitHubLogin("jd"),
"type": "User",
"avatar_url": "",
}
)
installation_json = github_types.GitHubInstallation(
{
"id": github_types.GitHubInstallationIdType(12345),
"target_type": gh_owner["type"],
"permissions": {},
"account": gh_owner,
}
)
team_slug1 = github_types.GitHubTeamSlug("team1")
team_slug2 = github_types.GitHubTeamSlug("team2")
team_slug3 = github_types.GitHubTeamSlug("team3")
sub = subscription.Subscription(
redis_cache, 0, "", frozenset([subscription.Features.PUBLIC_REPOSITORY])
)
client = FakeClient(gh_owner["login"])
installation = context.Installation(
installation_json, sub, client, redis_cache, mock.Mock()
)
assert client.called == 0
assert (await installation.get_team_members(team_slug1)) == ["member1", "member2"]
assert client.called == 1
assert (await installation.get_team_members(team_slug1)) == ["member1", "member2"]
assert client.called == 1
assert (await installation.get_team_members(team_slug2)) == ["member3", "member4"]
assert client.called == 2
# From local cache
assert (await installation.get_team_members(team_slug2)) == ["member3", "member4"]
assert client.called == 2
# From redis
installation._caches.team_members.clear()
assert (await installation.get_team_members(team_slug2)) == ["member3", "member4"]
assert client.called == 2
assert (await installation.get_team_members(team_slug3)) == []
assert client.called == 3
# From local cache
assert (await installation.get_team_members(team_slug3)) == []
assert client.called == 3
# From redis
installation._caches.team_members.clear()
assert (await installation.get_team_members(team_slug3)) == []
assert client.called == 3
await installation.clear_team_members_cache_for_team(
redis_cache, gh_owner, github_types.GitHubTeamSlug(team_slug2)
)
installation._caches.team_members.clear()
assert (await installation.get_team_members(team_slug2)) == ["member3", "member4"]
assert client.called == 4
# From local cache
assert (await installation.get_team_members(team_slug2)) == ["member3", "member4"]
assert client.called == 4
# From redis
installation._caches.team_members.clear()
assert (await installation.get_team_members(team_slug2)) == ["member3", "member4"]
assert client.called == 4
await installation.clear_team_members_cache_for_org(redis_cache, gh_owner)
installation._caches.team_members.clear()
assert (await installation.get_team_members(team_slug1)) == ["member1", "member2"]
assert client.called == 5
# From local cache
assert (await installation.get_team_members(team_slug1)) == ["member1", "member2"]
assert client.called == 5
# From redis
installation._caches.team_members.clear()
assert (await installation.get_team_members(team_slug1)) == ["member1", "member2"]
assert client.called == 5
assert (await installation.get_team_members(team_slug2)) == ["member3", "member4"]
assert client.called == 6
# From local cache
assert (await installation.get_team_members(team_slug2)) == ["member3", "member4"]
assert client.called == 6
# From redis
installation._caches.team_members.clear()
assert (await installation.get_team_members(team_slug2)) == ["member3", "member4"]
assert client.called == 6
assert (await installation.get_team_members(team_slug3)) == []
assert client.called == 7
# From local cache
assert (await installation.get_team_members(team_slug3)) == []
assert client.called == 7
# From redis
installation._caches.team_members.clear()
assert (await installation.get_team_members(team_slug3)) == []
assert client.called == 7
async def consume(self, stream_name: StreamNameType) -> None:
owner_login, owner_id = self._extract_owner(stream_name)
LOG.debug("consoming stream", gh_owner=owner_login)
try:
async with self._translate_exception_to_retries(stream_name):
sub = await subscription.Subscription.get_subscription(
self.redis_cache, owner_id
)
async with github.aget_client(owner_login) as client:
installation = context.Installation(
owner_id, owner_login, sub, client, self.redis_cache
)
async with self._translate_exception_to_retries(stream_name):
pulls = await self._extract_pulls_from_stream(installation)
if pulls:
client.set_requests_ratio(len(pulls))
await self._consume_pulls(installation, pulls)
await self._refresh_merge_trains(installation)
except StreamUnused:
LOG.info("unused stream, dropping it", gh_owner=owner_login, exc_info=True)
try:
await self.redis_stream.delete(stream_name)
except aredis.exceptions.ConnectionError:
LOG.warning(
"fail to drop stream, it will be retried", stream_name=stream_name
)
except StreamRetry as e:
log_method = (
LOG.error
if e.attempts >= STREAM_ATTEMPTS_LOGGING_THRESHOLD
else LOG.info
)
log_method(
"failed to process stream, retrying",
attempts=e.attempts,
retry_at=e.retry_at,
gh_owner=owner_login,
exc_info=True,
)
return
except vcr_errors_CannotOverwriteExistingCassetteException:
messages = await self.redis_stream.xrange(
stream_name, count=config.STREAM_MAX_BATCH
)
for message_id, message in messages:
LOG.info(msgpack.unpackb(message[b"event"], raw=False))
await self.redis_stream.execute_command("XDEL", stream_name, message_id)
except Exception:
# Ignore it, it will retried later
LOG.error("failed to process stream", gh_owner=owner_login, exc_info=True)
LOG.debug("cleanup stream start", stream_name=stream_name)
try:
await self.redis_stream.eval(
self.ATOMIC_CLEAN_STREAM_SCRIPT, 1, stream_name.encode(), time.time()
)
except aredis.exceptions.ConnectionError:
LOG.warning(
"fail to cleanup stream, it maybe partially replayed",
stream_name=stream_name,
)
LOG.debug("cleanup stream end", stream_name=stream_name)