async def test_train_add_remove_pull_idempotant(repository,
monkepatched_traincar):
t = merge_train.Train(repository, "branch")
await t.load()
config = get_config("five", priority=0)
await t.add_pull(await fake_context(repository, 1), config)
await t.add_pull(await fake_context(repository, 2), config)
await t.add_pull(await fake_context(repository, 3), config)
await t.refresh()
assert [[1], [1, 2], [1, 2, 3]] == get_cars_content(t)
config = get_config("five", priority=10)
await t.add_pull(await fake_context(repository, 1), config)
await t.refresh()
assert [[1], [1, 2], [1, 2, 3]] == get_cars_content(t)
t = merge_train.Train(repository, "branch")
await t.load()
assert [[1], [1, 2], [1, 2, 3]] == get_cars_content(t)
await t.remove_pull(await fake_context(repository, 2))
await t.refresh()
assert [[1], [1, 3]] == get_cars_content(t)
await t.remove_pull(await fake_context(repository, 2))
await t.refresh()
assert [[1], [1, 3]] == get_cars_content(t)
t = merge_train.Train(repository, "branch")
await t.load()
assert [[1], [1, 3]] == get_cars_content(t)
async def test_train_add_remove_pull_idempotant(
repository: context.Repository, context_getter: conftest.ContextGetterFixture
) -> None:
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
config = get_config("5x1", priority=0)
await t.add_pull(await context_getter(1), config)
await t.add_pull(await context_getter(2), config)
await t.add_pull(await context_getter(3), config)
await t.refresh()
assert [[1], [1, 2], [1, 2, 3]] == get_cars_content(t)
config = get_config("5x1", priority=10)
await t.add_pull(await context_getter(1), config)
await t.refresh()
assert [[1], [1, 2], [1, 2, 3]] == get_cars_content(t)
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
assert [[1], [1, 2], [1, 2, 3]] == get_cars_content(t)
await t.remove_pull(await context_getter(2))
await t.refresh()
assert [[1], [1, 3]] == get_cars_content(t)
await t.remove_pull(await context_getter(2))
await t.refresh()
assert [[1], [1, 3]] == get_cars_content(t)
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
assert [[1], [1, 3]] == get_cars_content(t)
async def test_train_mutiple_queue(
repository: context.Repository, context_getter: conftest.ContextGetterFixture
) -> None:
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
config_two = get_config("2x1", priority=0)
config_five = get_config("5x1", priority=0)
await t.add_pull(await context_getter(1), config_two)
await t.add_pull(await context_getter(2), config_two)
await t.add_pull(await context_getter(3), config_five)
await t.add_pull(await context_getter(4), config_five)
await t.refresh()
assert [[1], [1, 2]] == get_cars_content(t)
assert [3, 4] == get_waiting_content(t)
# Ensure we don't got over the train_size
await t.add_pull(await context_getter(5), config_two)
await t.refresh()
assert [[1], [1, 2]] == get_cars_content(t)
assert [5, 3, 4] == get_waiting_content(t)
await t.add_pull(await context_getter(6), config_five)
await t.add_pull(await context_getter(7), config_five)
await t.add_pull(await context_getter(8), config_five)
await t.add_pull(await context_getter(9), config_five)
await t.refresh()
assert [[1], [1, 2]] == get_cars_content(t)
assert [5, 3, 4, 6, 7, 8, 9] == get_waiting_content(t)
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
assert [[1], [1, 2]] == get_cars_content(t)
assert [5, 3, 4, 6, 7, 8, 9] == get_waiting_content(t)
await t.remove_pull(await context_getter(2))
await t.refresh()
assert [[1], [1, 5]] == get_cars_content(
t
), f"{get_cars_content(t)} {get_waiting_content(t)}"
assert [3, 4, 6, 7, 8, 9] == get_waiting_content(t)
await t.remove_pull(await context_getter(1))
await t.remove_pull(await context_getter(5))
await t.refresh()
assert [[3], [3, 4], [3, 4, 6], [3, 4, 6, 7], [3, 4, 6, 7, 8]] == get_cars_content(
t
)
assert [9] == get_waiting_content(t)
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
assert [[3], [3, 4], [3, 4, 6], [3, 4, 6, 7], [3, 4, 6, 7, 8]] == get_cars_content(
t
)
assert [9] == get_waiting_content(t)
async def test_train_mutiple_queue(repository, monkepatched_traincar):
t = merge_train.Train(repository, "branch")
await t.load()
config_two = get_config("two", priority=0)
config_five = get_config("five", priority=0)
await t.add_pull(await fake_context(repository, 1), config_two)
await t.add_pull(await fake_context(repository, 2), config_two)
await t.add_pull(await fake_context(repository, 3), config_five)
await t.add_pull(await fake_context(repository, 4), config_five)
await t.refresh()
assert [[1], [1, 2]] == get_cars_content(t)
assert [3, 4] == get_waiting_content(t)
# Ensure we don't got over the train_size
await t.add_pull(await fake_context(repository, 5), config_two)
await t.refresh()
assert [[1], [1, 2]] == get_cars_content(t)
assert [5, 3, 4] == get_waiting_content(t)
await t.add_pull(await fake_context(repository, 6), config_five)
await t.add_pull(await fake_context(repository, 7), config_five)
await t.add_pull(await fake_context(repository, 8), config_five)
await t.add_pull(await fake_context(repository, 9), config_five)
await t.refresh()
assert [[1], [1, 2]] == get_cars_content(t)
assert [5, 3, 4, 6, 7, 8, 9] == get_waiting_content(t)
t = merge_train.Train(repository, "branch")
await t.load()
assert [[1], [1, 2]] == get_cars_content(t)
assert [5, 3, 4, 6, 7, 8, 9] == get_waiting_content(t)
await t.remove_pull(await fake_context(repository, 2))
await t.refresh()
assert [[1], [1, 5]] == get_cars_content(t)
assert [3, 4, 6, 7, 8, 9] == get_waiting_content(t)
await t.remove_pull(await fake_context(repository, 1))
await t.remove_pull(await fake_context(repository, 5))
await t.refresh()
assert [[3], [3, 4], [3, 4, 6], [3, 4, 6, 7], [3, 4, 6, 7, 8]] == get_cars_content(
t
)
assert [9] == get_waiting_content(t)
t = merge_train.Train(repository, "branch")
await t.load()
assert [[3], [3, 4], [3, 4, 6], [3, 4, 6, 7], [3, 4, 6, 7, 8]] == get_cars_content(
t
)
assert [9] == get_waiting_content(t)
async def test_train_remove_duplicates(
repository: context.Repository, context_getter: conftest.ContextGetterFixture
) -> None:
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
await t.add_pull(await context_getter(1), get_config("2x1", 1000))
await t.add_pull(await context_getter(2), get_config("2x1", 1000))
await t.add_pull(await context_getter(3), get_config("2x1", 1000))
await t.add_pull(await context_getter(4), get_config("2x1", 1000))
await t.refresh()
assert [[1], [1, 2]] == get_cars_content(t)
assert [3, 4] == get_waiting_content(t)
# Insert bugs in queue
t._waiting_pulls.extend(
[
merge_train.EmbarkedPull(
t,
t._cars[0].still_queued_embarked_pulls[0].user_pull_request_number,
t._cars[0].still_queued_embarked_pulls[0].config,
t._cars[0].still_queued_embarked_pulls[0].queued_at,
),
t._waiting_pulls[0],
]
)
t._cars = t._cars + t._cars
assert [[1], [1, 2], [1], [1, 2]] == get_cars_content(t)
assert [1, 3, 3, 4] == get_waiting_content(t)
# Everything should be back to normal
await t.refresh()
assert [[1], [1, 2]] == get_cars_content(t)
assert [3, 4] == get_waiting_content(t)
async def test_train_queue_config_deleted(
report_failure: mock.Mock,
repository: context.Repository,
context_getter: conftest.ContextGetterFixture,
) -> None:
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
await t.add_pull(await context_getter(1), get_config("2x1", 1000))
await t.add_pull(await context_getter(2), get_config("2x1", 1000))
await t.add_pull(await context_getter(3), get_config("5x1", 1000))
await t.refresh()
assert [[1], [1, 2]] == get_cars_content(t)
assert [3] == get_waiting_content(t)
with mock.patch.object(
sys.modules[__name__],
"MERGIFY_CONFIG",
"""
queue_rules:
- name: five
conditions: []
speculative_checks: 5
""",
):
repository._caches.mergify_config.delete()
repository._caches.mergify_config_file.delete()
await t.refresh()
assert [] == get_cars_content(t)
assert [1, 2, 3] == get_waiting_content(t)
assert len(report_failure.mock_calls) == 1
async def test_train_priority_change(
repository: context.Repository,
context_getter: conftest.ContextGetterFixture,
) -> None:
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
await t.add_pull(await context_getter(1), get_config("2x1", 1000))
await t.add_pull(await context_getter(2), get_config("2x1", 1000))
await t.add_pull(await context_getter(3), get_config("2x1", 1000))
await t.refresh()
assert [[1], [1, 2]] == get_cars_content(t)
assert [3] == get_waiting_content(t)
assert (
t._cars[0].still_queued_embarked_pulls[0].config["effective_priority"]
== QUEUE_RULES["2x1"].config["priority"] * queue.QUEUE_PRIORITY_OFFSET + 1000
)
# NOTE(sileht): pull request got requeued with new configuration that don't
# update the position but update the prio
await t.add_pull(await context_getter(1), get_config("2x1", 2000))
await t.refresh()
assert [[1], [1, 2]] == get_cars_content(t)
assert [3] == get_waiting_content(t)
assert (
t._cars[0].still_queued_embarked_pulls[0].config["effective_priority"]
== QUEUE_RULES["2x1"].config["priority"] * queue.QUEUE_PRIORITY_OFFSET + 2000
)
async def test_train_disallow_checks_interruption_scenario_2(
repository: context.Repository, context_getter: conftest.ContextGetterFixture
) -> None:
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
urgent = get_config("urgent-1x4")
fastlane = get_config("fastlane-1x8-noint")
regular = get_config("regular-1x8-noint-from-fastlane-and-regular")
await t.add_pull(await context_getter(1), regular)
await t.add_pull(await context_getter(2), regular)
await t.refresh()
assert [[1, 2]] == get_cars_content(t)
assert [] == get_waiting_content(t)
# fastlane doesn't interrupt the checks as
# disallow_checks_interruption_from_queues of regular disallow it
await t.add_pull(await context_getter(3), fastlane)
await t.refresh()
assert [[1, 2]] == get_cars_content(t)
assert [3] == get_waiting_content(t)
# fastlane doesn't interrupt the checks because of noint, but goes before
# regular
await t.add_pull(await context_getter(4), regular)
await t.refresh()
assert [[1, 2]] == get_cars_content(t)
assert [3, 4] == get_waiting_content(t)
# urgent breaks everything, then we put the fastlane one, and all regulars goes behind
await t.add_pull(await context_getter(5), urgent)
await t.refresh()
assert [[5]] == get_cars_content(t)
assert [3, 1, 2, 4] == get_waiting_content(t)
async def test_train_interrupt_mixed_across_queue(
repository: context.Repository, context_getter: conftest.ContextGetterFixture
) -> None:
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
config = get_config("low-1x5-noint")
await t.add_pull(await context_getter(1), config)
await t.refresh()
assert [[1]] == get_cars_content(t)
assert [] == get_waiting_content(t)
await t.add_pull(await context_getter(2), config)
await t.refresh()
assert [[1]] == get_cars_content(t)
assert [2] == get_waiting_content(t)
await t.add_pull(await context_getter(3), config)
await t.refresh()
assert [[1]] == get_cars_content(t)
assert [2, 3] == get_waiting_content(t)
# Inserting pr in high queue always break started speculative checks
await t.add_pull(await context_getter(4), get_config("high-1x2"))
await t.refresh()
assert [[4]] == get_cars_content(t)
assert [1, 2, 3] == get_waiting_content(t)
async def test_train_no_interrupt_add_pull(
repository: context.Repository, context_getter: conftest.ContextGetterFixture
) -> None:
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
config = get_config("high-2x5-noint")
await t.add_pull(await context_getter(1), config)
await t.refresh()
assert [[1]] == get_cars_content(t)
assert [] == get_waiting_content(t)
await t.add_pull(await context_getter(2), config)
await t.refresh()
assert [[1], [1, 2]] == get_cars_content(t)
assert [] == get_waiting_content(t)
await t.add_pull(await context_getter(3), config)
await t.refresh()
assert [[1], [1, 2]] == get_cars_content(t)
assert [3] == get_waiting_content(t)
# Inserting high prio didn't break started speculative checks, but the PR
# move above other
await t.add_pull(await context_getter(4), get_config("high-2x5-noint", 20000))
await t.refresh()
assert [[1], [1, 2]] == get_cars_content(t)
assert [4, 3] == get_waiting_content(t)
def test_train_batch_split(repository: context.Repository) -> None:
now = datetime.datetime.utcnow()
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
p1_two = merge_train.EmbarkedPull(
t, github_types.GitHubPullRequestNumber(1), get_config("2x1"), now
)
p2_two = merge_train.EmbarkedPull(
t, github_types.GitHubPullRequestNumber(2), get_config("2x1"), now
)
p3_two = merge_train.EmbarkedPull(
t, github_types.GitHubPullRequestNumber(3), get_config("2x1"), now
)
p4_five = merge_train.EmbarkedPull(
t, github_types.GitHubPullRequestNumber(4), get_config("5x1"), now
)
assert ([p1_two], [p2_two, p3_two, p4_five]) == t._get_next_batch(
[p1_two, p2_two, p3_two, p4_five], "2x1", 1
)
assert ([p1_two, p2_two], [p3_two, p4_five]) == t._get_next_batch(
[p1_two, p2_two, p3_two, p4_five], "2x1", 2
)
assert ([p1_two, p2_two, p3_two], [p4_five]) == t._get_next_batch(
[p1_two, p2_two, p3_two, p4_five], "2x1", 10
)
assert ([], [p1_two, p2_two, p3_two, p4_five]) == t._get_next_batch(
[p1_two, p2_two, p3_two, p4_five], "5x1", 10
)
async def test_train_add_remove_pull_idempotant(repository, monkepatched_traincar):
t = merge_train.Train(repository, "branch")
await t.load()
config = queue.QueueConfig(
name="foo",
strict_method="merge",
priority=0,
effective_priority=0,
bot_account=None,
update_bot_account=None,
)
await t.add_pull(await fake_context(repository, 1), config)
await t.add_pull(await fake_context(repository, 2), config)
await t.add_pull(await fake_context(repository, 3), config)
await t.refresh()
assert [[1], [1, 2], [1, 2, 3]] == get_cars_content(t)
config = queue.QueueConfig(
name="foo",
strict_method="merge",
priority=10,
effective_priority=10,
bot_account=None,
update_bot_account=None,
)
await t.add_pull(await fake_context(repository, 1), config)
await t.refresh()
assert [[1], [1, 2], [1, 2, 3]] == get_cars_content(t)
t = merge_train.Train(repository, "branch")
await t.load()
assert [[1], [1, 2], [1, 2, 3]] == get_cars_content(t)
await t.remove_pull(await fake_context(repository, 2))
await t.refresh()
assert [[1], [1, 3]] == get_cars_content(t)
await t.remove_pull(await fake_context(repository, 2))
await t.refresh()
assert [[1], [1, 3]] == get_cars_content(t)
t = merge_train.Train(repository, "branch")
await t.load()
assert [[1], [1, 3]] == get_cars_content(t)
async def test_train_remove_middle_not_merged(repository, monkepatched_traincar):
t = merge_train.Train(repository, "branch")
await t.load()
await t.add_pull(await fake_context(repository, 1), get_config("five", 1000))
await t.add_pull(await fake_context(repository, 3), get_config("five", 100))
await t.add_pull(await fake_context(repository, 2), get_config("five", 1000))
await t.refresh()
assert [[1], [1, 2], [1, 2, 3]] == get_cars_content(t)
await t.remove_pull(await fake_context(repository, 2))
await t.refresh()
assert [[1], [1, 3]] == get_cars_content(t)
async def test_train_remove_middle_not_merged(
repository: context.Repository, context_getter: conftest.ContextGetterFixture
) -> None:
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
await t.add_pull(await context_getter(1), get_config("5x1", 1000))
await t.add_pull(await context_getter(3), get_config("5x1", 100))
await t.add_pull(await context_getter(2), get_config("5x1", 1000))
await t.refresh()
assert [[1], [1, 2], [1, 2, 3]] == get_cars_content(t)
await t.remove_pull(await context_getter(2))
await t.refresh()
assert [[1], [1, 3]] == get_cars_content(t)
async def test_train_remove_middle_merged(repository, monkepatched_traincar):
t = merge_train.Train(repository, "branch")
await t.load()
config = get_config("five")
await t.add_pull(await fake_context(repository, 1), config)
await t.add_pull(await fake_context(repository, 2), config)
await t.add_pull(await fake_context(repository, 3), config)
await t.refresh()
assert [[1], [1, 2], [1, 2, 3]] == get_cars_content(t)
await t.remove_pull(
await fake_context(repository, 2, merged=True, merge_commit_sha="new_sha1")
)
await t.refresh()
assert [[1], [1, 3]] == get_cars_content(t)
async def test_train_queue_pr_with_higher_prio_enters_in_queue_during_merging_2x5(
report_failure: mock.Mock,
repository: context.Repository,
context_getter: conftest.ContextGetterFixture,
fake_client: mock.Mock,
) -> None:
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
for i in range(41, 52):
await t.add_pull(await context_getter(i), get_config("2x5", 1000))
await t.refresh()
assert [
[41, 42, 43, 44, 45],
[41, 42, 43, 44, 45, 46, 47, 48, 49, 50],
] == get_cars_content(t)
assert [51] == get_waiting_content(t)
t._cars[0].checks_conclusion = check_api.Conclusion.SUCCESS
await t.save()
await t.refresh()
assert [
[41, 42, 43, 44, 45],
[41, 42, 43, 44, 45, 46, 47, 48, 49, 50],
] == get_cars_content(t)
assert [51] == get_waiting_content(t)
# merge half of the batch
for i in range(41, 44):
fake_client.update_base_sha(f"sha{i}")
await t.remove_pull(
await context_getter(i, merged=True, merge_commit_sha=f"sha{i}")
)
await t.refresh()
assert [
[44, 45],
[41, 42, 43, 44, 45, 46, 47, 48, 49, 50],
] == get_cars_content(t)
assert [51] == get_waiting_content(t)
await t.add_pull(await context_getter(7), get_config("2x5", 2000))
await t.refresh()
assert [[44, 45], [44, 45, 7, 46, 47, 48, 49]] == get_cars_content(t)
assert [50, 51] == get_waiting_content(t)
async def test_train_queue_splitted_on_failure_1x2(
report_failure: mock.Mock,
repository: context.Repository,
fake_client: mock.Mock,
context_getter: conftest.ContextGetterFixture,
) -> None:
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
for i in range(41, 43):
await t.add_pull(await context_getter(i), get_config("high-1x2", 1000))
for i in range(6, 20):
await t.add_pull(await context_getter(i), get_config("high-1x2", 1000))
await t.refresh()
assert [[41, 42]] == get_cars_content(t)
assert list(range(6, 20)) == get_waiting_content(t)
t._cars[0].checks_conclusion = check_api.Conclusion.FAILURE
await t.save()
assert [[41, 42]] == get_cars_content(t)
assert list(range(6, 20)) == get_waiting_content(t)
await t.load()
await t.refresh()
assert [
[41],
[41, 42],
] == get_cars_content(t)
assert list(range(6, 20)) == get_waiting_content(t)
assert len(t._cars[0].failure_history) == 1
assert len(t._cars[1].failure_history) == 0
assert t._cars[0].creation_state == "updated"
assert t._cars[1].creation_state == "created"
# mark [41] as failed
t._cars[1].checks_conclusion = check_api.Conclusion.FAILURE
await t.save()
await t.remove_pull(await context_getter(41, merged=False))
# It's 41 fault, we restart the train on 42
await t.refresh()
assert [[42, 6]] == get_cars_content(t)
assert list(range(7, 20)) == get_waiting_content(t)
assert len(t._cars[0].failure_history) == 0
assert t._cars[0].creation_state == "created" # type: ignore[comparison-overlap]
async def test_train_remove_last_cars(
repository: context.Repository, context_getter: conftest.ContextGetterFixture
) -> None:
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
await t.add_pull(await context_getter(1), get_config("high-1x1", 1000))
await t.add_pull(await context_getter(2), get_config("high-1x1", 1000))
await t.add_pull(await context_getter(3), get_config("high-1x1", 1000))
await t.refresh()
assert [[1]] == get_cars_content(t)
assert [2, 3] == get_waiting_content(t)
await t.remove_pull(await context_getter(1))
await t.refresh()
assert [[2]] == get_cars_content(t)
assert [3] == get_waiting_content(t)
async def test_train_remove_last_cars(repository, monkepatched_traincar):
t = merge_train.Train(repository, "branch")
await t.load()
await t.add_pull(await fake_context(repository, 1),
get_config("one", 1000))
await t.add_pull(await fake_context(repository, 2),
get_config("one", 1000))
await t.add_pull(await fake_context(repository, 3),
get_config("one", 1000))
await t.refresh()
assert [[1]] == get_cars_content(t)
assert [2, 3] == get_waiting_content(t)
await t.remove_pull(await fake_context(repository, 1))
await t.refresh()
assert [[2]] == get_cars_content(t)
assert [3] == get_waiting_content(t)
async def test_train_remove_head_merged(
repository: context.Repository, context_getter: conftest.ContextGetterFixture
) -> None:
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
config = get_config("5x1")
await t.add_pull(await context_getter(1), config)
await t.add_pull(await context_getter(2), config)
await t.add_pull(await context_getter(3), config)
await t.refresh()
assert [[1], [1, 2], [1, 2, 3]] == get_cars_content(t)
await t.remove_pull(
await context_getter(1, merged=True, merge_commit_sha="new_sha1")
)
await t.refresh()
assert [[1, 2], [1, 2, 3]] == get_cars_content(t)
async def test_train_with_speculative_checks_decreased(
repository: context.Repository, context_getter: conftest.ContextGetterFixture
) -> None:
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
config = get_config("5x1", 1000)
await t.add_pull(await context_getter(1), config)
QUEUE_RULES["5x1"].config["speculative_checks"] = 2
await t.add_pull(await context_getter(2), config)
await t.add_pull(await context_getter(3), config)
await t.add_pull(await context_getter(4), config)
await t.add_pull(await context_getter(5), config)
await t.refresh()
assert [[1], [1, 2], [1, 2, 3], [1, 2, 3, 4], [1, 2, 3, 4, 5]] == get_cars_content(
t
)
assert [] == get_waiting_content(t)
await t.remove_pull(
await context_getter(1, merged=True, merge_commit_sha="new_sha1")
)
with mock.patch.object(
sys.modules[__name__],
"MERGIFY_CONFIG",
"""
queue_rules:
- name: 5x1
conditions: []
speculative_checks: 2
""",
):
repository._caches.mergify_config.delete()
repository._caches.mergify_config_file.delete()
await t.refresh()
assert [[1, 2], [1, 2, 3]] == get_cars_content(t)
assert [4, 5] == get_waiting_content(t)
async def test_train_batch_max_wait_time(
repository: context.Repository, context_getter: conftest.ContextGetterFixture
) -> None:
with freeze_time("2021-09-22T08:00:00") as freezed_time:
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
config = get_config("batch-wait-time")
await t.add_pull(await context_getter(1), config)
await t.refresh()
assert [] == get_cars_content(t)
assert [1] == get_waiting_content(t)
# Enought PR to batch!
await t.add_pull(await context_getter(2), config)
await t.refresh()
assert [[1, 2]] == get_cars_content(t)
assert [] == get_waiting_content(t)
await t.add_pull(await context_getter(3), config)
await t.refresh()
assert [[1, 2]] == get_cars_content(t)
assert [3] == get_waiting_content(t)
d = await delayed_refresh._get_current_refresh_datetime(
repository, github_types.GitHubPullRequestNumber(3)
)
assert d is not None
assert d == freezed_time().replace(
tzinfo=datetime.timezone.utc
) + datetime.timedelta(minutes=5)
with freeze_time("2021-09-22T08:05:02"):
await t.refresh()
assert [[1, 2], [1, 2, 3]] == get_cars_content(t)
assert [] == get_waiting_content(t)
async def report(
url: str,
) -> typing.Union[context.Context, github.AsyncGithubInstallationClient, None]:
redis_links = redis_utils.RedisLinks(max_idle_time=0)
try:
owner_login, repo, pull_number = _url_parser(url)
except ValueError:
print(f"{url} is not valid")
return None
try:
installation_json = await github.get_installation_from_login(
owner_login)
client = github.aget_client(installation_json)
except exceptions.MergifyNotInstalled:
print(f"* Mergify is not installed on account {owner_login}")
return None
# Do a dumb request just to authenticate
await client.get("/")
print(f"* INSTALLATION ID: {installation_json['id']}")
if repo is None:
slug = None
else:
slug = owner_login + "/" + repo
owner_id = installation_json["account"]["id"]
cached_sub = await subscription.Subscription.get_subscription(
redis_links.cache, owner_id)
db_sub = await subscription.Subscription._retrieve_subscription_from_db(
redis_links.cache, owner_id)
cached_tokens = await user_tokens.UserTokens.get(redis_links.cache,
owner_id)
if config.SAAS_MODE:
db_tokens = typing.cast(
user_tokens.UserTokens,
(await user_tokens.UserTokensSaas._retrieve_from_db(
redis_links.cache, owner_id)),
)
else:
db_tokens = cached_tokens
print("* Features (db):")
for v in sorted(f.value for f in db_sub.features):
print(f" - {v}")
print("* Features (cache):")
for v in sorted(f.value for f in cached_sub.features):
print(f" - {v}")
installation = context.Installation(installation_json, cached_sub, client,
redis_links)
await report_dashboard_synchro(installation.installation["id"], cached_sub,
cached_tokens, "ENGINE-CACHE", slug)
await report_dashboard_synchro(installation.installation["id"], db_sub,
db_tokens, "DASHBOARD", slug)
await report_worker_status(owner_login)
if repo is not None:
repository = await installation.get_repository_by_name(repo)
print(
f"* REPOSITORY IS {'PRIVATE' if repository.repo['private'] else 'PUBLIC'}"
)
print(f"* DEFAULT BRANCH: {repository.repo['default_branch']}")
print("* CONFIGURATION:")
mergify_config = None
config_file = await repository.get_mergify_config_file()
if not config_file:
print(".mergify.yml is missing")
else:
print(f"Config filename: {config_file['path']}")
print(config_file["decoded_content"])
try:
mergify_config = await repository.get_mergify_config()
except rules.InvalidRules as e: # pragma: no cover
print(f"configuration is invalid {str(e)}")
if pull_number is None:
async for branch in typing.cast(
typing.AsyncGenerator[github_types.GitHubBranch, None],
client.items(
f"/repos/{owner_login}/{repo}/branches",
resource_name="branches",
page_limit=100,
),
):
q = merge_train.Train(repository, branch["name"])
await q.load()
await report_queue("TRAIN", q)
else:
repository = await installation.get_repository_by_name(
github_types.GitHubRepositoryName(repo))
try:
ctxt = await repository.get_pull_request_context(
github_types.GitHubPullRequestNumber(int(pull_number)))
except http.HTTPNotFound:
print(f"Pull request `{url}` does not exist")
return client
# FIXME queues could also be printed if no pull number given
# TODO(sileht): display train if any
q = await merge_train.Train.from_context(ctxt)
print(
f"* TRAIN: {', '.join([f'#{p}' for p in await q.get_pulls()])}"
)
print("* PULL REQUEST:")
pr_data = await ctxt.pull_request.items()
pprint.pprint(pr_data, width=160)
is_behind = await ctxt.is_behind
print(f"is_behind: {is_behind}")
print(f"mergeable_state: {ctxt.pull['mergeable_state']}")
print("* MERGIFY LAST CHECKS:")
for c in await ctxt.pull_engine_check_runs:
print(
f"[{c['name']}]: {c['conclusion']} | {c['output'].get('title')} | {c['html_url']}"
)
print("> " + "\n> ".join(
("No Summary", ) if c["output"]["summary"] is None else
c["output"]["summary"].split("\n")))
if mergify_config is not None:
print("* MERGIFY LIVE MATCHES:")
pull_request_rules = mergify_config["pull_request_rules"]
match = await pull_request_rules.get_pull_request_rule(ctxt)
summary_title, summary = await actions_runner.gen_summary(
ctxt, pull_request_rules, match)
print(f"[Summary]: success | {summary_title}")
print("> " + "\n> ".join(summary.strip().split("\n")))
return ctxt
return client
async def report(
url: str,
) -> typing.Union[context.Context, github.AsyncGithubInstallationClient, None]:
redis_cache = utils.create_aredis_for_cache(max_idle_time=0)
try:
owner, repo, pull_number = _url_parser(url)
except ValueError:
print(f"{url} is not valid")
return None
try:
client = github.aget_client(owner)
except exceptions.MergifyNotInstalled:
print(f"* Mergify is not installed on account {owner}")
return None
# Do a dumb request just to authenticate
await client.get("/")
if client.auth.installation is None:
print("No installation detected")
return None
print(f"* INSTALLATION ID: {client.auth.installation['id']}")
if client.auth.owner_id is None:
raise RuntimeError("Unable to get owner_id")
if repo is None:
slug = None
else:
slug = owner + "/" + repo
cached_sub = await subscription.Subscription.get_subscription(
redis_cache, client.auth.owner_id)
db_sub = await subscription.Subscription._retrieve_subscription_from_db(
redis_cache, client.auth.owner_id)
cached_tokens = await user_tokens.UserTokens.get(redis_cache,
client.auth.owner_id)
db_tokens = await user_tokens.UserTokens._retrieve_from_db(
redis_cache, client.auth.owner_id)
print(f"* SUBSCRIBED (cache/db): {cached_sub.active} / {db_sub.active}")
print("* Features (cache):")
for f in db_sub.features:
print(f" - {f.value}")
print("* Features (db):")
for f in cached_sub.features:
print(f" - {f.value}")
await report_dashboard_synchro(client.auth.installation["id"], cached_sub,
cached_tokens, "ENGINE-CACHE", slug)
await report_dashboard_synchro(client.auth.installation["id"], db_sub,
db_tokens, "DASHBOARD", slug)
await report_worker_status(owner)
installation = context.Installation(client.auth.owner_id, owner,
cached_sub, client, redis_cache)
if repo is not None:
repo_info: github_types.GitHubRepository = await client.item(
f"/repos/{owner}/{repo}")
repository = context.Repository(installation, repo_info["name"],
repo_info["id"])
print(
f"* REPOSITORY IS {'PRIVATE' if repo_info['private'] else 'PUBLIC'}"
)
print("* CONFIGURATION:")
mergify_config = None
config_file = await repository.get_mergify_config_file()
if config_file is None:
print(".mergify.yml is missing")
else:
print(f"Config filename: {config_file['path']}")
print(config_file["decoded_content"].decode())
try:
mergify_config = rules.get_mergify_config(config_file)
except rules.InvalidRules as e: # pragma: no cover
print(f"configuration is invalid {str(e)}")
else:
mergify_config["pull_request_rules"].rules.extend(
engine.MERGIFY_BUILTIN_CONFIG["pull_request_rules"].rules)
if pull_number is None:
async for branch in typing.cast(
typing.AsyncGenerator[github_types.GitHubBranch, None],
client.items(f"/repos/{owner}/{repo}/branches"),
):
# TODO(sileht): Add some informations on the train
q: queue.QueueBase = naive.Queue(repository, branch["name"])
await report_queue("QUEUES", q)
q = merge_train.Train(repository, branch["name"])
await q.load()
await report_queue("TRAIN", q)
else:
repository = context.Repository(
installation, github_types.GitHubRepositoryName(repo))
ctxt = await repository.get_pull_request_context(
github_types.GitHubPullRequestNumber(int(pull_number)))
# FIXME queues could also be printed if no pull number given
# TODO(sileht): display train if any
q = await naive.Queue.from_context(ctxt)
print(
f"* QUEUES: {', '.join([f'#{p}' for p in await q.get_pulls()])}"
)
q = await merge_train.Train.from_context(ctxt)
print(
f"* TRAIN: {', '.join([f'#{p}' for p in await q.get_pulls()])}"
)
print("* PULL REQUEST:")
pr_data = await ctxt.pull_request.items()
pprint.pprint(pr_data, width=160)
is_behind = await ctxt.is_behind
print(f"is_behind: {is_behind}")
print(f"mergeable_state: {ctxt.pull['mergeable_state']}")
print("* MERGIFY LAST CHECKS:")
for c in await ctxt.pull_engine_check_runs:
print(
f"[{c['name']}]: {c['conclusion']} | {c['output'].get('title')}"
)
print("> " + "\n> ".join(
("No Summary", ) if c["output"]["summary"] is None else
c["output"]["summary"].split("\n")))
if mergify_config is not None:
print("* MERGIFY LIVE MATCHES:")
match = await mergify_config["pull_request_rules"
].get_pull_request_rule(ctxt)
summary_title, summary = await actions_runner.gen_summary(
ctxt, match)
print(f"[Summary]: success | {summary_title}")
print("> " + "\n> ".join(summary.strip().split("\n")))
return ctxt
return client
async def test_train_queue_splitted_on_failure_5x3(
report_failure: mock.Mock,
repository: context.Repository,
context_getter: conftest.ContextGetterFixture,
fake_client: mock.Mock,
) -> None:
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
for i in range(41, 47):
await t.add_pull(await context_getter(i), get_config("5x3", 1000))
for i in range(7, 22):
await t.add_pull(await context_getter(i), get_config("5x3", 1000))
await t.refresh()
assert [
[41, 42, 43],
[41, 42, 43, 44, 45, 46],
[41, 42, 43, 44, 45, 46, 7, 8, 9],
[41, 42, 43, 44, 45, 46, 7, 8, 9, 10, 11, 12],
[41, 42, 43, 44, 45, 46, 7, 8, 9, 10, 11, 12, 13, 14, 15],
] == get_cars_content(t)
assert list(range(16, 22)) == get_waiting_content(t)
t._cars[0].checks_conclusion = check_api.Conclusion.FAILURE
await t.save()
assert [
[41, 42, 43],
[41, 42, 43, 44, 45, 46],
[41, 42, 43, 44, 45, 46, 7, 8, 9],
[41, 42, 43, 44, 45, 46, 7, 8, 9, 10, 11, 12],
[41, 42, 43, 44, 45, 46, 7, 8, 9, 10, 11, 12, 13, 14, 15],
] == get_cars_content(t)
assert list(range(16, 22)) == get_waiting_content(t)
await t.load()
await t.refresh()
assert [
[41],
[41, 42],
[41, 42, 43],
] == get_cars_content(t)
assert [44, 45, 46] + list(range(7, 22)) == get_waiting_content(t)
assert len(t._cars[0].failure_history) == 1
assert len(t._cars[1].failure_history) == 1
assert len(t._cars[2].failure_history) == 0
# mark [41] as failed
t._cars[0].checks_conclusion = check_api.Conclusion.FAILURE
await t.save()
await t.remove_pull(await context_getter(41, merged=False))
# nothing should move yet as we don't known yet if [41+42] is broken or not
await t.refresh()
assert [
[42, 43, 44],
[42, 43, 44, 45, 46, 7],
[42, 43, 44, 45, 46, 7, 8, 9, 10],
[42, 43, 44, 45, 46, 7, 8, 9, 10, 11, 12, 13],
[42, 43, 44, 45, 46, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16],
] == get_cars_content(t)
assert list(range(17, 22)) == get_waiting_content(t)
assert len(t._cars[0].failure_history) == 0
assert len(t._cars[1].failure_history) == 0
assert len(t._cars[2].failure_history) == 0
assert len(t._cars[3].failure_history) == 0
assert len(t._cars[4].failure_history) == 0
# mark [42+43+44] as ready and merge it
t._cars[0].checks_conclusion = check_api.Conclusion.SUCCESS
t._cars[1].checks_conclusion = check_api.Conclusion.FAILURE
await t.save()
fake_client.update_base_sha("sha42")
await t.remove_pull(await context_getter(42, merged=True, merge_commit_sha="sha42"))
fake_client.update_base_sha("sha43")
await t.remove_pull(await context_getter(43, merged=True, merge_commit_sha="sha43"))
fake_client.update_base_sha("sha44")
await t.remove_pull(await context_getter(44, merged=True, merge_commit_sha="sha44"))
await t.refresh()
assert [
[42, 43, 44, 45],
[42, 43, 44, 45, 46],
[42, 43, 44, 45, 46, 7],
] == get_cars_content(t)
assert list(range(8, 22)) == get_waiting_content(t)
assert len(t._cars[0].failure_history) == 1
assert len(t._cars[1].failure_history) == 1
assert len(t._cars[2].failure_history) == 0
# mark [45] and [46+46] as success, so it's 7 fault !
t._cars[0].checks_conclusion = check_api.Conclusion.SUCCESS
t._cars[1].checks_conclusion = check_api.Conclusion.SUCCESS
await t.save()
# Nothing change yet!
await t.refresh()
assert [
[42, 43, 44, 45],
[42, 43, 44, 45, 46],
[42, 43, 44, 45, 46, 7],
] == get_cars_content(t)
assert list(range(8, 22)) == get_waiting_content(t)
assert len(t._cars[0].failure_history) == 1
assert len(t._cars[1].failure_history) == 1
assert len(t._cars[2].failure_history) == 0
# Merge 45 and 46
fake_client.update_base_sha("sha45")
await t.remove_pull(await context_getter(45, merged=True, merge_commit_sha="sha45"))
fake_client.update_base_sha("sha46")
await t.remove_pull(await context_getter(46, merged=True, merge_commit_sha="sha46"))
await t.refresh()
assert [
[42, 43, 44, 45, 46, 7],
] == get_cars_content(t)
assert t._cars[0].checks_conclusion == check_api.Conclusion.FAILURE
assert len(t._cars[0].failure_history) == 0
# remove the failed 7
await t.remove_pull(await context_getter(7, merged=False))
# Train got cut after 43, and we restart from the begining
await t.refresh()
assert [
[8, 9, 10],
[8, 9, 10, 11, 12, 13],
[8, 9, 10, 11, 12, 13, 14, 15, 16],
[8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19],
[8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21],
] == get_cars_content(t)
assert [] == get_waiting_content(t)
async def test_train_queue_splitted_on_failure_2x5(
report_failure: mock.Mock,
repository: context.Repository,
fake_client: mock.Mock,
context_getter: conftest.ContextGetterFixture,
) -> None:
t = merge_train.Train(repository, github_types.GitHubRefType("main"))
await t.load()
for i in range(41, 46):
await t.add_pull(await context_getter(i), get_config("2x5", 1000))
for i in range(6, 20):
await t.add_pull(await context_getter(i), get_config("2x5", 1000))
await t.refresh()
assert [
[41, 42, 43, 44, 45],
[41, 42, 43, 44, 45, 6, 7, 8, 9, 10],
] == get_cars_content(t)
assert list(range(11, 20)) == get_waiting_content(t)
t._cars[0].checks_conclusion = check_api.Conclusion.FAILURE
await t.save()
assert [
[41, 42, 43, 44, 45],
[41, 42, 43, 44, 45, 6, 7, 8, 9, 10],
] == get_cars_content(t)
assert list(range(11, 20)) == get_waiting_content(t)
await t.load()
await t.refresh()
assert [
[41, 42],
[41, 42, 43, 44],
[41, 42, 43, 44, 45],
] == get_cars_content(t)
assert list(range(6, 20)) == get_waiting_content(t)
assert len(t._cars[0].failure_history) == 1
assert len(t._cars[1].failure_history) == 1
assert len(t._cars[2].failure_history) == 0
assert t._cars[0].creation_state == "created"
assert t._cars[1].creation_state == "created"
assert t._cars[2].creation_state == "created"
# mark [43+44] as failed
t._cars[1].checks_conclusion = check_api.Conclusion.FAILURE
await t.save()
# nothing should move yet as we don't known yet if [41+42] is broken or not
await t.refresh()
assert [
[41, 42],
[41, 42, 43, 44],
[41, 42, 43, 44, 45],
] == get_cars_content(t)
assert list(range(6, 20)) == get_waiting_content(t)
assert len(t._cars[0].failure_history) == 1
assert len(t._cars[1].failure_history) == 1
assert len(t._cars[2].failure_history) == 0
assert t._cars[0].creation_state == "created"
assert t._cars[1].creation_state == "created"
assert t._cars[2].creation_state == "created"
# mark [41+42] as ready and merge it
t._cars[0].checks_conclusion = check_api.Conclusion.SUCCESS
await t.save()
fake_client.update_base_sha("sha41")
await t.remove_pull(await context_getter(41, merged=True, merge_commit_sha="sha41"))
fake_client.update_base_sha("sha42")
await t.remove_pull(await context_getter(42, merged=True, merge_commit_sha="sha42"))
# [43+44] fail, so it's not 45, but is it 43 or 44?
await t.refresh()
assert [
[41, 42, 43],
[41, 42, 43, 44],
] == get_cars_content(t)
assert [45] + list(range(6, 20)) == get_waiting_content(t)
assert len(t._cars[0].failure_history) == 2
assert len(t._cars[1].failure_history) == 1
assert t._cars[0].creation_state == "created"
assert t._cars[1].creation_state == "created"
# mark [43] as failure
t._cars[0].checks_conclusion = check_api.Conclusion.FAILURE
await t.save()
await t.remove_pull(await context_getter(43, merged=False))
# Train got cut after 43, and we restart from the begining
await t.refresh()
assert [
[44, 45, 6, 7, 8],
[44, 45, 6, 7, 8, 9, 10, 11, 12, 13],
] == get_cars_content(t)
assert list(range(14, 20)) == get_waiting_content(t)
assert len(t._cars[0].failure_history) == 0
assert len(t._cars[1].failure_history) == 0
assert t._cars[0].creation_state == "created"
assert t._cars[1].creation_state == "created"
