def test_land_commit(env, git_gecko, git_wpt, git_wpt_upstream, pull_request, set_pr_status,
hg_gecko_try, mock_mach, mock_tasks):
pr = pull_request([(b"Test commit", {"README": b"example_change"})])
head_rev = pr._commits[0]["sha"]
trypush.Mach = mock_mach
downstream.new_wpt_pr(git_gecko, git_wpt, pr)
downstream_sync = set_pr_status(pr.number, "success")
git_wpt_upstream.head.commit = head_rev
git_wpt.remotes.origin.fetch()
landing.wpt_push(git_gecko, git_wpt, [head_rev], create_missing=False)
with SyncLock.for_process(downstream_sync.process_name) as downstream_lock:
with downstream_sync.as_mut(downstream_lock):
downstream_sync.data["force-metadata-ready"] = True
tree.is_open = lambda x: True
sync = landing.update_landing(git_gecko, git_wpt)
assert ("Setting bug %s add_blocks %s" % (sync.bug, downstream_sync.bug)
in env.bz.output.getvalue())
try_push = sync.latest_try_push
with SyncLock.for_process(sync.process_name) as lock:
with sync.as_mut(lock), try_push.as_mut(lock):
try_push.taskgroup_id = "abcdef"
with patch.object(try_push, "download_logs", Mock(return_value=[])):
with patch.object(tc.TaskGroup, "tasks",
property(Mock(return_value=mock_tasks(completed=["foo"])))):
landing.try_push_complete(git_gecko, git_wpt, try_push, sync)
assert "Pushed to try (stability)" in env.bz.output.getvalue()
assert try_push.status == "complete"
assert sync.status == "open"
try_push = sync.latest_try_push
with SyncLock.for_process(sync.process_name) as lock:
with sync.as_mut(lock), try_push.as_mut(lock):
try_push.taskgroup_id = "abcdef2"
with patch.object(try_push, "download_logs", Mock(return_value=[])):
with patch.object(tc.TaskGroup, "tasks",
property(Mock(return_value=mock_tasks(completed=["foo"])))):
landing.try_push_complete(git_gecko, git_wpt, try_push, sync)
new_head = git_gecko.remotes.mozilla.refs["bookmarks/mozilla/autoland"].commit
assert "Update web-platform-tests to %s" % head_rev in new_head.message
assert (new_head.tree["testing/web-platform/tests/README"].data_stream.read() ==
b"example_change")
sync_point = landing.load_sync_point(git_gecko, git_wpt)
assert sync_point["upstream"] == head_rev
# Update central to contain the landing
git_gecko.refs["mozilla/bookmarks/mozilla/central"].commit = new_head
with patch("sync.landing.start_next_landing") as start_next_landing:
landing.gecko_push(git_gecko, git_wpt, "mozilla-central",
git_gecko.cinnabar.git2hg(new_head))
assert start_next_landing.call_count == 1
assert sync.status == "complete"
def test_land_pr_after_status_change(env, git_gecko, git_wpt,
hg_gecko_upstream, upstream_gecko_commit):
bug = "1234"
test_changes = {"README": "Change README\n"}
rev = upstream_gecko_commit(test_changes=test_changes,
bug=bug,
message="Change README")
update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev)
pushed, landed, failed = upstream.gecko_push(git_gecko,
git_wpt,
"autoland",
rev,
raise_on_error=True)
syncs = upstream.UpstreamSync.for_bug(git_gecko, git_wpt, bug)
assert syncs.keys() == ["open"]
assert len(syncs["open"]) == 1
sync = syncs["open"].pop()
env.gh_wpt.get_pull(sync.pr).mergeable = True
env.gh_wpt.set_status(sync.pr, "failure", "http://test/", "tests failed",
"continuous-integration/travis-ci/pr")
with SyncLock("upstream", None) as lock:
with sync.as_mut(lock):
upstream.commit_status_changed(
git_gecko, git_wpt, sync,
"continuous-integration/travis-ci/pr", "failure",
"http://test/", sync.wpt_commits.head.sha1)
assert sync.last_pr_check == {
"state": "failure",
"sha": sync.wpt_commits.head.sha1
}
hg_gecko_upstream.bookmark("mozilla/central", "-r", rev)
update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev)
pushed, landed, failed = upstream.gecko_push(git_gecko,
git_wpt,
"mozilla-central",
rev,
raise_on_error=True)
env.gh_wpt.set_status(sync.pr, "success", "http://test/", "tests failed",
"continuous-integration/travis-ci/pr")
with SyncLock("upstream", None) as lock:
with sync.as_mut(lock):
upstream.commit_status_changed(
git_gecko, git_wpt, sync,
"continuous-integration/travis-ci/pr", "success",
"http://test/", sync.wpt_commits.head.sha1)
assert sync.last_pr_check == {
"state": "success",
"sha": sync.wpt_commits.head.sha1
}
assert sync.gecko_landed()
assert sync.status == "wpt-merged"
def test_land_pr_after_status_change(env, git_gecko, git_wpt,
hg_gecko_upstream, upstream_gecko_commit,
set_pr_status):
bug = 1234
test_changes = {"README": b"Change README\n"}
rev = upstream_gecko_commit(test_changes=test_changes,
bug=bug,
message=b"Change README")
update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev)
pushed, landed, failed = upstream.gecko_push(git_gecko,
git_wpt,
"autoland",
rev,
raise_on_error=True)
syncs = upstream.UpstreamSync.for_bug(git_gecko, git_wpt, bug)
assert list(syncs.keys()) == ["open"]
assert len(syncs["open"]) == 1
sync = syncs["open"].pop()
env.gh_wpt.get_pull(sync.pr).mergeable = True
set_pr_status(sync.pr, "failure")
with SyncLock("upstream", None) as lock:
with sync.as_mut(lock):
upstream.commit_check_changed(git_gecko, git_wpt, sync)
pr = env.gh_wpt.get_pull(sync.pr)
assert sync.last_pr_check == {
"state": "failure",
"sha": pr._commits[-1].sha
}
hg_gecko_upstream.bookmark("mozilla/central", "-r", rev)
update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev)
pushed, landed, failed = upstream.gecko_push(git_gecko,
git_wpt,
"mozilla-central",
rev,
raise_on_error=True)
set_pr_status(sync.pr, "success")
with SyncLock("upstream", None) as lock:
with sync.as_mut(lock):
upstream.commit_check_changed(git_gecko, git_wpt, sync)
assert sync.last_pr_check == {
"state": "success",
"sha": pr._commits[-1].sha
}
assert sync.gecko_landed()
assert sync.status == "wpt-merged"
def test_land_try(env, git_gecko, git_wpt, git_wpt_upstream, pull_request,
set_pr_status, hg_gecko_try, mock_mach):
pr = pull_request([(b"Test commit", {
"README": b"example_change",
"resources/testdriver_vendor.js": b"Some change"
})])
head_rev = pr._commits[0]["sha"]
trypush.Mach = mock_mach
downstream.new_wpt_pr(git_gecko, git_wpt, pr)
sync = set_pr_status(pr.number, "success")
git_wpt_upstream.head.commit = head_rev
git_wpt.remotes.origin.fetch()
landing.wpt_push(git_gecko, git_wpt, [head_rev], create_missing=False)
with SyncLock.for_process(sync.process_name) as downstream_lock:
with sync.as_mut(downstream_lock):
sync.data["force-metadata-ready"] = True
tree.is_open = lambda x: True
landing_sync = landing.update_landing(git_gecko, git_wpt)
assert landing_sync is not None
with SyncLock("landing", None) as lock:
with landing_sync.as_mut(lock):
worktree = landing_sync.gecko_worktree.get()
# Check that files we shouldn't move aren't
assert not os.path.exists(
os.path.join(worktree.working_dir,
env.config["gecko"]["path"]["wpt"], ".git"))
with open(
os.path.join(worktree.working_dir,
env.config["gecko"]["path"]["wpt"],
"resources/testdriver_vendor.js"), "rb") as f:
assert f.read() == b"Initial testdriver_vendor\n"
try_push = sync.latest_try_push
assert try_push is None
mach_command = mock_mach.get_log()[-1]
assert mach_command["command"] == "mach"
assert mach_command["args"] == ("try", "fuzzy", "-q",
"web-platform-tests !ccov !shippable",
"-q",
"web-platform-tests linux-32 shippable",
"-q",
"web-platform-tests mac !debug shippable",
"--disable-target-task-filter",
"--artifact")
def test_relanding_unchanged_upstreamed_pr(env, git_gecko, git_wpt,
hg_gecko_upstream, pull_request,
upstream_gecko_commit, mock_mach,
set_pr_status, git_wpt_upstream):
trypush.Mach = mock_mach
# Create an unrelated PR that didn't come from Gecko
pr0 = pull_request([(b"Non Gecko PR", {"SOMEFILE": b"Made changes"})])
unrelated_rev = pr0._commits[0]["sha"]
downstream.new_wpt_pr(git_gecko, git_wpt, pr0)
downstream_sync = set_pr_status(pr0.number, 'success')
git_wpt_upstream.head.commit = unrelated_rev
git_wpt.remotes.origin.fetch()
sync = create_and_upstream_gecko_bug(env, git_gecko, git_wpt,
hg_gecko_upstream,
upstream_gecko_commit)
# 'Merge' this upstream PR
pr = env.gh_wpt.get_pull(sync.pr)
pr["user"]["login"] = "******"
with SyncLock.for_process(sync.process_name) as upstream_sync_lock:
with sync.as_mut(upstream_sync_lock):
sync.push_commits()
assert str(git_wpt_upstream.active_branch) == "master"
git_wpt_upstream.git.merge('gecko/1234') # TODO avoid hardcoding?
# Create a ref on the upstream to simulate the pr than GH would setup
git_wpt_upstream.create_head(
'pr/%d' % pr['number'],
commit=git_wpt_upstream.refs['gecko/1234'].commit.hexsha)
git_wpt.remotes.origin.fetch()
pr['merge_commit_sha'] = str(git_wpt_upstream.active_branch.commit.hexsha)
pr['base'] = {'sha': unrelated_rev}
env.gh_wpt.commit_prs[pr['merge_commit_sha']] = pr['number']
# Update landing, the Non Gecko PR should be applied but not the Gecko one we upstreamed
def mock_create(repo,
msg,
metadata,
author=None,
amend=False,
allow_empty=False):
# This commit should not be making it this far, should've been dropped earlier
assert b'Bug 1234 [wpt PR 2] - [Gecko Bug 1234]' not in msg
return DEFAULT
m = Mock(side_effect=mock_create, wraps=sync_commit.Commit.create)
with patch('sync.commit.Commit.create', m):
landing_sync = landing.update_landing(git_gecko,
git_wpt,
include_incomplete=True)
commits = landing_sync.gecko_commits._commits
# Check that the upstreamed gecko PR didnt get pushed to try
assert not any([c.bug == sync.bug for c in commits])
# Check that our unrelated PR got pushed to try
assert any([c.bug == downstream_sync.bug for c in commits])
def setup_repo(env, git_wpt, git_gecko, hg_gecko_upstream, upstream_gecko_commit):
bug = "1234"
changes = {"README": "Changes to README\n"}
upstream_gecko_commit(test_changes=changes, bug=bug,
message="Change README")
test_changes = {"CONFIG": "Change CONFIG\n"}
rev = upstream_gecko_commit(test_changes=test_changes, bug=bug,
message="Change CONFIG")
update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev)
upstream.gecko_push(git_gecko, git_wpt, "autoland", rev, raise_on_error=True)
syncs = upstream.UpstreamSync.for_bug(git_gecko, git_wpt, bug)
sync = syncs["open"].pop()
env.gh_wpt.get_pull(sync.pr).mergeable = True
hg_gecko_upstream.bookmark("mozilla/central", "-r", rev)
update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev)
pushed, landed, failed = upstream.gecko_push(git_gecko,
git_wpt,
"central",
rev,
raise_on_error=True)
assert len(pushed) == 0
assert len(landed) == 1
assert len(failed) == 0
# Push the commits to upstream wpt
with SyncLock.for_process(sync.process_name) as upstream_sync_lock:
with sync.as_mut(upstream_sync_lock):
sync.push_commits()
return list(landed)[0]
def test_revert_pr(env, git_gecko, git_wpt, git_wpt_upstream, pull_request,
pull_request_fn, set_pr_status, wpt_worktree, mock_mach):
pr = pull_request([("Test commit", {
"README": "Example change\n"
})], "Test PR")
downstream.new_wpt_pr(git_gecko, git_wpt, pr)
sync = load.get_pr_sync(git_gecko, git_wpt, pr["number"])
with SyncLock.for_process(sync.process_name) as lock:
with sync.as_mut(lock):
commit = sync.wpt_commits[0]
sync.wpt_commits.base = sync.data[
"wpt-base"] = git_wpt_upstream.head.commit.hexsha
git_wpt_upstream.git.merge(commit.sha1)
def revert_fn():
git_wpt.remotes["origin"].fetch()
wpt_work = wpt_worktree()
wpt_work.git.revert(commit.sha1, no_edit=True)
wpt_work.git.push("origin", "HEAD:refs/heads/revert")
git_wpt_upstream.commit("revert")
return "revert"
pr_revert = pull_request_fn(revert_fn, title="Revert Test PR")
downstream.new_wpt_pr(git_gecko, git_wpt, pr_revert)
sync_revert = load.get_pr_sync(git_gecko, git_wpt, pr_revert["number"])
# Refresh the instance data
sync.data._load()
assert sync.skip
assert sync_revert.skip
def test_dependent_commit(env, git_gecko, git_wpt, pull_request,
upstream_wpt_commit, pull_request_commit):
upstream_wpt_commit("First change", {"README": "Example change\n"})
pr = pull_request([("Test change", {
"README": "Example change 1\n"
})], "Test PR")
downstream.new_wpt_pr(git_gecko, git_wpt, pr)
sync = load.get_pr_sync(git_gecko, git_wpt, pr["number"])
with SyncLock.for_process(sync.process_name) as lock:
with sync.as_mut(lock):
assert len(sync.gecko_commits) == 2
assert sync.gecko_commits[0].msg.splitlines()[0] == "First change"
assert sync.gecko_commits[0].metadata["wpt-type"] == "dependency"
assert sync.gecko_commits[1].metadata.get("wpt-type") is None
assert "Test change" in sync.gecko_commits[1].msg.splitlines()[0]
old_gecko_commits = sync.gecko_commits[:]
# Check that rerunning doesn't affect anything
sync.update_commits()
assert ([item.sha1 for item in sync.gecko_commits
] == [item.sha1 for item in old_gecko_commits])
head_sha = pull_request_commit(pr.number,
[("fixup! Test change", {
"README": "Example change 2\n"
})])
downstream.update_repositories(git_gecko, git_wpt)
sync.update_commits()
assert len(sync.gecko_commits) == 3
assert ([item.sha1 for item in sync.gecko_commits[:2]
] == [item.sha1 for item in old_gecko_commits])
assert sync.gecko_commits[-1].metadata["wpt-commit"] == head_sha
def test_metadata_update(env, git_gecko, git_wpt, pull_request, pull_request_commit):
from conftest import gecko_changes, git_commit
pr = pull_request([("Test commit", {"README": "Example change\n"})],
"Test PR")
downstream.new_wpt_pr(git_gecko, git_wpt, pr)
sync = load.get_pr_sync(git_gecko, git_wpt, pr["number"])
assert len(sync.gecko_commits) == 1
with SyncLock.for_process(sync.process_name) as lock:
with sync.as_mut(lock):
gecko_work = sync.gecko_worktree.get()
changes = gecko_changes(env, meta_changes={"example.ini": "Example change"})
git_commit(gecko_work, """Update metadata
wpt-pr: %s
wpt-type: metadata
""" % pr.number, changes)
assert len(sync.gecko_commits) == 2
assert sync.gecko_commits[-1].metadata.get("wpt-type") == "metadata"
metadata_commit = sync.gecko_commits[-1]
head_sha = pull_request_commit(pr.number,
[("fixup! Test commit",
{"README": "Example change 1\n"})])
downstream.update_repositories(git_gecko, git_wpt)
sync.update_commits()
assert len(sync.gecko_commits) == 3
assert sync.gecko_commits[-1].metadata.get("wpt-type") == "metadata"
assert sync.gecko_commits[-1].msg == metadata_commit.msg
assert sync.gecko_commits[-2].metadata.get("wpt-commit") == head_sha
def test_pr_commits_fast_forward(env, git_wpt, git_gecko, git_wpt_upstream,
hg_gecko_upstream, upstream_gecko_commit):
sync = setup_repo(env, git_wpt, git_gecko, hg_gecko_upstream,
upstream_gecko_commit)
base = git_wpt_upstream.head.commit.hexsha
pr = env.gh_wpt.get_pull(sync.pr)
# Create a ref on the upstream to simulate the pr than GH would setup
pr_head_commit = git_wpt_upstream.refs['gecko/1234'].commit.hexsha
git_wpt_upstream.create_head('pr/%d' % pr['number'], commit=pr_head_commit)
# Fast forward merge our Sync PR
git_wpt_upstream.git.merge('gecko/1234')
git_wpt_upstream.head.commit = pr_head_commit
pr['merge_commit_sha'] = pr_head_commit
pr['base'] = {'sha': base}
git_wpt.remotes.origin.fetch()
with SyncLock.for_process(sync.process_name) as upstream_sync_lock:
with sync.as_mut(upstream_sync_lock):
upstream.update_pr(git_gecko, git_wpt, sync, "closed",
pr['merge_commit_sha'], pr['base']['sha'],
"test")
pr_commits = sync.pr_commits
for wpt_commit, pr_commit in zip(sync.wpt_commits._commits, pr_commits):
assert wpt_commit.commit == pr_commit.commit
def test_landing_pr_on_central(env, git_gecko, git_wpt, git_wpt_upstream, pull_request,
set_pr_status, upstream_gecko_commit, mock_mach):
# Ensure we handle the case where a PR already identical changes on central
trypush.Mach = mock_mach
test_changes = {"example/test.html": b"Change test\n"}
more_changes = {"LICENSE": b"Change license\n"}
for changes in [test_changes, more_changes]:
pr = pull_request([(b"Test commit", changes)])
downstream.new_wpt_pr(git_gecko, git_wpt, pr)
downstream_sync = set_pr_status(pr.number, "success")
head_rev = pr._commits[0]["sha"]
git_wpt_upstream.head.commit = head_rev
git_wpt.remotes.origin.fetch()
landing.wpt_push(git_gecko, git_wpt, [head_rev], create_missing=False)
with SyncLock.for_process(downstream_sync.process_name) as downstream_lock:
with downstream_sync.as_mut(downstream_lock):
downstream_sync.data["force-metadata-ready"] = True
# Repeat the changes in m-c
upstream_gecko_commit(test_changes=test_changes, bug=1234,
message=b"Change README")
tree.is_open = lambda x: True
sync = landing.update_landing(git_gecko, git_wpt)
assert len(sync.wpt_commits) == 2
assert len(sync.gecko_commits) == 3
assert sync.gecko_commits[0].is_empty()
assert sync.gecko_commits[-1].is_landing
def test_pr_commits_merge(env, git_wpt, git_gecko, git_wpt_upstream,
hg_gecko_upstream, upstream_gecko_commit):
sync = setup_repo(env, git_wpt, git_gecko, hg_gecko_upstream,
upstream_gecko_commit)
# Make changes on master
git_wpt_upstream.branches.master.checkout()
base = git_wpt_upstream.head.commit.hexsha
git_commit(git_wpt_upstream, "Some other Commit",
{"RANDOME_FILE": "Changes to this\n"})
pr = env.gh_wpt.get_pull(sync.pr)
# Create a ref on the upstream to simulate the pr than GH would setup
git_wpt_upstream.create_head(
'pr/%d' % pr['number'],
commit=git_wpt_upstream.refs['gecko/1234'].commit.hexsha)
# Merge our Sync PR
git_wpt_upstream.git.merge('gecko/1234')
pr['merge_commit_sha'] = str(git_wpt_upstream.active_branch.commit.hexsha)
pr['base'] = {'sha': base}
git_wpt.remotes.origin.fetch()
with SyncLock.for_process(sync.process_name) as upstream_sync_lock:
with sync.as_mut(upstream_sync_lock):
upstream.update_pr(git_gecko, git_wpt, sync, "closed",
pr['merge_commit_sha'], pr['base']['sha'],
"test")
pr_commits = sync.pr_commits
for wpt_commit, pr_commit in zip(sync.wpt_commits._commits, pr_commits):
assert wpt_commit.commit == pr_commit.commit
def try_push(env, git_gecko, git_wpt, git_wpt_upstream, pull_request, set_pr_status,
hg_gecko_try, mock_mach):
pr = pull_request([(b"Test commit", {"README": b"example_change",
"LICENSE": b"Some change"})])
head_rev = pr._commits[0]["sha"]
trypush.Mach = mock_mach
with patch("sync.tree.is_open", Mock(return_value=True)):
downstream.new_wpt_pr(git_gecko, git_wpt, pr)
sync = set_pr_status(pr.number, "success")
with SyncLock.for_process(sync.process_name) as sync_lock:
git_wpt_upstream.head.commit = head_rev
git_wpt.remotes.origin.fetch()
landing.wpt_push(git_gecko, git_wpt, [head_rev], create_missing=False)
with sync.as_mut(sync_lock):
env.gh_wpt.get_pull(sync.pr).merged = True
sync.data["affected-tests"] = {"Example": "affected"}
sync.next_try_push()
sync.data["force-metadata-ready"] = True
try_push = sync.latest_try_push
with try_push.as_mut(sync_lock):
try_push.taskgroup_id = "abcdef"
return sync.latest_try_push
def test_try_task_states_all_success(mock_tasks, try_push):
tasks = Mock(return_value=mock_tasks(completed=["foo", "bar"] * 5))
with SyncLock.for_process(try_push.process_name) as lock:
with try_push.as_mut(lock):
with patch.object(tc.TaskGroup, "tasks", property(tasks)):
tasks = try_push.tasks()
assert tasks.success()
assert tasks.success_rate() == 1.0
def test_github_label_on_error(env, git_gecko, git_wpt, pull_request):
pr = pull_request([("Testing", {"README": "Example change\n"})], "Test PR")
downstream.new_wpt_pr(git_gecko, git_wpt, pr)
sync = load.get_pr_sync(git_gecko, git_wpt, pr["number"])
with SyncLock.for_process(sync.process_name) as lock:
with sync.as_mut(lock):
sync.error = "Infrastructure Failed"
assert env.gh_wpt.get_pull(
pr["number"])["labels"] == ["mozilla:gecko-blocked"]
with SyncLock.for_process(sync.process_name) as lock:
with sync.as_mut(lock):
sync.update_commits()
assert env.gh_wpt.get_pull(pr["number"])["labels"] == []
def test_processname_seq_id(git_gecko, local_gecko_commit):
process_name_no_seq_id = base.ProcessName("sync", "upstream", "1234", "0")
with SyncLock("upstream", None) as lock:
sync.SyncData.create(lock, git_gecko, process_name_no_seq_id,
{"test": 1})
process_name_seq_id = base.ProcessName.with_seq_id(git_gecko, "sync",
"upstream", "1234")
assert process_name_seq_id.seq_id == 1
def test_accept_failures(landing_with_try_push, mock_tasks):
sync = landing_with_try_push
with patch.object(tc.TaskGroup, "tasks",
property(Mock(return_value=mock_tasks(failed=["foo", "foobar"],
completed=["bar"])))):
assert sync.try_result() == landing.TryPushResult.too_many_failures
with SyncLock.for_process(sync.process_name) as lock:
with sync.latest_try_push.as_mut(lock):
sync.latest_try_push.accept_failures = True
assert sync.try_result() == landing.TryPushResult.acceptable_failures
def test_try_push_exceeds_failure_threshold(git_gecko, git_wpt, landing_with_try_push, mock_tasks):
# 2/3 failure rate, too high
sync = landing_with_try_push
try_push = sync.latest_try_push
with patch.object(tc.TaskGroup, "tasks",
property(Mock(return_value=mock_tasks(failed=["foo", "foobar"],
completed=["bar"])))):
with SyncLock.for_process(sync.process_name) as lock:
with sync.as_mut(lock), try_push.as_mut(lock):
with pytest.raises(AbortError):
landing.try_push_complete(git_gecko, git_wpt, try_push, sync)
assert try_push.status == "open"
assert "too many test failures" in sync.error["message"]
def test_ref_duplicate(git_gecko):
def create_initial():
p = base.ProcessName("sync", "upstream", "1", "0")
with SyncLock("upstream", None) as lock:
sync.SyncData.create(lock, git_gecko, p, {"test": 1})
create_initial()
# Ensure that the p object has been gc'd
gc.collect()
q = base.ProcessName("sync", "upstream", "1", "0")
with pytest.raises(ValueError):
with SyncLock("upstream", None) as lock:
sync.SyncData.create(lock, git_gecko, q, {"test": 2})
def test_retrigger_failures(mock_tasks, try_push):
failed = ["foo", "foo", "bar", "baz", "foo-aarch64"]
ex = ["bar", "boo", "bork"]
tasks = Mock(return_value=mock_tasks(
completed=["foo", "bar"] * 5, failed=failed, exception=ex))
retrigger_count = 5
with SyncLock.for_process(try_push.process_name) as lock:
with try_push.as_mut(lock):
with patch.object(tc.TaskGroup, "tasks", property(tasks)):
with patch('sync.trypush.auth_tc.retrigger',
return_value=["job"] * retrigger_count):
tasks = try_push.tasks()
jobs = tasks.retrigger_failures(count=retrigger_count)
assert jobs == retrigger_count * (len(set(failed + ex)) - 1)
def test_next_try_push_infra_fail(env, git_gecko, git_wpt, pull_request,
set_pr_status, MockTryCls, hg_gecko_try,
mock_mach, mock_taskgroup):
taskgroup = mock_taskgroup("taskgroup-complete-build-failed.json")
try_tasks = trypush.TryPushTasks(taskgroup)
pr = pull_request([("Test commit", {
"README": "Example change\n"
})], "Test PR")
downstream.new_wpt_pr(git_gecko, git_wpt, pr)
try_patch = patch("sync.trypush.TryPush.tasks",
Mock(return_value=try_tasks))
tree_open_patch = patch("sync.tree.is_open", Mock(return_value=True))
taskgroup_patch = patch("sync.tc.TaskGroup", Mock(return_value=taskgroup))
mach_patch = patch("sync.trypush.Mach", mock_mach)
with tree_open_patch, try_patch, taskgroup_patch, mach_patch:
sync = set_pr_status(pr, "success")
env.gh_wpt.get_pull(sync.pr).merged = True
with SyncLock.for_process(sync.process_name) as lock:
with sync.as_mut(lock):
assert len(sync.try_pushes()) == 0
sync.data["affected-tests"] = {"testharness": ["example"]}
try_push = sync.next_try_push(try_cls=MockTryCls)
with try_push.as_mut(lock):
try_push["taskgroup-id"] = taskgroup.taskgroup_id
try_push.status = "complete"
try_push.infra_fail = True
# This try push still has completed builds and tests, so we say metadata is ready.
assert sync.next_action == downstream.DownstreamAction.ready
assert sync.next_try_push(try_cls=MockTryCls) is None
# There should be a comment to flag failed builds
msg = "There were infrastructure failures for the Try push (%s):\nbuild-win32/opt\nbuild-win32/debug\nbuild-win64/opt\nbuild-win64/debug\n" % try_push.treeherder_url # noqa: E501
assert msg in env.bz.output.getvalue()
# Replace the taskgroup with one where there were no completed tests
taskgroup = mock_taskgroup(
"taskgroup-no-tests-build-failed.json")
try_tasks = trypush.TryPushTasks(taskgroup)
# The next action should flag for manual fix now
assert sync.next_action == downstream.DownstreamAction.manual_fix
def test_land_pr(env, git_gecko, git_wpt, hg_gecko_upstream,
upstream_gecko_commit):
bug = "1234"
test_changes = {"README": "Change README\n"}
rev = upstream_gecko_commit(test_changes=test_changes,
bug=bug,
message="Change README")
update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev)
pushed, landed, failed = upstream.gecko_push(git_gecko,
git_wpt,
"autoland",
rev,
raise_on_error=True)
syncs = upstream.UpstreamSync.for_bug(git_gecko, git_wpt, bug)
assert syncs.keys() == ["open"]
assert len(syncs["open"]) == 1
sync = syncs["open"].pop()
env.gh_wpt.get_pull(sync.pr).mergeable = True
original_remote_branch = sync.remote_branch
hg_gecko_upstream.bookmark("mozilla/central", "-r", rev)
update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev)
pushed, landed, failed = upstream.gecko_push(git_gecko,
git_wpt,
"mozilla-central",
rev,
raise_on_error=True)
syncs = upstream.UpstreamSync.for_bug(git_gecko, git_wpt, bug)
assert syncs == {"wpt-merged": {sync}}
assert sync.gecko_landed()
assert sync.status == "wpt-merged"
assert original_remote_branch not in git_wpt.remotes.origin.refs
pr = env.gh_wpt.get_pull(sync.pr)
assert pr.merged
with SyncLock.for_process(sync.process_name) as lock:
with sync.as_mut(lock):
upstream.update_pr(git_gecko, git_wpt, sync, "closed",
pr["merge_commit_sha"], '',
pr["merged_by"]["login"])
user = env.config["web-platform-tests"]["github"]["user"]
assert ("Upstream PR merged by %s" %
user) in env.bz.output.getvalue().strip().split('\n')
def test_no_download_logs_after_all_try_tasks_success(git_gecko, git_wpt, landing_with_try_push,
mock_tasks, env):
tasks = Mock(return_value=mock_tasks(completed=["bar", "baz", "boo"]))
sync = landing_with_try_push
try_push = sync.latest_try_push
with SyncLock.for_process(sync.process_name) as lock:
with try_push.as_mut(lock), sync.as_mut(lock):
with patch.object(tc.TaskGroup,
"tasks",
property(tasks)), patch(
'sync.landing.push_to_gecko'):
try_push.download_logs = Mock(return_value=[])
landing.try_push_complete(git_gecko, git_wpt, try_push, sync)
# no intermittents in the try push
try_push.download_logs.assert_not_called()
assert try_push.status == "complete"
def test_delete(env, git_gecko, git_wpt, upstream_gecko_commit):
# Do some stuff to create an example sync
bug = 1234
test_changes = {"README": b"Change README\n"}
rev = upstream_gecko_commit(test_changes=test_changes,
bug=bug,
message=b"Change README")
update_repositories(git_gecko, git_wpt, wait_gecko_commit=rev)
_, _, _ = upstream.gecko_push(git_gecko,
git_wpt,
"autoland",
rev,
raise_on_error=True)
sync = upstream.UpstreamSync.for_bug(git_gecko, git_wpt, bug,
flat=True).pop()
process_name = sync.process_name
sync_path = "/".join(str(item) for item in process_name.as_tuple())
ref = git.Reference(git_gecko, "refs/syncs/data")
assert ref.commit.tree[sync_path]
gecko_ref = git.Reference(git_gecko, "refs/heads/%s" % sync_path)
assert gecko_ref.is_valid()
wpt_ref = git.Reference(git_wpt, "refs/heads/%s" % sync_path)
assert wpt_ref.is_valid()
with SyncLock.for_process(process_name) as lock:
with sync.as_mut(lock):
sync.delete()
for idx_cls in (index.SyncIndex, index.PrIdIndex, index.BugIdIndex):
idx = idx_cls(git_gecko)
assert not idx.get(idx.make_key(sync))
ref = git.Reference(git_gecko, "refs/syncs/data")
with pytest.raises(KeyError):
ref.commit.tree[sync_path]
ref = git.Reference(git_gecko, "refs/heads/%s" % sync_path)
assert not ref.is_valid()
ref = git.Reference(git_wpt, "refs/heads/%s" % sync_path)
assert not ref.is_valid()
def test_landing_metadata(env, git_gecko, git_wpt, git_wpt_upstream,
pull_request, set_pr_status, hg_gecko_try,
mock_mach):
from conftest import create_file_data, gecko_changes
trypush.Mach = mock_mach
pr = pull_request([("Test commit", {
"example/test1.html": "example_change"
})])
head_rev = pr._commits[0]["sha"]
downstream.new_wpt_pr(git_gecko, git_wpt, pr)
downstream_sync = set_pr_status(pr, "success")
# Create a metadata commit
with SyncLock.for_process(downstream_sync.process_name) as downstream_lock:
with downstream_sync.as_mut(downstream_lock):
git_work = downstream_sync.gecko_worktree.get()
changes = gecko_changes(env,
meta_changes={
"example/test1.html":
"[test1.html]\n expected: FAIL"
})
file_data, _ = create_file_data(changes, git_work.working_dir)
downstream_sync.ensure_metadata_commit()
git_work.index.add(file_data)
downstream_sync._commit_metadata()
assert downstream_sync.metadata_commit is not None
downstream_sync.data["force-metadata-ready"] = True
git_wpt_upstream.head.commit = head_rev
git_wpt.remotes.origin.fetch()
landing.wpt_push(git_gecko, git_wpt, [head_rev], create_missing=False)
tree.is_open = lambda x: True
landing_sync = landing.update_landing(git_gecko, git_wpt)
assert len(landing_sync.gecko_commits) == 3
assert landing_sync.gecko_commits[-1].metadata["wpt-type"] == "landing"
assert landing_sync.gecko_commits[-2].metadata["wpt-type"] == "metadata"
for item in file_data:
assert item in landing_sync.gecko_commits[-2].commit.stats.files
def test_create_delete(env, git_gecko, hg_gecko_upstream):
git_gecko.remotes.mozilla.fetch()
process_name = base.ProcessName("sync", "downstream", "1", "0")
wt = worktree.Worktree(git_gecko, process_name)
with SyncLock.for_process(process_name) as lock:
base.BranchRefObject.create(lock, git_gecko, process_name, "FETCH_HEAD")
with wt.as_mut(lock):
wt.get()
assert os.path.exists(wt.path)
# This is a gecko repo so it ought to have created a state directory
state_path = repos.Gecko.get_state_path(env.config, wt.path)
assert os.path.exists(state_path)
wt.delete()
assert not os.path.exists(wt.path)
assert not os.path.exists(state_path)
def test_wpt_pr_approved(env, git_gecko, git_wpt, pull_request, set_pr_status,
hg_gecko_try, mock_wpt, mock_tasks, mock_mach):
mock_wpt.set_data("tests-affected", "")
pr = pull_request([("Test commit", {
"README": "Example change\n"
})], "Test PR")
pr._approved = False
with patch("sync.tree.is_open",
Mock(return_value=True)), patch("sync.trypush.Mach", mock_mach):
downstream.new_wpt_pr(git_gecko, git_wpt, pr)
sync = set_pr_status(pr, "success")
with SyncLock.for_process(sync.process_name) as lock:
with sync.as_mut(lock):
sync.data["affected-tests"] = {"testharness": ["example"]}
assert sync.latest_try_push is None
assert sync.last_pr_check == {"state": "success", "sha": pr.head}
pr._approved = True
# A Try push is not run after approval.
assert sync.latest_try_push is None
# If we 'merge' the PR, then we will see a stability try push
handlers.handle_pr(
git_gecko, git_wpt, {
"action": "closed",
"number": pr.number,
"pull_request": {
"number": pr.number,
"merge_commit_sha": "a" * 25,
"base": {
"sha": "b" * 25
},
"merged": True,
"state": "closed",
"merged_by": {
"login": "******"
}
}
})
try_push = sync.latest_try_push
assert try_push.stability
def test_try_task_states(mock_tasks, try_push):
tasks = Mock(return_value=mock_tasks(completed=["foo", "bar"] * 5,
failed=["foo", "foo", "bar", "baz"]))
with SyncLock.for_process(try_push.process_name) as lock:
with try_push.as_mut(lock):
with patch.object(tc.TaskGroup, "tasks", property(tasks)):
tasks = try_push.tasks()
states = tasks.wpt_states()
assert not tasks.success()
assert set(states.keys()) == {"baz", "foo", "bar"}
assert states["foo"]["states"][tc.SUCCESS] == 5
assert states["foo"]["states"][tc.FAIL] == 2
assert states["bar"]["states"][tc.SUCCESS] == 5
assert states["bar"]["states"][tc.FAIL] == 1
assert states["baz"]["states"][tc.FAIL] == 1
retriggered_states = tasks.retriggered_wpt_states()
assert tasks.success_rate() == float(10) / len(tasks)
# baz is not retriggered, only occurs once
assert retriggered_states.keys() == ["foo", "bar"]
def test_relanding_changed_upstreamed_pr(env, git_gecko, git_wpt,
hg_gecko_upstream,
upstream_gecko_commit, mock_mach,
git_wpt_upstream):
trypush.Mach = mock_mach
sync = create_and_upstream_gecko_bug(env, git_gecko, git_wpt,
hg_gecko_upstream,
upstream_gecko_commit)
# 'Merge' this upstream PR
pr = env.gh_wpt.get_pull(sync.pr)
pr["base"] = {"sha": git_wpt_upstream.head.commit.hexsha}
pr["user"]["login"] = "******"
with SyncLock.for_process(sync.process_name) as upstream_sync_lock:
with sync.as_mut(upstream_sync_lock):
sync.push_commits()
git_wpt_upstream.branches['gecko/1234'].checkout()
extra_commit = git_commit(git_wpt_upstream, "Fixed pr before merge",
{"EXTRA": "This fixes it"})
git_wpt_upstream.branches.master.checkout()
assert str(git_wpt_upstream.active_branch) == "master"
git_wpt_upstream.git.merge('gecko/1234') # TODO avoid hardcoding?
# Create a ref on the upstream to simulate the pr than GH would setup
git_wpt_upstream.create_head(
'pr/%d' % pr['number'],
commit=git_wpt_upstream.refs['gecko/1234'].commit.hexsha)
git_wpt.remotes.origin.fetch()
pr['merge_commit_sha'] = str(git_wpt_upstream.active_branch.commit.hexsha)
env.gh_wpt.commit_prs[pr['merge_commit_sha']] = pr['number']
landing_sync = landing.update_landing(git_gecko,
git_wpt,
include_incomplete=True)
commits = landing_sync.gecko_commits._commits
assert len(commits) == 2
# Check that the first commit is our "fix commit" which didn't come from Gecko
assert commits[0].metadata['wpt-commits'] == extra_commit.hexsha
# Check that the other commit is the bot's push commit
assert commits[1].metadata['MANUAL PUSH'] == "wpt sync bot"
def inner(pr, status="success"):
from sync import load
env.gh_wpt.set_status(pr["number"], status, "http://test/",
"description",
"continuous-integration/travis-ci/pr")
sync = load.get_pr_sync(git_gecko, git_wpt, pr["number"])
with SyncLock.for_process(sync.process_name) as lock:
with sync.as_mut(lock):
with patch("sync.tree.is_open", Mock(return_value=True)):
downstream.commit_status_changed(
git_gecko,
git_wpt,
sync,
"continuous-integration/travis-ci/pr",
status,
"http://test/",
pr["head"],
raise_on_error=True)
return sync