def new_wpt_pr(git_gecko, git_wpt, pr_data, raise_on_error=True, repo_update=True):
""" Start a new downstream sync """
if pr_data["user"]["login"] == env.config["web-platform-tests"]["github"]["user"]:
raise ValueError("Tried to create a downstream sync for a PR created "
"by the wpt bot")
if repo_update:
update_repositories(git_gecko, git_wpt)
pr_id = pr_data["number"]
if DownstreamSync.for_pr(git_gecko, git_wpt, pr_id):
return
wpt_base = "origin/%s" % pr_data["base"]["ref"]
with SyncLock("downstream", str(pr_id)) as lock:
sync = DownstreamSync.new(lock,
git_gecko,
git_wpt,
wpt_base,
pr_id,
pr_data["title"],
pr_data["body"] or "")
with sync.as_mut(lock):
try:
sync.update_commits()
sync.update_github_check()
except Exception as e:
sync.error = e
if raise_on_error:
raise
traceback.print_exc()
logger.error(e)
def push_to_gecko(git_gecko, git_wpt, sync, allow_push=True):
if not allow_push:
logger.info("Landing in bug %s is ready for push.\n"
"Working copy is in %s" %
(sync.bug, sync.gecko_worktree.get().working_dir))
return
update_repositories(git_gecko, git_wpt)
wpt_head, commits = landable_commits(git_gecko,
git_wpt,
sync.wpt_commits.base.sha1,
sync.wpt_commits.head.sha1,
include_incomplete=True)
push(sync)
for _, sync, _ in commits:
if sync is not None:
if isinstance(sync, downstream.DownstreamSync):
# If we can't perform a notification by now it isn't ever going
# to work
try:
if not sync.skip:
sync.try_notify()
except Exception as e:
logger.error(e.message)
finally:
sync.finish()
if not sync.results_notified:
env.bz.comment(
sync.bug, "Result changes from PR not available.")
def do_pr(git_gecko, git_wpt, pr_id, *args, **kwargs):
import update
if pr_id is None:
pr_id = sync_from_path(git_gecko, git_wpt).pr
pr = env.gh_wpt.get_pull(int(pr_id))
update_repositories(git_gecko, git_wpt, True)
update.update_pr(git_gecko, git_wpt, pr, kwargs["rebase"])
def do_retrigger(git_gecko, git_wpt, **kwargs):
import errors
import update
import upstream
from landing import load_sync_point, unlanded_with_type
update_repositories(git_gecko, git_wpt, True)
print("Retriggering upstream syncs with errors")
for sync in upstream.UpstreamSync.load_all(git_gecko,
git_wpt,
status="open"):
if sync.error:
try:
upstream.update_sync(git_gecko, git_wpt, sync)
except errors.AbortError as e:
print("Update failed:\n%s" % e)
pass
print("Retriggering downstream syncs on master")
sync_point = load_sync_point(git_gecko, git_wpt)
prev_wpt_head = sync_point["upstream"]
unlandable = unlanded_with_type(git_gecko, git_wpt, None, prev_wpt_head)
errors = update.retrigger(git_gecko, git_wpt, unlandable)
if errors:
print("The following PRs have errors:\n%s" % "\n".join(errors))
def commit_status_changed(git_gecko,
git_wpt,
sync,
context,
status,
url,
head_sha,
raise_on_error=False):
update_repositories(git_gecko, git_wpt)
if sync.skip:
return
try:
logger.debug("Got status %s for PR %s" % (status, sync.pr))
if status == "pending":
# We got new commits that we missed
sync.update_commits()
return
check_state, _ = env.gh_wpt.get_combined_status(sync.pr)
sync.last_pr_check = {"state": check_state, "sha": head_sha}
if check_state == "success":
sync.next_try_push()
except Exception as e:
sync.error = e
if raise_on_error:
raise
traceback.print_exc()
logger.error(e)
def update_sync(git_gecko,
git_wpt,
sync,
raise_on_error=True,
repo_update=True):
if sync.status in ("wpt-merged", "complete"):
logger.info("Nothing to do for sync with status %s" % sync.status)
return set(), set(), set()
if repo_update:
update_repositories(git_gecko, git_wpt)
assert isinstance(sync, UpstreamSync)
update_syncs = {sync.bug: (sync, sync.gecko_commits.head.sha1)}
pushed_syncs, failed_syncs = update_sync_prs(sync._lock,
git_gecko,
git_wpt, {},
update_syncs,
raise_on_error=raise_on_error)
if sync not in failed_syncs:
landed_syncs = try_land_syncs(sync._lock, [sync])
else:
landed_syncs = set()
return pushed_syncs, failed_syncs, landed_syncs
def do_pr(git_gecko, git_wpt, pr_ids, *args, **kwargs):
import update
if not pr_ids:
pr_ids = [sync_from_path(git_gecko, git_wpt).pr]
for pr_id in pr_ids:
pr = env.gh_wpt.get_pull(int(pr_id))
update_repositories(git_gecko, git_wpt)
update.update_pr(git_gecko, git_wpt, pr, kwargs["rebase"])
def __call__(self, git_gecko, git_wpt):
logger.info("Running retrigger")
update_repositories(git_gecko, git_wpt)
sync_point = landing.load_sync_point(git_gecko, git_wpt)
prev_wpt_head = sync_point["upstream"]
unlanded = landing.unlanded_with_type(git_gecko, git_gecko, None,
prev_wpt_head)
update.retrigger(git_gecko, git_wpt, unlanded)
def __call__(self, git_gecko, git_wpt):
newrelic.agent.set_transaction_name("RetriggerHandler")
logger.info("Running retrigger")
update_repositories(git_gecko, git_wpt)
sync_point = landing.load_sync_point(git_gecko, git_wpt)
prev_wpt_head = sync_point["upstream"]
unlanded = landing.unlanded_with_type(git_gecko, git_gecko, None,
prev_wpt_head)
update.retrigger(git_gecko, git_wpt, unlanded)
def handle_status(git_gecko, git_wpt, event):
newrelic.agent.set_transaction_name("handle_status")
if event["context"] == "upstream/gecko":
# Never handle changes to our own status
return
repo_update = event.get("_wptsync", {}).get("repo_update", True)
if repo_update:
update_repositories(None, git_wpt, False)
rev = event["sha"]
# First check if the PR is head of any pull request
pr_id = pr_for_commit(git_wpt, rev)
newrelic.agent.add_custom_parameter("rev", rev)
newrelic.agent.add_custom_parameter("context", event["context"])
newrelic.agent.add_custom_parameter("state", event["state"])
if not pr_id:
# This usually happens if we got behind, so the commit is no longer the latest one
# There are a few possibilities for what happened:
# * Something new was pushed. In that case ignoring this message is fine
# * The PR got merged in a way that changes the SHAs. In that case we assume that
# the sync will get triggered later like when there's a push for the commit
logger.warning(
"Got status for commit %s which is not the current HEAD of any PR\n"
"context: %s url: %s state: %s" %
(rev, event["context"], event["target_url"], event["state"]))
return
else:
logger.info("Got status for commit %s from PR %s\n"
"context: %s url: %s state: %s" %
(rev, pr_id, event["context"], event["target_url"],
event["state"]))
sync = get_pr_sync(git_gecko, git_wpt, pr_id)
if not sync:
# Presumably this is a thing we ought to be downstreaming, but missed somehow
logger.info(
"Got a status update for PR %s which is unknown to us; starting downstreaming"
% pr_id)
from update import schedule_pr_task
schedule_pr_task("opened", env.gh_wpt.get_pull(pr_id))
update_func = None
if isinstance(sync, upstream.UpstreamSync):
update_func = upstream.commit_status_changed
elif isinstance(sync, downstream.DownstreamSync):
update_func = downstream.commit_status_changed
if update_func is not None:
with SyncLock.for_process(sync.process_name) as lock:
with sync.as_mut(lock):
update_func(git_gecko, git_wpt, sync, event["context"],
event["state"], event["target_url"], event["sha"])
def update_from_github(git_gecko, git_wpt, sync_classes, statuses=None):
if statuses is None:
statuses = ["*"]
update_repositories(git_gecko, git_wpt)
for cls in sync_classes:
for status in statuses:
if status != "*" and status not in cls.statuses:
continue
syncs = cls.load_by_status(git_gecko, git_wpt, status)
for sync in syncs:
if not sync.pr:
continue
logger.info("Updating sync for PR %s" % sync.pr)
pr = env.gh_wpt.get_pull(sync.pr)
update_pr(git_gecko, git_wpt, pr)
def handle_status(git_gecko, git_wpt, event):
if event["context"] == "upstream/gecko":
# Never handle changes to our own status
return
update_repositories(None, git_wpt, False)
rev = event["sha"]
# First check if the PR is head of any pull request
pr_id = pr_for_commit(git_wpt, rev)
if not pr_id:
# This usually happens if we got behind, so the commit is no longer the latest one
# There are a few possibilities for what happened:
# * Something new was pushed. In that case ignoring this message is fine
# * The PR got merged in a way that changes the SHAs. In that case we assume that
# the syncc will get triggered later like when there's a push for the commit
logger.warning(
"Got status for commit %s which is not the current HEAD of any PR\n"
"context: %s url: %s state: %s" %
(rev, event["context"], event["target_url"], event["state"]))
return
else:
logger.info("Got status for commit %s from PR %s\n"
"context: %s url: %s state: %s" %
(rev, pr_id, event["context"], event["target_url"],
event["state"]))
sync = get_pr_sync(git_gecko, git_wpt, pr_id)
if not sync:
# Presumably this is a thing we ought to be downstreaming, but missed somehow
logger.info(
"Got a status update for PR %s which is unknown to us; starting downstreaming"
% pr_id)
from update import schedule_pr_task
schedule_pr_task("opened", env.gh_wpt.get_pull(pr_id))
if isinstance(sync, upstream.UpstreamSync):
upstream.commit_status_changed(git_gecko, git_wpt, sync,
event["context"], event["state"],
event["target_url"], event["sha"])
elif isinstance(sync, downstream.DownstreamSync):
downstream.commit_status_changed(git_gecko, git_wpt, sync,
event["context"], event["state"],
event["target_url"], event["sha"])
def __call__(self, git_gecko, git_wpt, body):
newrelic.agent.set_transaction_name("PushHandler")
repo = body["_meta"]["routing_key"]
if "/" in repo:
repo_name = repo.rsplit("/", 1)[1]
else:
repo_name = repo
# Commands can override the base rev and select only certain processses
base_rev = body.get("_wptsync", {}).get("base_rev")
processes = body.get("_wptsync", {}).get("processes")
# Not sure if it's ever possible to get multiple heads here in a way that
# matters for us
rev = body["payload"]["data"]["heads"][0]
logger.info("Handling commit %s to repo %s" % (rev, repo))
newrelic.agent.add_custom_parameter("repo", repo)
newrelic.agent.add_custom_parameter("rev", rev)
update_repositories(git_gecko,
git_wpt,
include_autoland=True,
wait_gecko_commit=rev)
try:
git_rev = git_gecko.cinnabar.hg2git(rev)
except ValueError:
pass
else:
if gecko_repo(git_gecko, git_rev) is None:
logger.info("Skipping commit as it isn't in a branch we track")
return
if processes is None or "upstream" in processes:
upstream.gecko_push(git_gecko,
git_wpt,
repo_name,
rev,
base_rev=base_rev)
if processes is None or "landing" in processes:
landing.gecko_push(git_gecko,
git_wpt,
repo_name,
rev,
base_rev=base_rev)
def do_retrigger(git_gecko, git_wpt, **kwargs):
import errors
import update
import upstream
from landing import current, load_sync_point, unlanded_with_type
update_repositories(git_gecko, git_wpt)
if kwargs["upstream"]:
print("Retriggering upstream syncs with errors")
for sync in upstream.UpstreamSync.load_by_status(
git_gecko, git_wpt, "open"):
if sync.error:
with SyncLock.for_process(sync.process_name) as lock:
with sync.as_mut(lock):
try:
upstream.update_sync(git_gecko,
git_wpt,
sync,
repo_update=False)
except errors.AbortError as e:
print("Update failed:\n%s" % e)
pass
if kwargs["downstream"]:
print("Retriggering downstream syncs on master")
current_landing = current(git_gecko, git_wpt)
if current_landing is None:
sync_point = load_sync_point(git_gecko, git_wpt)
prev_wpt_head = sync_point["upstream"]
else:
prev_wpt_head = current_landing.wpt_commits.head.sha1
unlandable = unlanded_with_type(git_gecko, git_wpt, None,
prev_wpt_head)
errors = update.retrigger(git_gecko, git_wpt, unlandable)
if errors:
print("The following PRs have errors:\n%s" % "\n".join(errors))
def update_landing(git_gecko,
git_wpt,
prev_wpt_head=None,
new_wpt_head=None,
include_incomplete=False,
retry=False):
"""Create or continue a landing of wpt commits to gecko.
:param prev_wpt_head: The sha1 of the previous wpt commit landed to gecko.
:param wpt_head: The sha1 of the latest possible commit to land to gecko,
or None to use the head of the master branch"
:param include_incomplete: By default we don't attempt to land anything that
hasn't completed a metadata update. This flag disables
that and just lands everything up to the specified commit.
:param retry: Create a new try push for the landing even if there's an existing one"""
landing = current(git_gecko, git_wpt)
sync_point = load_sync_point(git_gecko, git_wpt)
if landing is None:
update_repositories(git_gecko, git_wpt)
if prev_wpt_head is None:
prev_wpt_head = sync_point["upstream"]
landable = landable_commits(git_gecko,
git_wpt,
prev_wpt_head,
wpt_head=new_wpt_head,
include_incomplete=include_incomplete)
if landable is None:
return
wpt_head, commits = landable
landing = LandingSync.new(git_gecko, git_wpt, prev_wpt_head, wpt_head)
else:
if prev_wpt_head and landing.wpt_commits.base.sha1 != prev_wpt_head:
raise AbortError("Existing landing base commit %s doesn't match"
"supplied previous wpt head %s" %
(landing.wpt_commits.base.sha1, prev_wpt_head))
elif new_wpt_head and landing.wpt_commits.head.sha1 != new_wpt_head:
raise AbortError("Existing landing head commit %s doesn't match"
"supplied wpt head %s" %
(landing.wpt_commits.head.sha1, new_wpt_head))
head = landing.gecko_commits.head.sha1
if git_gecko.is_ancestor(head, env.config["gecko"]["refs"]["central"]):
logger.info("Landing reached central")
landing.finish()
elif git_gecko.is_ancestor(head, landing.gecko_integration_branch()):
logger.info("Landing is on inbound but not yet on central")
return
wpt_head, commits = landable_commits(
git_gecko,
git_wpt,
landing.wpt_commits.base.sha1,
landing.wpt_commits.head.sha1,
include_incomplete=include_incomplete)
assert wpt_head == landing.wpt_commits.head.sha1
landing.apply_prs(prev_wpt_head, commits)
landing.update_bug_components()
landing.update_sync_point(sync_point)
if landing.latest_try_push is None:
trypush.TryPush.create(landing,
hacks=False,
try_cls=trypush.TryFuzzyCommit,
exclude=["pgo", "ccov", "msvc"])
elif retry:
landing.gecko_rebase(landing.gecko_integration_branch())
trypush.TryPush.create(landing,
hacks=False,
try_cls=trypush.TryFuzzyCommit,
exclude=["pgo", "ccov", "msvc"])
else:
logger.info("Got existing try push %s" % landing.latest_try_push)
for _, sync, _ in commits:
if isinstance(sync, downstream.DownstreamSync):
try:
sync.try_notify()
except Exception as e:
logger.error(e.message)
return landing
def handle_push(git_gecko, git_wpt, event):
newrelic.agent.set_transaction_name("handle_push")
update_repositories(None, git_wpt, False)
landing.wpt_push(git_gecko, git_wpt,
[item["id"] for item in event["commits"]])
def handle_push(git_gecko, git_wpt, event):
update_repositories(None, git_wpt, False)
landing.wpt_push(git_gecko, git_wpt,
[item["id"] for item in event["commits"]])
