def rebase_apply():
logger.info("Applying with a rebase onto latest inbound")
new_base = self.gecko_integration_branch()
if len(self.gecko_commits) > 0:
gecko_work = self.gecko_worktree.get()
if self.gecko_commits[0].metadata[
"wpt-type"] == "dependent":
# If we have any dependent commits first reset to the new
# head. This prevents conflicts if the dependents already
# landed
# TODO: Actually check if they landed?
reset_head = new_base
else:
reset_head = "HEAD"
gecko_work.git.reset(reset_head, hard=True)
try:
self.gecko_rebase(new_base)
except git.GitCommandError:
try:
gecko_work.git.rebase(abort=True)
except git.GitCommandError:
pass
raise AbortError("Rebasing onto latest gecko failed")
self.wpt_to_gecko_commits(base=new_base)
return True
def push(landing):
"""Push from git_work_gecko to inbound."""
success = False
landing_tree = env.config["gecko"]["landing"]
old_head = None
err = None
while not success:
try:
logger.info("Rebasing onto %s" %
landing.gecko_integration_branch())
landing.gecko_rebase(landing.gecko_integration_branch())
except git.GitCommandError as e:
err = "Rebase failed:\n%s" % e
logger.error(err)
env.bz.comment(landing.bug, err)
raise AbortError(err)
if old_head == landing.gecko_commits.head.sha1:
err = ("Landing push failed and rebase didn't change head:%s" %
("\n%s" % err if err else ""))
logger.error(err)
env.bz.comment(landing.bug, err)
raise AbortError(err)
old_head = landing.gecko_commits.head.sha1
if not tree.is_open(landing_tree):
logger.info("%s is closed" % landing_tree)
raise RetryableError(AbortError("Tree is closed"))
try:
logger.info("Pushing landing")
landing.git_gecko.remotes.mozilla.push(
"%s:%s" %
(landing.branch_name, landing.gecko_integration_branch().split(
"/", 1)[1]))
except git.GitCommandError as e:
changes = landing.git_gecko.remotes.mozilla.fetch()
err = "Pushing update to remote failed:\n%s" % e
if not changes:
logger.error(err)
env.bz.comment(landing.bug, err)
raise AbortError(err)
else:
success = True
def push_commits(self):
remote_branch = self.get_or_create_remote_branch()
logger.info("Pushing commits from bug %s to branch %s" % (self.bug, remote_branch))
push_info = self.git_wpt.remotes.origin.push("refs/heads/%s:%s" %
(self.branch_name, remote_branch),
force=True,
set_upstream=True)
for item in push_info:
if item.flags & item.ERROR:
raise AbortError(item.summary)
def show(self, src_prefix=None, **kwargs):
show_args = ()
if src_prefix:
show_args = ("--", src_prefix)
try:
show_kwargs = {"binary": True,
"stdout_as_string": False}
show_kwargs.update(kwargs)
return self.repo.git.show(self.sha1, *show_args, **show_kwargs) + "\n"
except git.GitCommandError as e:
raise AbortError(e.message)
def save(self):
if not self.saved:
body = {
"type": "write",
"key": self.id(),
"value": self.to_json(),
}
resp = self.node.service_rpc(self.SERVICE, body)
if resp["body"]["type"] == "write_ok":
self.saved = True
else:
raise AbortError("Unable to save thunk {}".format(self._id()))
def create(cls, lock, sync, affected_tests=None, stability=False, hacks=True,
try_cls=TryFuzzyCommit, rebuild_count=None, check_open=True, **kwargs):
logger.info("Creating try push for PR %s" % sync.pr)
if check_open and not tree.is_open("try"):
logger.info("try is closed")
raise RetryableError(AbortError("Try is closed"))
# Ensure the required indexes exist
TaskGroupIndex.get_or_create(sync.git_gecko)
try_idx = TryCommitIndex.get_or_create(sync.git_gecko)
git_work = sync.gecko_worktree.get()
if rebuild_count is None:
rebuild_count = 0 if not stability else env.config['gecko']['try']['stability_count']
if not isinstance(rebuild_count, (int, long)):
logger.error("Could not find config for Stability rebuild count, using default 5")
rebuild_count = 5
with try_cls(sync.git_gecko, git_work, affected_tests, rebuild_count, hacks=hacks,
**kwargs) as c:
try_rev = c.push()
data = {
"try-rev": try_rev,
"stability": stability,
"gecko-head": sync.gecko_commits.head.sha1,
"wpt-head": sync.wpt_commits.head.sha1,
"status": "open",
"bug": sync.bug,
}
process_name = base.ProcessName.with_seq_id(sync.git_gecko,
cls.obj_type,
sync.sync_type,
getattr(sync, sync.obj_id))
rv = super(TryPush, cls).create(lock, sync.git_gecko, process_name, data)
# Add to the index
if try_rev:
try_idx.insert(try_idx.make_key(try_rev), process_name)
with rv.as_mut(lock):
rv.created = taskcluster.fromNowJSON("0 days")
env.bz.comment(sync.bug,
"Pushed to try%s %s" %
(" (stability)" if stability else "",
rv.treeherder_url))
return rv
def read_treeherder(self, status, output):
if status != 0:
logger.error("Failed to push to try:\n%s" % output)
raise RetryableError(AbortError("Failed to push to try"))
rev_match = rev_re.search(output)
if not rev_match:
logger.warning("No revision found in string:\n\n{}\n".format(output))
# Assume that the revision is HEAD
# This happens in tests and isn't a problem, but would be in real code,
# so that's not ideal
try:
try_rev = self.git_gecko.cinnabar.git2hg(self.worktree.head.commit.hexsha)
except ValueError:
return None
else:
try_rev = rev_match.group('rev')
return try_rev
def move_commits(repo,
revish,
message,
dest_repo,
skip_empty=True,
msg_filter=None,
metadata=None,
src_prefix=None,
dest_prefix=None,
amend=False,
three_way=True,
rev_name=None,
author=None,
exclude=None,
patch_fallback=False):
if rev_name is None:
rev_name = revish
diff_args = ()
if src_prefix:
diff_args = ("--", src_prefix)
try:
patch = repo.git.diff(revish,
binary=True,
submodule="diff",
pretty="email",
stdout_as_string=False,
*diff_args) + "\n"
logger.info("Created patch")
except git.GitCommandError as e:
raise AbortError(e.message)
return _apply_patch(patch,
message,
rev_name,
dest_repo,
skip_empty,
msg_filter,
metadata,
src_prefix,
dest_prefix,
amend,
three_way,
author=author,
exclude=exclude,
patch_fallback=patch_fallback)
def gecko_rebase(self, new_base_ref, abort_on_fail=False):
new_base = sync_commit.GeckoCommit(self.git_gecko, new_base_ref)
git_worktree = self.gecko_worktree.get()
set_new_base = True
try:
git_worktree.git.rebase(new_base.sha1)
except git.GitCommandError as e:
if abort_on_fail:
set_new_base = False
try:
git_worktree.git.rebase(abort=True)
except git.GitCommandError:
pass
raise AbortError("Rebasing onto latest gecko failed:\n%s" % e)
finally:
if set_new_base:
self.data["gecko-base"] = new_base_ref
self.gecko_commits.base = new_base
def add_commit(self, gecko_commit):
git_work = self.wpt_worktree.get()
metadata = {"gecko-commit": gecko_commit.canonical_rev,
"gecko-integration-branch": self.repository}
if os.path.exists(os.path.join(git_work.working_dir, gecko_commit.canonical_rev + ".diff")):
# If there's already a patch file here then don't try to create a new one
# because we'll presumbaly fail again
raise AbortError("Skipping due to existing patch")
wpt_commit = gecko_commit.move(git_work,
metadata=metadata,
msg_filter=commit_message_filter,
src_prefix=env.config["gecko"]["path"]["wpt"])
if wpt_commit:
self.wpt_commits.head = wpt_commit
return wpt_commit, True
def create(cls,
sync,
affected_tests=None,
stability=False,
hacks=True,
try_cls=TrySyntaxCommit,
rebuild_count=None,
check_open=True,
**kwargs):
logger.info("Creating try push for PR %s" % sync.pr)
if check_open and not tree.is_open("try"):
logger.info("try is closed")
raise RetryableError(AbortError("Try is closed"))
git_work = sync.gecko_worktree.get()
if rebuild_count is None:
rebuild_count = 0 if not stability else 10
with try_cls(sync.git_gecko,
git_work,
affected_tests,
rebuild_count,
hacks=hacks,
**kwargs) as c:
try_rev = c.push()
data = {
"try-rev": try_rev,
"stability": stability,
"gecko-head": sync.gecko_commits.head.sha1,
"wpt-head": sync.wpt_commits.head.sha1,
}
process_name = base.ProcessName.with_seq_id(sync.git_gecko, "syncs",
cls.obj_type,
sync.sync_type, "open",
getattr(sync, sync.obj_id))
rv = super(TryPush, cls).create(sync.git_gecko, process_name, data)
rv.created = taskcluster.fromNowJSON("0 days")
env.bz.comment(
sync.bug, "Pushed to try%s %s" %
(" (stability)" if stability else "", cls.treeherder_url(try_rev)))
return rv
def _apply_patch(patch, message, rev_name, dest_repo, skip_empty=True, msg_filter=None,
metadata=None, src_prefix=None, dest_prefix=None, amend=False, three_way=True,
author=None, exclude=None, patch_fallback=False):
assert type(patch) == str
if skip_empty and (not patch or patch.isspace() or
not any(line.startswith("diff ") for line in patch.splitlines())):
return None
if metadata is None:
metadata = {}
if msg_filter:
msg, metadata_extra = msg_filter(message)
else:
msg, metadata_extra = message, None
if metadata_extra:
metadata.update(metadata_extra)
msg = Commit.make_commit_msg(msg, metadata).encode("utf8")
with Store(dest_repo, rev_name + ".message", msg) as message_path:
strip_dirs = len(src_prefix.split("/")) + 1 if src_prefix else 1
with Store(dest_repo, rev_name + ".diff", patch) as patch_path:
# Without this tests were failing with "Index does not match"
dest_repo.git.update_index(refresh=True)
apply_kwargs = {}
if dest_prefix:
apply_kwargs["directory"] = dest_prefix
if three_way:
apply_kwargs["3way"] = True
else:
apply_kwargs["reject"] = True
try:
logger.info("Trying to apply patch")
dest_repo.git.apply(patch_path, index=True, binary=True,
p=strip_dirs, **apply_kwargs)
logger.info("Patch applied")
except git.GitCommandError as e:
err_msg = """git apply failed
%s returned status %s
Patch saved as :%s
Commit message saved as: %s
%s""" % (e.command, e.status, patch_path, message_path, e.stderr)
if patch_fallback and not dest_repo.is_dirty():
dest_repo.git.reset(hard=True)
cmd = ["patch", "-p%s" % strip_dirs, "-f", "-r=-",
"--no-backup-if-mismatch"]
if dest_prefix:
cmd.append("--directory=%s" % dest_prefix)
logger.info(" ".join(cmd))
proc = subprocess.Popen(cmd, stdin=subprocess.PIPE)
(stdout, stderr) = proc.communicate(patch)
if not proc.returncode == 0:
err_msg = ("%s\n\nPatch failed (status %i):\nstdout:\n%s\nstderr:\n%s" %
(err_msg, proc.returncode, stdout, stderr))
else:
err_msg = None
prefix = "+++ "
paths = []
for line in patch.splitlines():
if line.startswith(prefix):
path = "%s/%s" % (
dest_prefix,
"/".join(line[len(prefix):].split("/")[strip_dirs:]))
paths.append(path)
dest_repo.git.add(*paths)
if err_msg:
raise AbortError(err_msg)
if exclude:
exclude_paths = [os.path.join(dest_prefix, exclude_path)
if dest_prefix else exclude_path
for exclude_path in exclude]
exclude_paths = [item for item in exclude_paths
if os.path.exists(os.path.join(dest_repo.working_dir, item))]
try:
dest_repo.git.checkout("HEAD", *exclude_paths)
except git.GitCommandError as e:
logger.info(e)
try:
logger.info("Creating commit")
return Commit.create(dest_repo, msg, None, amend=amend, author=author)
except git.GitCommandError as e:
if amend and e.status == 1 and "--allow-empty" in e.stdout:
logger.warning("Amending commit made it empty, resetting")
dest_repo.git.reset("HEAD^")
return None
elif not amend and e.status == 1 and "nothing added to commit" in e.stdout:
logger.warning("Commit added no changes to destination repo")
return None
else:
dest_repo.git.reset(hard=True)
raise
def try_push_complete(git_gecko, git_wpt, try_push, sync):
if not try_push.taskgroup_id:
logger.error("No taskgroup id set for try push")
return
if not try_push.status == "complete":
# Ensure we don't have some old set of tasks
try_push.wpt_tasks(force_update=True)
if not try_push.is_complete(allow_unscheduled=True):
logger.info("Try push is not complete")
return
if not try_push.validate_tasks():
if len(sync.latest_busted_try_pushes()) > 5:
message = ("Too many busted try pushes. "
"Check the try results for infrastructure issues.")
sync.error = message
env.bz.comment(sync.bug, message)
try_push.status = "complete"
try_push.infra_fail = True
raise AbortError(message)
elif len(try_push.failed_builds()):
message = ("Try push had build failures")
sync.error = message
env.bz.comment(sync.bug, message)
try_push.status = "complete"
try_push.infra_fail = True
raise AbortError(message)
else:
logger.info("Try push %r for PR %s complete" % (try_push, sync.pr))
disabled = []
if not try_push.success():
if sync.affected_tests():
log_files = try_push.download_raw_logs()
if not log_files:
raise ValueError("No log files found for try push %r" %
try_push)
disabled = sync.update_metadata(
log_files, stability=try_push.stability)
else:
env.bz.comment(
sync.bug,
("The PR was not expected to affect any tests, "
"but the try push wasn't a success. Check the try "
"results for infrastructure issues"))
# TODO: consider marking the push an error here so that we can't land without
# manual intervention
if try_push.stability and disabled:
logger.info("The following tests were disabled:\n%s" %
"\n".join(disabled))
# TODO notify relevant people about test expectation changes, stability
env.bz.comment(sync.bug, ("The following tests were disabled "
"based on stability try push:\n %s" %
"\n".join(disabled)))
try_push.status = "complete"
next_try_push = sync.next_try_push()
if not next_try_push or next_try_push.stability:
pr = env.gh_wpt.get_pull(sync.pr)
if pr.merged:
sync.try_notify()
def reapply_local_commits(self, gecko_commits_landed):
# The local commits to apply are everything that hasn't been landed at this
# point in the process
commits = [
item for item in self.unlanded_gecko_commits()
if item.canonical_rev not in gecko_commits_landed
]
landing_commit = self.gecko_commits[-1]
git_work_gecko = self.gecko_worktree.get()
logger.debug("Reapplying commits: %s" % " ".join(item.canonical_rev
for item in commits))
if not commits:
return
already_applied = landing_commit.metadata.get("reapplied-commits")
if already_applied:
already_applied = [
item.strip() for item in already_applied.split(",")
]
else:
already_applied = []
already_applied_set = set(already_applied)
unapplied_gecko_commits = [
item for item in commits
if item.canonical_rev not in already_applied_set
]
try:
for i, commit in enumerate(unapplied_gecko_commits):
def msg_filter(_):
msg = landing_commit.msg
reapplied_commits = (
already_applied +
[commit.canonical_rev for commit in commits[:i + 1]])
metadata = {
"reapplied-commits": ", ".join(reapplied_commits)
}
return msg, metadata
logger.info("Reapplying %s - %s" % (commit.sha1, commit.msg))
# Passing in a src_prefix here means that we only generate a patch for the
# part of the commit that affects wpt, but then we need to undo it by adding
# the same dest prefix
commit = commit.move(
git_work_gecko,
msg_filter=msg_filter,
src_prefix=env.config["gecko"]["path"]["wpt"],
dest_prefix=env.config["gecko"]["path"]["wpt"],
three_way=True,
amend=True)
if commit is None:
break
except AbortError as e:
err_msg = (
"Landing wpt failed because reapplying commits failed:\n%s" %
(e.message, ))
env.bz.comment(self.bug, err_msg)
raise AbortError(err_msg)
def update_landing(git_gecko,
git_wpt,
prev_wpt_head=None,
new_wpt_head=None,
include_incomplete=False,
retry=False):
"""Create or continue a landing of wpt commits to gecko.
:param prev_wpt_head: The sha1 of the previous wpt commit landed to gecko.
:param wpt_head: The sha1 of the latest possible commit to land to gecko,
or None to use the head of the master branch"
:param include_incomplete: By default we don't attempt to land anything that
hasn't completed a metadata update. This flag disables
that and just lands everything up to the specified commit.
:param retry: Create a new try push for the landing even if there's an existing one"""
landing = current(git_gecko, git_wpt)
sync_point = load_sync_point(git_gecko, git_wpt)
if landing is None:
update_repositories(git_gecko, git_wpt)
if prev_wpt_head is None:
prev_wpt_head = sync_point["upstream"]
landable = landable_commits(git_gecko,
git_wpt,
prev_wpt_head,
wpt_head=new_wpt_head,
include_incomplete=include_incomplete)
if landable is None:
return
wpt_head, commits = landable
landing = LandingSync.new(git_gecko, git_wpt, prev_wpt_head, wpt_head)
else:
if prev_wpt_head and landing.wpt_commits.base.sha1 != prev_wpt_head:
raise AbortError("Existing landing base commit %s doesn't match"
"supplied previous wpt head %s" %
(landing.wpt_commits.base.sha1, prev_wpt_head))
elif new_wpt_head and landing.wpt_commits.head.sha1 != new_wpt_head:
raise AbortError("Existing landing head commit %s doesn't match"
"supplied wpt head %s" %
(landing.wpt_commits.head.sha1, new_wpt_head))
head = landing.gecko_commits.head.sha1
if git_gecko.is_ancestor(head, env.config["gecko"]["refs"]["central"]):
logger.info("Landing reached central")
landing.finish()
elif git_gecko.is_ancestor(head, landing.gecko_integration_branch()):
logger.info("Landing is on inbound but not yet on central")
return
wpt_head, commits = landable_commits(
git_gecko,
git_wpt,
landing.wpt_commits.base.sha1,
landing.wpt_commits.head.sha1,
include_incomplete=include_incomplete)
assert wpt_head == landing.wpt_commits.head.sha1
landing.apply_prs(prev_wpt_head, commits)
landing.update_bug_components()
landing.update_sync_point(sync_point)
if landing.latest_try_push is None:
trypush.TryPush.create(landing,
hacks=False,
try_cls=trypush.TryFuzzyCommit,
exclude=["pgo", "ccov", "msvc"])
elif retry:
landing.gecko_rebase(landing.gecko_integration_branch())
trypush.TryPush.create(landing,
hacks=False,
try_cls=trypush.TryFuzzyCommit,
exclude=["pgo", "ccov", "msvc"])
else:
logger.info("Got existing try push %s" % landing.latest_try_push)
for _, sync, _ in commits:
if isinstance(sync, downstream.DownstreamSync):
try:
sync.try_notify()
except Exception as e:
logger.error(e.message)
return landing
def try_push_complete(git_gecko,
git_wpt,
try_push,
sync,
allow_push=True,
accept_failures=False):
intermittents = []
if not try_push.success():
target_success_rate = 0.5 if not try_push.stability else 0.8
if not accept_failures and len(try_push.failed_builds()):
message = ("Try push had build failures")
sync.error = message
env.bz.comment(sync.bug, message)
try_push.status = "complete"
try_push.infra_fail = True
raise AbortError(message)
elif (not accept_failures and not try_push.stability
and try_push.failure_limit_exceeded(target_success_rate)):
record_too_many_failures(sync, try_push)
try_push.status = "complete"
return
if not try_push.stability:
update_metadata(sync, try_push)
sync.gecko_rebase(sync.gecko_landing_branch())
trypush.TryPush.create(sync,
hacks=False,
stability=True,
rebuild_count=0,
try_cls=trypush.TryFuzzyCommit,
exclude=["pgo", "ccov", "msvc"])
try_push.status = "complete"
return
else:
retriggered = try_push.retriggered_wpt_states(force_update=True)
if not retriggered:
if not accept_failures and try_push.failure_limit_exceeded(
target_success_rate):
record_too_many_failures(sync, try_push)
try_push.status = "complete"
return
num_new_jobs = try_push.retrigger_failures()
logger.info("%s new tasks scheduled on try for %s" %
(num_new_jobs, sync.bug))
if num_new_jobs:
env.bz.comment(
sync.bug,
("Retriggered failing web-platform-test tasks on "
"try before final metadata update."))
return
intermittents = []
for name, data in retriggered.iteritems():
total = float(sum(data["states"].itervalues()))
# assuming that only failures cause metadata updates
if data["states"][tc.SUCCESS] / total >= try_push._min_success:
intermittents.append(name)
update_metadata(sync, try_push, intermittents)
try_push.status = "complete"
push_to_gecko(git_gecko, git_wpt, sync, allow_push)
def try_push_complete(git_gecko, git_wpt, try_push, sync):
if not try_push.taskgroup_id:
logger.error("No taskgroup id set for try push")
return
if not try_push.status == "complete":
# Ensure we don't have some old set of tasks
tasks = try_push.tasks()
if not tasks.complete(allow_unscheduled=True):
logger.info("Try push %s is not complete" % try_push.treeherder_url)
return
logger.info("Try push %s is complete" % try_push.treeherder_url)
try:
if not tasks.validate():
try_push.infra_fail = True
if len(sync.latest_busted_try_pushes()) > 5:
message = ("Too many busted try pushes. "
"Check the try results for infrastructure issues.")
sync.error = message
env.bz.comment(sync.bug, message)
try_push.status = "complete"
raise AbortError(message)
elif len(tasks.failed_builds()):
message = ("Try push had build failures")
sync.error = message
env.bz.comment(sync.bug, message)
try_push.status = "complete"
try_push.infra_fail = True
raise AbortError(message)
else:
logger.info("Try push %r for PR %s complete" % (try_push, sync.pr))
disabled = []
if tasks.has_failures():
if sync.affected_tests():
log_files = []
wpt_tasks = try_push.download_logs(tasks.wpt_tasks)
for task in wpt_tasks:
for run in task.get("status", {}).get("runs", []):
log = run.get("_log_paths", {}).get("wptreport.json")
if log:
log_files.append(log)
if not log_files:
raise ValueError("No log files found for try push %r" % try_push)
disabled = sync.update_metadata(log_files, stability=try_push.stability)
else:
env.bz.comment(sync.bug, ("The PR was not expected to affect any tests, "
"but the try push wasn't a success. "
"Check the try results for infrastructure "
"issues"))
# TODO: consider marking the push an error here so that we can't
# land without manual intervention
if try_push.stability and disabled:
logger.info("The following tests were disabled:\n%s" % "\n".join(disabled))
# TODO notify relevant people about test expectation changes, stability
env.bz.comment(sync.bug, ("The following tests were disabled "
"based on stability try push:\n %s" %
"\n".join(disabled)))
try_push.status = "complete"
sync.next_try_push()
finally:
sync.update_github_check()
else:
sync.next_try_push()
sync.update_github_check()
if sync.landable_status == LandableStatus.ready:
sync.try_notify()