def new_wpt_pr(git_gecko, git_wpt, pr_data, raise_on_error=True, repo_update=True): """ Start a new downstream sync """ if pr_data["user"]["login"] == env.config["web-platform-tests"]["github"]["user"]: raise ValueError("Tried to create a downstream sync for a PR created " "by the wpt bot") if repo_update: update_repositories(git_gecko, git_wpt) pr_id = pr_data["number"] if DownstreamSync.for_pr(git_gecko, git_wpt, pr_id): return wpt_base = "origin/%s" % pr_data["base"]["ref"] with SyncLock("downstream", str(pr_id)) as lock: sync = DownstreamSync.new(lock, git_gecko, git_wpt, wpt_base, pr_id, pr_data["title"], pr_data["body"] or "") with sync.as_mut(lock): try: sync.update_commits() sync.update_github_check() except Exception as e: sync.error = e if raise_on_error: raise traceback.print_exc() logger.error(e)
def push_to_gecko(git_gecko, git_wpt, sync, allow_push=True): if not allow_push: logger.info("Landing in bug %s is ready for push.\n" "Working copy is in %s" % (sync.bug, sync.gecko_worktree.get().working_dir)) return update_repositories(git_gecko, git_wpt) wpt_head, commits = landable_commits(git_gecko, git_wpt, sync.wpt_commits.base.sha1, sync.wpt_commits.head.sha1, include_incomplete=True) push(sync) for _, sync, _ in commits: if sync is not None: if isinstance(sync, downstream.DownstreamSync): # If we can't perform a notification by now it isn't ever going # to work try: if not sync.skip: sync.try_notify() except Exception as e: logger.error(e.message) finally: sync.finish() if not sync.results_notified: env.bz.comment( sync.bug, "Result changes from PR not available.")
def do_pr(git_gecko, git_wpt, pr_id, *args, **kwargs): import update if pr_id is None: pr_id = sync_from_path(git_gecko, git_wpt).pr pr = env.gh_wpt.get_pull(int(pr_id)) update_repositories(git_gecko, git_wpt, True) update.update_pr(git_gecko, git_wpt, pr, kwargs["rebase"])
def do_retrigger(git_gecko, git_wpt, **kwargs): import errors import update import upstream from landing import load_sync_point, unlanded_with_type update_repositories(git_gecko, git_wpt, True) print("Retriggering upstream syncs with errors") for sync in upstream.UpstreamSync.load_all(git_gecko, git_wpt, status="open"): if sync.error: try: upstream.update_sync(git_gecko, git_wpt, sync) except errors.AbortError as e: print("Update failed:\n%s" % e) pass print("Retriggering downstream syncs on master") sync_point = load_sync_point(git_gecko, git_wpt) prev_wpt_head = sync_point["upstream"] unlandable = unlanded_with_type(git_gecko, git_wpt, None, prev_wpt_head) errors = update.retrigger(git_gecko, git_wpt, unlandable) if errors: print("The following PRs have errors:\n%s" % "\n".join(errors))
def commit_status_changed(git_gecko, git_wpt, sync, context, status, url, head_sha, raise_on_error=False): update_repositories(git_gecko, git_wpt) if sync.skip: return try: logger.debug("Got status %s for PR %s" % (status, sync.pr)) if status == "pending": # We got new commits that we missed sync.update_commits() return check_state, _ = env.gh_wpt.get_combined_status(sync.pr) sync.last_pr_check = {"state": check_state, "sha": head_sha} if check_state == "success": sync.next_try_push() except Exception as e: sync.error = e if raise_on_error: raise traceback.print_exc() logger.error(e)
def update_sync(git_gecko, git_wpt, sync, raise_on_error=True, repo_update=True): if sync.status in ("wpt-merged", "complete"): logger.info("Nothing to do for sync with status %s" % sync.status) return set(), set(), set() if repo_update: update_repositories(git_gecko, git_wpt) assert isinstance(sync, UpstreamSync) update_syncs = {sync.bug: (sync, sync.gecko_commits.head.sha1)} pushed_syncs, failed_syncs = update_sync_prs(sync._lock, git_gecko, git_wpt, {}, update_syncs, raise_on_error=raise_on_error) if sync not in failed_syncs: landed_syncs = try_land_syncs(sync._lock, [sync]) else: landed_syncs = set() return pushed_syncs, failed_syncs, landed_syncs
def do_pr(git_gecko, git_wpt, pr_ids, *args, **kwargs): import update if not pr_ids: pr_ids = [sync_from_path(git_gecko, git_wpt).pr] for pr_id in pr_ids: pr = env.gh_wpt.get_pull(int(pr_id)) update_repositories(git_gecko, git_wpt) update.update_pr(git_gecko, git_wpt, pr, kwargs["rebase"])
def __call__(self, git_gecko, git_wpt): logger.info("Running retrigger") update_repositories(git_gecko, git_wpt) sync_point = landing.load_sync_point(git_gecko, git_wpt) prev_wpt_head = sync_point["upstream"] unlanded = landing.unlanded_with_type(git_gecko, git_gecko, None, prev_wpt_head) update.retrigger(git_gecko, git_wpt, unlanded)
def __call__(self, git_gecko, git_wpt): newrelic.agent.set_transaction_name("RetriggerHandler") logger.info("Running retrigger") update_repositories(git_gecko, git_wpt) sync_point = landing.load_sync_point(git_gecko, git_wpt) prev_wpt_head = sync_point["upstream"] unlanded = landing.unlanded_with_type(git_gecko, git_gecko, None, prev_wpt_head) update.retrigger(git_gecko, git_wpt, unlanded)
def handle_status(git_gecko, git_wpt, event): newrelic.agent.set_transaction_name("handle_status") if event["context"] == "upstream/gecko": # Never handle changes to our own status return repo_update = event.get("_wptsync", {}).get("repo_update", True) if repo_update: update_repositories(None, git_wpt, False) rev = event["sha"] # First check if the PR is head of any pull request pr_id = pr_for_commit(git_wpt, rev) newrelic.agent.add_custom_parameter("rev", rev) newrelic.agent.add_custom_parameter("context", event["context"]) newrelic.agent.add_custom_parameter("state", event["state"]) if not pr_id: # This usually happens if we got behind, so the commit is no longer the latest one # There are a few possibilities for what happened: # * Something new was pushed. In that case ignoring this message is fine # * The PR got merged in a way that changes the SHAs. In that case we assume that # the sync will get triggered later like when there's a push for the commit logger.warning( "Got status for commit %s which is not the current HEAD of any PR\n" "context: %s url: %s state: %s" % (rev, event["context"], event["target_url"], event["state"])) return else: logger.info("Got status for commit %s from PR %s\n" "context: %s url: %s state: %s" % (rev, pr_id, event["context"], event["target_url"], event["state"])) sync = get_pr_sync(git_gecko, git_wpt, pr_id) if not sync: # Presumably this is a thing we ought to be downstreaming, but missed somehow logger.info( "Got a status update for PR %s which is unknown to us; starting downstreaming" % pr_id) from update import schedule_pr_task schedule_pr_task("opened", env.gh_wpt.get_pull(pr_id)) update_func = None if isinstance(sync, upstream.UpstreamSync): update_func = upstream.commit_status_changed elif isinstance(sync, downstream.DownstreamSync): update_func = downstream.commit_status_changed if update_func is not None: with SyncLock.for_process(sync.process_name) as lock: with sync.as_mut(lock): update_func(git_gecko, git_wpt, sync, event["context"], event["state"], event["target_url"], event["sha"])
def update_from_github(git_gecko, git_wpt, sync_classes, statuses=None): if statuses is None: statuses = ["*"] update_repositories(git_gecko, git_wpt) for cls in sync_classes: for status in statuses: if status != "*" and status not in cls.statuses: continue syncs = cls.load_by_status(git_gecko, git_wpt, status) for sync in syncs: if not sync.pr: continue logger.info("Updating sync for PR %s" % sync.pr) pr = env.gh_wpt.get_pull(sync.pr) update_pr(git_gecko, git_wpt, pr)
def handle_status(git_gecko, git_wpt, event): if event["context"] == "upstream/gecko": # Never handle changes to our own status return update_repositories(None, git_wpt, False) rev = event["sha"] # First check if the PR is head of any pull request pr_id = pr_for_commit(git_wpt, rev) if not pr_id: # This usually happens if we got behind, so the commit is no longer the latest one # There are a few possibilities for what happened: # * Something new was pushed. In that case ignoring this message is fine # * The PR got merged in a way that changes the SHAs. In that case we assume that # the syncc will get triggered later like when there's a push for the commit logger.warning( "Got status for commit %s which is not the current HEAD of any PR\n" "context: %s url: %s state: %s" % (rev, event["context"], event["target_url"], event["state"])) return else: logger.info("Got status for commit %s from PR %s\n" "context: %s url: %s state: %s" % (rev, pr_id, event["context"], event["target_url"], event["state"])) sync = get_pr_sync(git_gecko, git_wpt, pr_id) if not sync: # Presumably this is a thing we ought to be downstreaming, but missed somehow logger.info( "Got a status update for PR %s which is unknown to us; starting downstreaming" % pr_id) from update import schedule_pr_task schedule_pr_task("opened", env.gh_wpt.get_pull(pr_id)) if isinstance(sync, upstream.UpstreamSync): upstream.commit_status_changed(git_gecko, git_wpt, sync, event["context"], event["state"], event["target_url"], event["sha"]) elif isinstance(sync, downstream.DownstreamSync): downstream.commit_status_changed(git_gecko, git_wpt, sync, event["context"], event["state"], event["target_url"], event["sha"])
def __call__(self, git_gecko, git_wpt, body): newrelic.agent.set_transaction_name("PushHandler") repo = body["_meta"]["routing_key"] if "/" in repo: repo_name = repo.rsplit("/", 1)[1] else: repo_name = repo # Commands can override the base rev and select only certain processses base_rev = body.get("_wptsync", {}).get("base_rev") processes = body.get("_wptsync", {}).get("processes") # Not sure if it's ever possible to get multiple heads here in a way that # matters for us rev = body["payload"]["data"]["heads"][0] logger.info("Handling commit %s to repo %s" % (rev, repo)) newrelic.agent.add_custom_parameter("repo", repo) newrelic.agent.add_custom_parameter("rev", rev) update_repositories(git_gecko, git_wpt, include_autoland=True, wait_gecko_commit=rev) try: git_rev = git_gecko.cinnabar.hg2git(rev) except ValueError: pass else: if gecko_repo(git_gecko, git_rev) is None: logger.info("Skipping commit as it isn't in a branch we track") return if processes is None or "upstream" in processes: upstream.gecko_push(git_gecko, git_wpt, repo_name, rev, base_rev=base_rev) if processes is None or "landing" in processes: landing.gecko_push(git_gecko, git_wpt, repo_name, rev, base_rev=base_rev)
def do_retrigger(git_gecko, git_wpt, **kwargs): import errors import update import upstream from landing import current, load_sync_point, unlanded_with_type update_repositories(git_gecko, git_wpt) if kwargs["upstream"]: print("Retriggering upstream syncs with errors") for sync in upstream.UpstreamSync.load_by_status( git_gecko, git_wpt, "open"): if sync.error: with SyncLock.for_process(sync.process_name) as lock: with sync.as_mut(lock): try: upstream.update_sync(git_gecko, git_wpt, sync, repo_update=False) except errors.AbortError as e: print("Update failed:\n%s" % e) pass if kwargs["downstream"]: print("Retriggering downstream syncs on master") current_landing = current(git_gecko, git_wpt) if current_landing is None: sync_point = load_sync_point(git_gecko, git_wpt) prev_wpt_head = sync_point["upstream"] else: prev_wpt_head = current_landing.wpt_commits.head.sha1 unlandable = unlanded_with_type(git_gecko, git_wpt, None, prev_wpt_head) errors = update.retrigger(git_gecko, git_wpt, unlandable) if errors: print("The following PRs have errors:\n%s" % "\n".join(errors))
def update_landing(git_gecko, git_wpt, prev_wpt_head=None, new_wpt_head=None, include_incomplete=False, retry=False): """Create or continue a landing of wpt commits to gecko. :param prev_wpt_head: The sha1 of the previous wpt commit landed to gecko. :param wpt_head: The sha1 of the latest possible commit to land to gecko, or None to use the head of the master branch" :param include_incomplete: By default we don't attempt to land anything that hasn't completed a metadata update. This flag disables that and just lands everything up to the specified commit. :param retry: Create a new try push for the landing even if there's an existing one""" landing = current(git_gecko, git_wpt) sync_point = load_sync_point(git_gecko, git_wpt) if landing is None: update_repositories(git_gecko, git_wpt) if prev_wpt_head is None: prev_wpt_head = sync_point["upstream"] landable = landable_commits(git_gecko, git_wpt, prev_wpt_head, wpt_head=new_wpt_head, include_incomplete=include_incomplete) if landable is None: return wpt_head, commits = landable landing = LandingSync.new(git_gecko, git_wpt, prev_wpt_head, wpt_head) else: if prev_wpt_head and landing.wpt_commits.base.sha1 != prev_wpt_head: raise AbortError("Existing landing base commit %s doesn't match" "supplied previous wpt head %s" % (landing.wpt_commits.base.sha1, prev_wpt_head)) elif new_wpt_head and landing.wpt_commits.head.sha1 != new_wpt_head: raise AbortError("Existing landing head commit %s doesn't match" "supplied wpt head %s" % (landing.wpt_commits.head.sha1, new_wpt_head)) head = landing.gecko_commits.head.sha1 if git_gecko.is_ancestor(head, env.config["gecko"]["refs"]["central"]): logger.info("Landing reached central") landing.finish() elif git_gecko.is_ancestor(head, landing.gecko_integration_branch()): logger.info("Landing is on inbound but not yet on central") return wpt_head, commits = landable_commits( git_gecko, git_wpt, landing.wpt_commits.base.sha1, landing.wpt_commits.head.sha1, include_incomplete=include_incomplete) assert wpt_head == landing.wpt_commits.head.sha1 landing.apply_prs(prev_wpt_head, commits) landing.update_bug_components() landing.update_sync_point(sync_point) if landing.latest_try_push is None: trypush.TryPush.create(landing, hacks=False, try_cls=trypush.TryFuzzyCommit, exclude=["pgo", "ccov", "msvc"]) elif retry: landing.gecko_rebase(landing.gecko_integration_branch()) trypush.TryPush.create(landing, hacks=False, try_cls=trypush.TryFuzzyCommit, exclude=["pgo", "ccov", "msvc"]) else: logger.info("Got existing try push %s" % landing.latest_try_push) for _, sync, _ in commits: if isinstance(sync, downstream.DownstreamSync): try: sync.try_notify() except Exception as e: logger.error(e.message) return landing
def handle_push(git_gecko, git_wpt, event): newrelic.agent.set_transaction_name("handle_push") update_repositories(None, git_wpt, False) landing.wpt_push(git_gecko, git_wpt, [item["id"] for item in event["commits"]])
def handle_push(git_gecko, git_wpt, event): update_repositories(None, git_wpt, False) landing.wpt_push(git_gecko, git_wpt, [item["id"] for item in event["commits"]])