def land_commit( sha: str, repo: Repository, base_repo: Repository, pr: Optional[PullRequest], files: Iterable[Union[File, str]], *, target_app: Optional[str] = None, ): """ :param sha: The hash of the commit we are building :param repo: The repo containing the above commit :param base_repo: The *base* cs61a-apps repo containing the deploy.yaml config :param pr: The PR made to trigger the build, if any :param files: Files changed in the commit, used for target determination :param target_app: App to rebuild, if not all :param dequeue_only: Only pop targets off the queue, do not build any new targets """ if target_app: targets = [target_app] else: targets = determine_targets( repo, files if repo.full_name == base_repo.full_name else []) pr_number = pr.number if pr else 0 enqueue_builds(targets, pr_number, pack(repo.clone_url, sha)) dequeue_and_build(base_repo)
def webhook(): if not hmac.compare_digest( "sha1=" + hmac.new( get_secret(secret_name="GITHUB_WEBHOOK_SECRET").encode("ascii"), request.get_data(), "sha1", ).hexdigest(), request.headers["X-Hub-Signature"], ): abort(401) payload = request.json g = Github(get_secret(secret_name="GITHUB_ACCESS_TOKEN")) if "pusher" in payload and payload["ref"] == "refs/heads/master": base_repo = g.get_repo(GITHUB_REPO) repo = g.get_repo(payload["repository"]["id"]) sha = payload["after"] land_commit( sha, repo, base_repo, None, [ file for commit in payload["commits"] for file in commit["added"] + commit["modified"] + commit["removed"] ], ) delete_unused_services() if "pull_request" in payload: repo_id = payload["repository"]["id"] repo = g.get_repo(repo_id) pr = repo.get_pull(payload["pull_request"]["number"]) if payload["action"] in ("opened", "synchronize", "reopened"): if repo.full_name != GITHUB_REPO: land_commit(pr.head.sha, repo, g.get_repo(GITHUB_REPO), pr, []) else: for target in determine_targets(repo, pr.get_files()): report_build_status( target, pr.number, pack(repo.clone_url, pr.head.sha), BuildStatus.pushed, None, None, private=True, ) elif payload["action"] == "closed": set_pr_comment("PR closed, shutting down PR builds...", pr) delete_unused_services(pr.number) set_pr_comment("All PR builds shut down.", pr) return ""
def update_status(packed_ref: str, pr_number: int): g = Github(get_secret(secret_name="GITHUB_ACCESS_TOKEN")) repo_url, sha = unpack(packed_ref) repo_name = urlparse(repo_url).path.split(".")[0][ 1: ] # This is awful ... but it works repo = g.get_repo(repo_name) # First we will update the commit-specific status indicator with connect_db() as db: statuses = db( "SELECT app, status FROM builds WHERE packed_ref=%s", [packed_ref] ).fetchall() statuses = [(app, BuildStatus(status)) for app, status in statuses] if all(status == BuildStatus.success for _, status in statuses): repo.get_commit(sha).create_status( "success", "https://logs.cs61a.org/service/buildserver", "All modified services built!", "Pusher", ) elif any(status == BuildStatus.failure for _, status in statuses): repo.get_commit(sha).create_status( "failure", "https://logs.cs61a.org/service/buildserver", "Pusher failed to build a modified service", "Pusher", ) elif all( status in (BuildStatus.building, BuildStatus.queued) for _, status in statuses ): repo.get_commit(sha).create_status( "pending", "https://logs.cs61a.org/service/buildserver", "Pusher is building all modified services", "Pusher", ) else: # There are no failures, but not everything is building / built repo.get_commit(sha).create_status( "pending", "https://logs.cs61a.org/service/buildserver", "You must build all modified apps before merging", "Pusher", ) if pr_number == 0: return pr = repo.get_pull(pr_number) # Now we will update the PR comment, looking at builds for all packed_refs in the PR apps = determine_targets(repo, pr.get_files()) success = [] failure = [] running = [] queued = [] triggerable = [] with connect_db() as db: for app in apps: successful_build = db( "SELECT url, log_url, unix, packed_ref FROM builds WHERE app=%s AND pr_number=%s AND status='success' ORDER BY unix DESC LIMIT 1", [app, pr_number], ).fetchone() if successful_build: url, log_url, success_unix, packed_ref = successful_build _, sha = unpack(packed_ref) if url: for link in url.split(","): success.append((app, link, sha, log_url)) else: success.append((app, None, sha, log_url)) failed_build = db( "SELECT unix, log_url, packed_ref FROM builds WHERE app=%s AND pr_number=%s AND status='failure' ORDER BY unix DESC LIMIT 1", [app, pr_number], ).fetchone() if failed_build: unix, log_url, packed_ref = failed_build if not successful_build or success_unix < unix: _, sha = unpack(packed_ref) failure.append((app, sha, log_url)) running_build = db( "SELECT packed_ref FROM builds WHERE app=%s AND pr_number=%s AND status='building'", [app, pr_number], ).fetchone() if running_build: [packed_ref] = running_build _, sha = unpack(packed_ref) running.append((app, sha)) queued_build = db( "SELECT packed_ref FROM builds WHERE app=%s AND pr_number=%s AND status='queued'", [app, pr_number], ).fetchone() if queued_build: [packed_ref] = queued_build _, sha = unpack(packed_ref) queued.append((app, sha)) latest_commit_build = db( "SELECT * FROM builds WHERE app=%s AND pr_number=%s AND packed_ref=%s AND status!='pushed'", [app, pr_number, pack(repo_url, pr.head.sha)], ).fetchone() if not latest_commit_build: triggerable.append(app) if repo.name == "berkeley-cs61a": message = f"## Build Status ([pr/{pr_number}]({pr.html_url}))\n\n" elif repo.name == "cs61a-apps": message = f"## Build Status ([apps/{pr_number}]({pr.html_url}))\n\n" else: message = f"## Build Status (#{pr_number})\n\n" if success: message += ( "**Successful Builds**\n" + "\n".join( f" - [{host}](https://{host}) ({sha}) [[logs]({log_url})]" if host else f" - `{app}` ({sha}) [[logs]({log_url})]" for app, host, sha, log_url in success ) + "\n\n" ) if failure: message += ( "**Failed Builds**\n" + "\n".join( f" - `{app}` ({sha}) [[logs]({log_url})]" for app, sha, log_url in failure ) + "\n\n" ) if running: message += ( "**Running Builds**\n" + "\n".join(f" - `{app}` ({sha})" for app, sha in running) + "\n\n" ) if queued: message += ( "**Queued Builds**\n" + "\n".join(f" - `{app}` ({sha})" for app, sha in queued) + "\n\n" ) if (success or failure or running or queued) and triggerable: message += "-----\n" if triggerable: message += ( f"**[Click here]({url_for('trigger_build', pr_number=pr.number)})** to trigger all builds " f"for the most recent commit ({pr.head.sha})\n\n" "Or trigger builds individually:\n" ) + "\n".join( f" - [Click here]({url_for('trigger_build', pr_number=pr.number, app=app)}) " f"to build `{app}` at the most recent commit ({pr.head.sha})" for app in triggerable ) set_pr_comment(message, pr)
def land_commit( sha: str, repo: Repository, base_repo: Repository, pr: Optional[PullRequest], files: Iterable[Union[File, str]], *, target_app: Optional[str] = None, dequeue_only=False, ): """ :param sha: The hash of the commit we are building :param repo: The repo containing the above commit :param base_repo: The *base* cs61a-apps repo containing the deploy.yaml config :param pr: The PR made to trigger the build, if any :param files: Files changed in the commit, used for target determination :param target_app: App to rebuild, if not all :param dequeue_only: Only pop targets off the queue, do not build any new targets """ if dequeue_only: targets = [] elif target_app: targets = [target_app] else: targets = determine_targets( repo, files if repo.full_name == base_repo.full_name else []) pr_number = pr.number if pr else 0 grouped_targets = enqueue_builds(targets, pr_number, pack(repo.clone_url, sha)) for packed_ref, targets in grouped_targets.items(): repo_clone_url, sha = unpack(packed_ref) # If the commit is made on the base repo, take the config from the current commit. # Otherwise, retrieve it from master clone_commit( base_repo.clone_url, sha if repo_clone_url == base_repo.clone_url else base_repo.get_branch(base_repo.default_branch).commit.sha, ) apps = [App(target) for target in targets] for app in apps: with tempfile.TemporaryFile("w+") as logs: try: with redirect_descriptor(stdout, logs), redirect_descriptor( stderr, logs): land_app(app, pr_number, sha, repo) if app.config is not None: update_service_routes([app], pr_number) except: traceback.print_exc(file=logs) logs.seek(0) report_build_status( app.name, pr_number, pack(repo.clone_url, sha), BuildStatus.failure, None, logs.read(), private=repo.full_name == base_repo.full_name, ) else: logs.seek(0) report_build_status( app.name, pr_number, pack(repo.clone_url, sha), BuildStatus.success, None if app.config is None else ",".join( hostname.to_str() for hostname in get_pr_subdomains(app, pr_number)), logs.read(), private=repo.full_name == base_repo.full_name, ) if grouped_targets: # because we ran a build, we need to clear the queue of anyone we blocked # we run this in a new worker to avoid timing out clear_queue(repo=repo.full_name, pr_number=pr_number, noreply=True)