def dequeue_and_build(base_repo: Repository):
grouped_targets = dequeue_builds()
for packed_ref, targets in grouped_targets.items():
repo_clone_url, sha = unpack(packed_ref)
repo_name = repo_name_from_packed_ref(packed_ref)
repo = get_github().get_repo(repo_name)
# If the commit is made on the base repo, take the config from the current commit.
# Otherwise, retrieve it from master
clone_commit(
base_repo.clone_url,
sha if repo_clone_url == base_repo.clone_url else
base_repo.get_branch(base_repo.default_branch).commit.sha,
)
for app_name, pr_number in targets:
app = App(app_name)
with tempfile.TemporaryFile("w+") as logs:
try:
with redirect_descriptor(stdout,
logs), redirect_descriptor(
stderr, logs):
land_app(app, pr_number, sha, repo)
if app.config is not None:
update_service_routes([app], pr_number)
except:
traceback.print_exc(file=logs)
logs.seek(0)
report_build_status(
app.name,
pr_number,
pack(repo.clone_url, sha),
BuildStatus.failure,
None,
logs.read(),
private=repo.full_name == base_repo.full_name,
)
else:
logs.seek(0)
report_build_status(
app.name,
pr_number,
pack(repo.clone_url, sha),
BuildStatus.success,
None if app.config is None else ",".join(
hostname.to_str()
for hostname in get_pr_subdomains(app, pr_number)),
logs.read(),
private=repo.full_name == base_repo.full_name,
)
if grouped_targets:
# because we ran a build, we need to clear the queue of anyone we blocked
# we run this in a new worker to avoid timing out
clear_queue(noreply=True)
def clone_repo(repo_str: str):
cloned_repo = g.get_repo(repo_str)
cloned_sha = (sha if cloned_repo.full_name == repo.full_name else
cloned_repo.get_branch(
cloned_repo.default_branch).commit.sha)
clone_commit(cloned_repo.clone_url, cloned_sha, in_place=True)
def land_commit(
sha: str,
repo: Repository,
base_repo: Repository,
pr: Optional[PullRequest],
files: Iterable[Union[File, str]],
*,
target_app: Optional[str] = None,
dequeue_only=False,
):
"""
:param sha: The hash of the commit we are building
:param repo: The repo containing the above commit
:param base_repo: The *base* cs61a-apps repo containing the deploy.yaml config
:param pr: The PR made to trigger the build, if any
:param files: Files changed in the commit, used for target determination
:param target_app: App to rebuild, if not all
:param dequeue_only: Only pop targets off the queue, do not build any new targets
"""
if dequeue_only:
targets = []
elif target_app:
targets = [target_app]
else:
targets = determine_targets(
repo, files if repo.full_name == base_repo.full_name else [])
pr_number = pr.number if pr else 0
grouped_targets = enqueue_builds(targets, pr_number,
pack(repo.clone_url, sha))
for packed_ref, targets in grouped_targets.items():
repo_clone_url, sha = unpack(packed_ref)
# If the commit is made on the base repo, take the config from the current commit.
# Otherwise, retrieve it from master
clone_commit(
base_repo.clone_url,
sha if repo_clone_url == base_repo.clone_url else
base_repo.get_branch(base_repo.default_branch).commit.sha,
)
apps = [App(target) for target in targets]
for app in apps:
with tempfile.TemporaryFile("w+") as logs:
try:
with redirect_descriptor(stdout,
logs), redirect_descriptor(
stderr, logs):
land_app(app, pr_number, sha, repo)
if app.config is not None:
update_service_routes([app], pr_number)
except:
traceback.print_exc(file=logs)
logs.seek(0)
report_build_status(
app.name,
pr_number,
pack(repo.clone_url, sha),
BuildStatus.failure,
None,
logs.read(),
private=repo.full_name == base_repo.full_name,
)
else:
logs.seek(0)
report_build_status(
app.name,
pr_number,
pack(repo.clone_url, sha),
BuildStatus.success,
None if app.config is None else ",".join(
hostname.to_str()
for hostname in get_pr_subdomains(app, pr_number)),
logs.read(),
private=repo.full_name == base_repo.full_name,
)
if grouped_targets:
# because we ran a build, we need to clear the queue of anyone we blocked
# we run this in a new worker to avoid timing out
clear_queue(repo=repo.full_name, pr_number=pr_number, noreply=True)
# this is used to trigger the worker via Cloud Build
import sys
from github import Github
from app_config import App
from build import clone_commit
from common.rpc.secrets import get_secret
from conf import GITHUB_REPO
from worker import land_app_worker
if __name__ == "__main__":
g = Github(get_secret(secret_name="GITHUB_ACCESS_TOKEN"))
_, app_name, pr_number, sha, repo_id = sys.argv
base_repo = g.get_repo(GITHUB_REPO)
clone_commit(
base_repo.clone_url,
sha if repo_id == base_repo.full_name else base_repo.get_branch(
base_repo.default_branch).commit.sha,
)
land_app_worker(App(app_name), int(pr_number), sha, g.get_repo(repo_id))