def run_highcpu_build(
app: App,
pr_number: int,
sha: str,
repo: Repository,
):
for f in os.listdir("./deploy_files"):
if f == "cloudbuild.yaml":
# don't use this one, we don't want any caching
continue
shutil.copyfile(f"./deploy_files/{f}", f"./{f}")
shutil.copyfile("./dockerfiles/buildserver.Dockerfile", "./Dockerfile")
with open("Dockerfile", "a+") as f:
f.seek(0)
contents = f.read()
contents = contents.replace("$BASE_IMAGE", app.config["build_image"])
contents = contents.replace("$APP_NAME", app.name)
contents = contents.replace("$PR_NUMBER", str(pr_number))
contents = contents.replace("$SHA", sha)
contents = contents.replace("$REPO_ID", repo.full_name)
contents = contents.replace("$MASTER_SECRET", get_master_secret())
f.seek(0)
f.truncate()
f.write(contents)
sh(
"gcloud",
"builds",
"submit",
"-q",
"--tag",
"gcr.io/cs61a-140900/temp-{}".format(
gen_service_name(app.name, pr_number)),
# "--machine-type=N1_HIGHCPU_32",
)
def hash_all(show_progress=False):
global tracked_files
files = (
sh("git",
"ls-files",
"--exclude-standard",
capture_output=True,
quiet=True).splitlines() # All tracked files
+ sh(
"git",
"ls-files",
"-o",
"--exclude-standard",
capture_output=True,
quiet=True,
).splitlines() # Untracked but not ignored files
)
out = {}
for file in tqdm(files) if show_progress else files:
h = get_hash(file)
if isinstance(file, bytes):
file = file.decode("ascii")
out[relpath(file)] = h
tracked_files = set(out) | set(remote_state)
return out
def main():
"""Start the Sandbox and IDE servers."""
sh("nginx")
sandbox_port = get_open_port()
sb = subprocess.Popen(
["gunicorn", "-b", f":{sandbox_port}", "-w", "4", "sandbox:app", "-t", "3000"],
env=os.environ,
)
proxy(f"sb.{HOSTNAME} *.sb.{HOSTNAME}", sandbox_port, f"sb.{HOSTNAME}")
proxy(f"*.sb.pr.{HOSTNAME}", sandbox_port, f"sb.pr.{HOSTNAME}")
ide_port = get_open_port()
ide = subprocess.Popen(
["gunicorn", "-b", f":{ide_port}", "-w", "4", "ide:app", "-t", "3000"],
env=os.environ,
)
proxy(f"ide.{HOSTNAME}", ide_port, f"ide.{HOSTNAME}")
proxy(f"*.ide.pr.{HOSTNAME}", ide_port, f"ide.pr.{HOSTNAME}")
with open(f"/etc/nginx/sites-enabled/default", "w") as f:
f.write(
DEFAULT_SERVER.format(
ide_port=ide_port, sb_port=sandbox_port, nginx_port=NGINX_PORT
)
)
sh("nginx", "-s", "reload")
ide.communicate() # make sure docker doesn't close this container
def run_service_deploy(app: App, pr_number: int):
if pr_number != 0:
return # do not deploy PR builds to prod!
for file in os.listdir("."):
sh(
"gcloud",
"compute",
"scp",
"--recurse",
file,
app.config["service"]["host"] + ":" +
app.config["service"]["root"],
"--zone",
app.config["service"]["zone"],
)
sh(
"gcloud",
"compute",
"ssh",
app.config["service"]["host"],
"--command=sudo systemctl restart {}".format(
app.config["service"]["name"]),
"--zone",
app.config["service"]["zone"],
)
def kill():
username = request.form.get("username")
pid = get_server_pid(username)
if pid:
sh("kill", pid.decode("utf-8")[:-1])
sh("sleep", "2") # give the server a couple of seconds to shutdown
return redirect(session.pop(SK_RETURN_TO, url_for("index")))
def run_static_deploy(app: App, pr_number: int):
bucket = f"gs://{gen_service_name(app.name, pr_number)}.buckets.cs61a.org"
prod_bucket = f"gs://{gen_service_name(app.name, 0)}.buckets.cs61a.org"
try:
sh("gsutil", "mb", "-b", "on", bucket)
except CalledProcessError:
# bucket already exists
pass
sh("gsutil", "-m", "rsync", "-dRc", ".", bucket)
def initialize_sandbox(force=False):
with db_lock("sandboxes", g.username):
initialized = check_sandbox_initialized(g.username)
if initialized and not force:
raise Exception("Sandbox is already initialized")
elif initialized:
sh("rm", "-rf", get_working_directory(g.username))
Path(get_working_directory(g.username)).mkdir(parents=True,
exist_ok=True)
os.chdir(get_working_directory(g.username))
sh("git", "init")
sh(
"git",
"fetch",
"--depth=1",
f"https://{get_secret(secret_name='GITHUB_ACCESS_TOKEN')}@github.com/{REPO}",
"master",
)
sh("git", "checkout", "FETCH_HEAD", "-f")
os.mkdir("published") # needed for lazy-loading builds
if is_prod_build():
add_domain(name="sandbox", domain=f"{g.username}.sb.cs61a.org")
with connect_db() as db:
db("UPDATE sandboxes SET initialized=TRUE WHERE username=%s",
[g.username])
def build_worker(username):
# Note that we are not necessarily running in an app context
try:
while True:
targets = get_pending_targets(username)
if not targets:
break
target = targets[0]
src_version = get_src_version(username)
os.chdir(get_working_directory(username))
os.chdir("src")
ok = False
try:
sh("make", "-n", "VIRTUAL_ENV=../env", target, env=ENV)
except CalledProcessError as e:
if e.returncode == 2:
# target does not exist, no need to build
update_version(username, target, src_version)
ok = True
if not ok:
# target exists, time to build!
try:
sh(
"make",
"VIRTUAL_ENV=../env",
target,
env={
**ENV, "LAZY_LOADING": "true"
},
capture_output=True,
quiet=True,
)
except CalledProcessError as e:
log_name = paste_text(
data=((e.stdout or b"").decode("utf-8") +
(e.stderr or b"").decode("utf-8")))
update_version(username, target, src_version, log_name)
else:
update_version(username, target, src_version)
with connect_db() as db:
db(
"UPDATE builds SET pending=FALSE WHERE username=%s AND target=%s",
[username, target],
)
except:
# in the event of failure, cancel all builds and trigger refresh
increment_manual_version(username)
clear_pending_builds(username)
raise
def run_pypi_deploy(app: App, pr_number: int):
sh("python", "-m", "venv", "env")
update_setup_py(app, pr_number)
sh("env/bin/pip", "install", "setuptools")
sh("env/bin/pip", "install", "wheel")
sh("env/bin/python", "setup.py", "sdist", "bdist_wheel")
sh(
"twine",
"upload",
*(f"dist/{file}" for file in os.listdir("dist")),
env=dict(
TWINE_USERNAME="******",
TWINE_PASSWORD=get_secret(secret_name="PYPI_PASSWORD"),
),
)
def create_secret(service):
if not is_staff("cs61a"):
return login()
email = get_user()["email"]
if not is_admin(course="cs61a", email=email):
abort(401)
if service not in list_services():
abort(404)
out = reversed([
entry["timestamp"] + " " + escape(entry["textPayload"])
for entry in loads(
sh(
"gcloud",
"logging",
"read",
f"projects/cs61a-140900/logs/run.googleapis.com AND resource.labels.service_name={service}",
"--limit",
"100",
"--format",
"json",
capture_output=True,
)) if "textPayload" in entry
])
return "<pre>" + "\n".join(map(str, out)) + "</pre>"
def get_pr_subdomains(app: App, pr_number: int) -> List[Hostname]:
services = json.loads(
sh(
"gcloud",
"run",
"services",
"list",
"--platform",
"managed",
"--format",
"json",
capture_output=True,
))
def get_hostname(service_name):
for service in services:
if service["metadata"]["name"] == service_name:
return urlparse(service["status"]["address"]["url"]).netloc
return None
out = []
if app.config["deploy_type"] in CLOUD_RUN_DEPLOY_TYPES:
hostname = get_hostname(gen_service_name(app.name, pr_number))
assert hostname is not None
for pr_consumer in app.config["pr_consumers"]:
out.append(PRHostname(pr_consumer, pr_number, hostname))
elif app.config["deploy_type"] == "static":
for consumer in app.config["static_consumers"]:
hostname = get_hostname(gen_service_name(consumer, pr_number))
if hostname is None:
# consumer does not have a PR build, point to master build
hostname = get_hostname(gen_service_name(consumer, 0))
assert hostname is not None, "Invalid static resource consumer service"
out.append(PRHostname(consumer, pr_number, hostname))
if not app.config["static_consumers"]:
out.append(
PRHostname(
app.name,
pr_number,
get_hostname(gen_service_name(STATIC_SERVER, 0)),
))
elif app.config["deploy_type"] == "hosted":
for pr_consumer in app.config["pr_consumers"]:
out.append(
PRHostname(
pr_consumer,
pr_number,
f"{gen_service_name(app.name, pr_number)}.hosted.cs61a.org",
))
elif app.config["deploy_type"] == "pypi":
out.append(
PyPIHostname(app.config["package_name"],
app.deployed_pypi_version))
elif app.config["deploy_type"] in NO_PR_BUILD_DEPLOY_TYPES:
pass
else:
assert False, "Unknown deploy type, failed to create PR domains"
return out
def run_cloud_function_deploy(app: App, pr_number: int):
if pr_number != 0:
return
sh(
"gcloud",
"functions",
"deploy",
app.name,
"--runtime",
"python37",
"--trigger-http",
"--entry-point",
"index",
"--timeout",
"500",
)
def get_server_pid(username):
try:
return sh("pgrep",
"-f",
" ".join(get_server_cmd(username)),
capture_output=True)
except subprocess.CalledProcessError:
return False
def get_repo_files() -> List[str]:
return [
file.decode("ascii") if isinstance(file, bytes) else file
for file in sh("git",
"ls-files",
"--exclude-standard",
capture_output=True,
quiet=True).splitlines() # All tracked files
+ sh(
"git",
"ls-files",
"-o",
"--exclude-standard",
capture_output=True,
quiet=True,
).splitlines() # Untracked but not ignored files
]
def __init__(self, working_dir: str):
self.working_dir = working_dir
self.file_versions = {}
self.file_cnt = 0
self.rule_cnt = 0
self.sh_cnt = 0
self.rules = []
self.actions = {}
self.is_annotating = True
mkdir(self.BUILD_DIRECTORY)
sh("git", "init", cwd=self.BUILD_DIRECTORY)
self._write_file("WORKSPACE")
self.log(f"STARTING TEST")
def find_target():
if not hasattr(find_target, "out"):
remote = sh(
"git",
"config",
"--get",
"remote.origin.url",
capture_output=True,
quiet=True,
).decode("utf-8")
if REPO not in remote:
raise Exception(
"You must run this command in the berkeley-cs61a repo directory"
)
find_target.out = (sh("git",
"rev-parse",
"--show-toplevel",
capture_output=True,
quiet=True).decode("utf-8").strip())
return find_target.out
def create_pr_subdomain(app, pr_number, pr_host):
target_domain = f"{pr_number}.{app}.pr.cs61a.org"
conf_path = f"{pr_confs}/{target_domain}.conf"
expected_cert_name = f"*.{app}.pr.cs61a.org"
nginx_config = Server(
Location(
"/",
proxy_pass=f"https://{pr_host}/",
proxy_read_timeout="1800",
proxy_connect_timeout="1800",
proxy_send_timeout="1800",
send_timeout="1800",
proxy_set_header={
"Host": pr_host,
"X-Forwarded-For-Host": target_domain,
},
),
server_name=target_domain,
listen="80",
)
if not os.path.exists(conf_path):
with open(conf_path, "w") as f:
f.write(str(nginx_config))
sh("nginx", "-s", "reload")
cert = proxy_cb.cert_else_false(expected_cert_name, force_exact=True)
for _ in range(2):
if cert:
break
proxy_cb.run_bot(domains=[expected_cert_name], args=["certonly"])
cert = proxy_cb.cert_else_false(expected_cert_name, force_exact=True)
if not cert:
error = f"Hosted Apps failed to sign a certificate for {expected_cert_name}!"
post_message(message=error, channel="infra")
return dict(success=False, reason=error)
proxy_cb.attach_cert(cert, target_domain)
return dict(success=True)
def run_61a_website_build():
env = dict(CLOUD_STORAGE_BUCKET="website-pdf-cache.buckets.cs61a.org", )
def build(target):
# need to re-run make for stupid reasons
out = sh(
"make",
"--no-print-directory",
"-C",
"src",
target,
env=env,
capture_output=True,
)
print(out.decode("utf-8", "replace"))
build("all")
sh("cp", "-aT", "published", "released")
build("unreleased")
sh("cp", "-aT", "published", "unreleased")
clean_all_except(["released", "unreleased"])
def build(target):
# need to re-run make for stupid reasons
out = sh(
"make",
"--no-print-directory",
"-C",
"src",
target,
env=env,
capture_output=True,
)
print(out.decode("utf-8", "replace"))
def run_sphinx_build():
sh("python3", "-m", "venv", "env")
sh("env/bin/pip", "install", "-r", "requirements.txt")
sh("env/bin/sphinx-build", "-b", "dirhtml", "..", "_build")
clean_all_except(["_build"])
copytree("_build", ".", dirs_exist_ok=True)
rmtree("_build")
def build_docker_image(app: App, pr_number: int) -> str:
for f in os.listdir("../../deploy_files"):
shutil.copyfile(f"../../deploy_files/{f}", f"./{f}")
service_name = gen_service_name(app.name, pr_number)
prod_service_name = gen_service_name(app.name, 0)
with open("cloudbuild.yaml", "a+") as f:
f.seek(0)
contents = f.read()
contents = contents.replace("PROD_SERVICE_NAME", prod_service_name)
contents = contents.replace("SERVICE_NAME", service_name)
f.seek(0)
f.truncate()
f.write(contents)
with open("Dockerfile", "a+") as f:
f.seek(0)
contents = f.read()
contents = contents.replace("<APP_MASTER_SECRET>",
gen_master_secret(app, pr_number))
f.seek(0)
f.truncate()
f.write(contents)
sh("gcloud", "builds", "submit", "-q", "--config", "cloudbuild.yaml")
return f"gcr.io/{PROJECT_ID}/{service_name}"
def venv(dir, req, reset):
"""Create a virtual environment in DIR.
DIR is the location of the virtual env
(minus the `env` folder itself). REQ is
the location of the requirements file.
Both of these arguments default to `./`.
If you want to forcibly recreate an env,
use the RESET option.
"""
if not dir.endswith("/"):
dir = dir + "/"
if req.endswith("requirements.txt"):
req = req[:-16]
if not req.endswith("/"):
req = req + "/"
if os.path.exists(f"{dir}{ENV}"):
if reset:
shutil.rmtree(f"{dir}{ENV}")
else:
print("This environment already exists!")
return
sh("python3", "-m", "venv", f"{dir}{ENV}")
sh(f"{dir}{ENV}/bin/pip3", "install", "-r", f"{req}{REQ}")
def clone():
sh("git", "init")
sh(
"git",
"fetch",
"--depth=1",
f"https://{get_secret(secret_name='GITHUB_ACCESS_TOKEN')}@github.com{path}",
sha,
)
sh("git", "checkout", "FETCH_HEAD", "-f")
def get_base_hostname(app: str) -> str:
services = json.loads(
sh(
"gcloud",
"run",
"services",
"list",
"--platform",
"managed",
"--format",
"json",
capture_output=True,
))
for service in services:
if service["metadata"]["name"] == gen_service_name(app, 0):
return urlparse(service["status"]["address"]["url"]).netloc
raise KeyError
def run_make_command(target):
os.chdir(get_working_directory(g.username))
os.chdir("src")
clear_pending_builds(g.username)
try:
yield from sh(
"make",
"VIRTUAL_ENV=../env",
target,
env=ENV,
stream_output=True,
shell=True,
)
finally:
increment_manual_version(g.username)
def list_services():
return [
service["metadata"]["name"] for service in loads(
sh(
"gcloud",
"run",
"services",
"list",
"--platform",
"managed",
"--region",
"us-west1",
"--format",
"json",
"-q",
capture_output=True,
))
]
def list_services():
"""Returns the list of services from Google Cloud Run necessary to access app logs
:return: list of services
"""
return [
service["metadata"]["name"] for service in loads(
sh(
"gcloud",
"run",
"services",
"list",
"--platform",
"managed",
"--region",
"us-west1",
"--format",
"json",
"-q",
capture_output=True,
))
]
def build(self, rule: Rule):
if not self.is_annotating:
self.log("")
self.log(f"BUILDING rule {rule.name}")
contents = "\n\n".join(
rule.to_declaration(self) for rule in self.rules) + "\n"
with open(self.INPUT_PATH, "w") as f:
f.write(
repr({
action.shell_id: dict(
inputs=[file.path for file in action.inputs],
data=action.data,
)
for action, _ in self.actions.items()
}))
self._write_file("BUILD", contents)
try:
output = sh(
"bt",
f":{rule.name}",
"-q",
quiet=True,
cwd=self.BUILD_DIRECTORY,
capture_output=True,
).decode("utf-8")
except CalledProcessError as e:
print(e.stdout.decode("utf-8"))
raise
if self.FAILURE in output:
raise Exception("Unexpected scenario encountered")
self.log(f"COMPLETED BUILDING rule {rule.name}\n")
self.is_annotating = True
def gen_service_account(app: App):
# set up and create service account
hashstate = HashState()
permissions = sorted(app.config["permissions"])
hashstate.record(permissions)
service_account_name = f"managed-{hashstate.state()}"[:
30] # max len of account ID is 30 chars
service_account_email = (
f"{service_account_name}@{PROJECT_ID}.iam.gserviceaccount.com")
existing_accounts = json.loads(
sh(
"gcloud",
"iam",
"service-accounts",
"list",
"--format",
"json",
capture_output=True,
))
for account in existing_accounts:
if account["email"] == service_account_email:
break
else:
# need to create service account
sh(
"gcloud",
"iam",
"service-accounts",
"create",
service_account_name,
f"--description",
f'Managed service account with permissions: {" ".join(permissions)}',
"--display-name",
"Managed service account - DO NOT EDIT MANUALLY",
)
sleep(60) # it takes a while to create service accounts
role_lookup = dict(
# permissions that most apps might need
storage="roles/storage.admin",
database="roles/cloudsql.client",
logging="roles/logging.admin",
# only buildserver needs these
iam_admin="roles/resourcemanager.projectIamAdmin",
cloud_run_admin="roles/run.admin",
cloud_functions_admin="roles/cloudfunctions.admin",
)
for permission in permissions:
if permission == "rpc":
pass # handled later
else:
role = role_lookup[permission]
try:
sh(
"gcloud",
"projects",
"add-iam-policy-binding",
PROJECT_ID,
f"--member",
f"serviceAccount:{service_account_email}",
f"--role",
role,
)
except CalledProcessError:
# abort
sh(
"gcloud",
"iam",
"service-accounts",
"delete",
service_account_email,
)
raise
return service_account_name
def run_dockerfile_deploy(app: App, pr_number: int):
image = build_docker_image(app, pr_number)
service_name = gen_service_name(app.name, pr_number)
if app.name == "buildserver":
# we exempt buildserver to avoid breaking CI/CD in case of bugs
service_account = None
else:
service_account = gen_service_account(app)
sh(
"gcloud",
"beta",
"run",
"deploy",
service_name,
"--image",
image,
*(("--service-account", service_account) if service_account else ()),
"--region",
"us-west1",
"--platform",
"managed",
"--timeout",
"45m",
"--cpu",
str(app.config["cpus"]),
"--memory",
app.config["memory_limit"],
"--concurrency",
str(app.config["concurrency"]),
"--allow-unauthenticated",
"--add-cloudsql-instances",
DB_INSTANCE_NAME,
"--update-env-vars",
",".join(f"{key}={val}"
for key, val in gen_env_variables(app, pr_number).items()),
"-q",
)
if pr_number == 0:
domains = json.loads(
sh(
"gcloud",
"beta",
"run",
"domain-mappings",
"list",
"--platform",
"managed",
"--region",
"us-west1",
"--format",
"json",
capture_output=True,
))
for domain in app.config["first_party_domains"]:
for domain_config in domains:
if domain_config["metadata"]["name"] == domain:
break
else:
sh(
"gcloud",
"beta",
"run",
"domain-mappings",
"create",
"--service",
service_name,
"--domain",
domain,
"--platform",
"managed",
"--region",
"us-west1",
)
jobs = json.loads(
sh(
"gcloud",
"scheduler",
"jobs",
"list",
"-q",
"--format=json",
capture_output=True,
))
for job in jobs:
name = job["name"].split("/")[-1]
if name.startswith(f"{app}-"):
sh("gcloud", "scheduler", "jobs", "delete", name, "-q")
for job in app.config["tasks"]:
sh(
"gcloud",
"beta",
"scheduler",
"jobs",
"create",
"http",
f"{app}-{job['name']}",
f"--schedule={job['schedule']}",
f"--uri=https://{app}.cs61a.org/jobs/{job['name']}",
"--attempt-deadline=1200s",
"-q",
)