def build(ctx, c, nocache=False, push=False):
"""
Build container images
"""
# -----------------------
# NOTE - to allow container-specific dockerignore files, we need to switch on DOCKER_BUILDKIT=1
# this might change in future:
#
# https://stackoverflow.com/questions/40904409/how-to-specify-different-dockerignore-files-for-different-builds-in-the-same-pr
# https://github.com/moby/moby/issues/12886#issuecomment-480575928
# -----------------------
shell_env = copy(os.environ)
shell_env["DOCKER_BUILDKIT"] = "1"
_check_valid_containers(c)
version = get_faasm_version()
for container in c:
# Check if we need to template a special dockerignore file
# It seems the dockerignore file needs to be at <dockerfile_path>.dockerignore
dockerfile = join("docker", "{}.dockerfile".format(container))
dockerignore_template_path = join(
"docker", "{}.dockerignore.j2".format(container))
dockerignore_target_path = "{}.dockerignore".format(dockerfile)
if exists(join(PROJ_ROOT, dockerignore_template_path)):
print("Templating new dockerignore file for {}".format(container))
jinja_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(PROJ_ROOT))
template = jinja_env.get_template(dockerignore_template_path)
with open(join(PROJ_ROOT, dockerignore_target_path), "w") as fh:
fh.write(template.render())
tag_name = "faasm/{}:{}".format(container, version)
if nocache:
no_cache_str = "--no-cache"
else:
no_cache_str = ""
faasm_ver = get_faasm_version()
cmd = "docker build {} -t {} --build-arg FAASM_VERSION={} -f {} .".format(
no_cache_str, tag_name, faasm_ver, dockerfile)
print(cmd)
res = call(cmd, shell=True, cwd=PROJ_ROOT, env=shell_env)
if res != 0:
raise RuntimeError("Failed docker build for {}".format(tag_name))
if push:
_do_push(container, version)
def _do_container_build(name, nocache=False, push=False):
tag_name = _get_docker_tag(name)
faasm_ver = get_faasm_version()
if nocache:
no_cache_str = "--no-cache"
else:
no_cache_str = ""
dockerfile = join(PROJ_ROOT, "docker", "{}.dockerfile".format(name))
build_cmd = [
"docker build",
no_cache_str,
"-t {}".format(tag_name),
"-f {}".format(dockerfile),
"--build-arg FAASM_VERSION={}".format(faasm_ver),
".",
]
build_cmd = " ".join(build_cmd)
print(build_cmd)
run(build_cmd, shell=True, check=True, env={"DOCKER_BUILDKIT": "1"})
if push:
_do_push(name)
def copy_release_bundles(ctx, old_ver):
new_ver = get_faasm_version()
print("Copying sysroot {} -> {}".format(old_ver, new_ver))
copy_object_in_s3(MISC_S3_BUCKET,
_get_sysroot_tar_name(old_ver),
_get_sysroot_tar_name(new_ver),
public=True)
print("Copying runtime root {} -> {}".format(old_ver, new_ver))
copy_object_in_s3(MISC_S3_BUCKET,
_get_runtime_tar_name(old_ver),
_get_runtime_tar_name(new_ver),
public=True)
print("Copying toolchain {} -> {}".format(old_ver, new_ver))
copy_object_in_s3(MISC_S3_BUCKET,
_get_toolchain_tar_name(old_ver),
_get_toolchain_tar_name(new_ver),
public=True)
print("\nObjects in bucket:")
objs = list_files_s3(MISC_S3_BUCKET, "")
for o in objs:
print(o)
def docker_push(ctx, c):
version = get_faasm_version()
_check_valid_containers(c)
for container in c:
_do_push(container, version)
def docker_pull(ctx, c):
version = get_faasm_version()
_check_valid_containers(c)
for container in c:
res = call("docker pull faasm/{}:{}".format(container, version),
shell=True,
cwd=PROJ_ROOT)
if res != 0:
raise RuntimeError("Failed docker pull for {}".format(container))
def push(ctx, c):
"""
Push container images
"""
version = get_faasm_version()
_check_valid_containers(c)
for container in c:
_do_push(container, version)
def _do_knative_native_local(img_name):
version = get_faasm_version()
img_name = "{}:{}".format(img_name, version)
cmd = [
"docker", "run", "-p 8080:8080", "--env LOG_LEVEL=debug",
"--env FAASM_INVOKE_HOST=0.0.0.0", "--env FAASM_INVOKE_PORT=8080",
"--env HOST_TYPE=knative", img_name
]
cmd_string = " ".join(cmd)
print(cmd_string)
call(cmd_string, shell=True, cwd=PROJ_ROOT)
def _get_release():
version = get_faasm_version()
r = _get_repo()
rels = r.get_releases()
tag_name = _tag_name(version)
rel = rels[0]
if rel.tag_name != tag_name:
print("Expected latest release to have tag {} but had {}".format(
tag_name, rel.tag_name))
exit(1)
return rel
def _deploy_knative_fn(name,
image,
replicas,
concurrency,
annotations,
extra_env=None,
shell_env=None):
version = get_faasm_version()
image = "{}:{}".format(image, version)
cmd = [
"kn",
"service",
"create",
name,
"--image",
image,
"--namespace",
"faasm",
"--force",
]
cmd.extend({
"--min-scale={}".format(replicas),
"--max-scale={}".format(replicas),
"--concurrency-limit={}".format(concurrency),
})
# Add annotations
for annotation in annotations:
cmd.append("--annotation {}".format(annotation))
# Add standard environment
for key, value in KNATIVE_ENV.items():
cmd.append("--env {}={}".format(key, value))
# Add extra environment
extra_env = extra_env if extra_env else {}
for key, value in extra_env.items():
cmd.append("--env {}={}".format(key, value))
cmd_string = " ".join(cmd)
print(cmd_string)
shell_env_dict = os.environ.copy()
if shell_env:
shell_env_dict.update(shell_env)
call(cmd_string, shell=True, env=shell_env_dict)
def build_knative_native(ctx,
user,
func,
host=False,
clean=False,
nopush=False):
if host:
build_dir = join(PROJ_ROOT, "build", "knative_native")
target = "{}-knative".format(func)
clean_dir(build_dir, clean)
cmd = [
"cmake", "-DCMAKE_CXX_COMPILER=/usr/bin/clang++",
"-DCMAKE_C_COMPILER=/usr/bin/clang",
"-DFAASM_BUILD_TYPE=knative-native", "-DCMAKE_BUILD_TYPE=Debug",
"-DFAASM_AWS_SUPPORT=OFF", PROJ_ROOT
]
call(" ".join(cmd), cwd=build_dir, shell=True)
make_cmd = "cmake --build . --target {} -- -j".format(target)
call(make_cmd, cwd=build_dir, shell=True)
else:
# Build the container
version = get_faasm_version()
tag_name = "{}:{}".format(_native_image_name(func), version)
cmd = [
"docker", "build", "--no-cache" if clean else "", "-t", tag_name,
"--build-arg", "FAASM_VERSION={}".format(version), "--build-arg",
"FAASM_USER={}".format(user), "--build-arg",
"FAASM_FUNC={}".format(func), "-f",
"docker/knative-native.dockerfile", "."
]
env = copy(os.environ)
env["DOCKER_BUILDKIT"] = "1"
cmd_string = " ".join(cmd)
print(cmd_string)
res = call(cmd_string, shell=True, cwd=PROJ_ROOT)
if res != 0:
print("Building container failed")
return 1
# Push the container
if not nopush:
cmd = "docker push {}".format(tag_name)
call(cmd, shell=True, cwd=PROJ_ROOT)
def gh_create_release(ctx):
# Get the head of master
r = _get_repo()
b = r.get_branch(branch="master")
head = b.commit
version = get_faasm_version()
# Create a tag from the head
tag_name = _tag_name(version)
r.create_git_tag(
tag_name,
"Release {}\n".format(version),
head.sha,
"commit",
)
r.create_git_release(tag_name,
"Faasm {}".format(version),
"Release {}\n".format(version),
draft=True)
def _do_deploy_knative_native(func_name, image_name, replicas):
faasm_config = get_faasm_config()
if not faasm_config.has_section("Kubernetes"):
print("Must have faasm config set up with kubernetes section")
return 1
version = get_faasm_version()
# Host and port required for chaining native functions
invoke_host, invoke_port = get_kubernetes_host_port()
_deploy_knative_fn(
func_name,
image_name,
replicas,
1,
NATIVE_WORKER_ANNOTATIONS,
extra_env={
"COLD_START_DELAY_MS": "500",
"FAASM_INVOKE_HOST": invoke_host,
"FAASM_INVOKE_PORT": invoke_port,
},
)
def get_sysroot_tar_name(version=None):
version = version if version else get_faasm_version()
return "faasm-sysroot-{}.tar.gz".format(version)
def _get_tar_name():
ver = get_faasm_version()
return "faasm-runtime-root-{}.tar.gz".format(ver)
def _get_toolchain_tar_name():
ver = get_faasm_version()
return "faasm-toolchain-{}.tar.gz".format(ver)
def _get_artifact_url(name, version=None):
version = version if version else get_faasm_version()
url = "https://github.com/lsds/Faasm/releases/download/v{}/{}".format(version, name)
return url
def get_runtime_tar_name(version=None):
version = version if version else get_faasm_version()
return "faasm-runtime-root-{}.tar.gz".format(version)
def get_toolchain_tar_name(version=None):
version = version if version else get_faasm_version()
return "faasm-toolchain-{}.tar.gz".format(version)
def _get_sysroot_tar_name():
ver = get_faasm_version()
return "faasm-sysroot-{}.tar.gz".format(ver)
