def build(ctx, c, nocache=False, push=False):
"""
Build latest version of container images
"""
# Use buildkit for nicer logging
shell_env = copy(os.environ)
shell_env["DOCKER_BUILDKIT"] = "1"
_check_valid_containers(c)
faasm_ver = get_faasm_version()
for container in c:
# Check if we need to template a special dockerignore file It seems the
# dockerignore file needs to be at <dockerfile_path>.dockerignore
dockerfile = join("docker", "{}.dockerfile".format(container))
tag_name = "faasm/{}:{}".format(container, faasm_ver)
faasm_ver = get_faasm_version()
cmd = "docker build {} -t {} --build-arg FAASM_VERSION={} -f {} .".format(
"--no-cache" if nocache else "", tag_name, faasm_ver, dockerfile
)
print(cmd)
run(cmd, shell=True, check=True, cwd=PROJ_ROOT, env=shell_env)
if push:
_do_push(container, faasm_ver)
def build(ctx, c, nocache=False, push=False):
"""
Build container images
"""
# -----------------------
# NOTE - to allow container-specific dockerignore files, we need to switch on DOCKER_BUILDKIT=1
# this might change in future:
#
# https://stackoverflow.com/questions/40904409/how-to-specify-different-dockerignore-files-for-different-builds-in-the-same-pr
# https://github.com/moby/moby/issues/12886#issuecomment-480575928
# -----------------------
shell_env = copy(os.environ)
shell_env["DOCKER_BUILDKIT"] = "1"
_check_valid_containers(c)
faasm_ver = get_faasm_version()
for container in c:
# Check if we need to template a special dockerignore file
# It seems the dockerignore file needs to be at <dockerfile_path>.dockerignore
dockerfile = join("docker", "{}.dockerfile".format(container))
dockerignore_template_path = join(
"docker", "{}.dockerignore.j2".format(container)
)
dockerignore_target_path = "{}.dockerignore".format(dockerfile)
if exists(join(PROJ_ROOT, dockerignore_template_path)):
print("Templating new dockerignore file for {}".format(container))
jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(PROJ_ROOT))
template = jinja_env.get_template(dockerignore_template_path)
with open(join(PROJ_ROOT, dockerignore_target_path), "w") as fh:
fh.write(template.render())
tag_name = "faasm/{}:{}".format(container, faasm_ver)
if nocache:
no_cache_str = "--no-cache"
else:
no_cache_str = ""
faasm_ver = get_faasm_version()
cmd = "docker build {} -t {} --build-arg FAASM_VERSION={} -f {} .".format(
no_cache_str, tag_name, faasm_ver, dockerfile
)
print(cmd)
res = call(cmd, shell=True, cwd=PROJ_ROOT, env=shell_env)
if res != 0:
raise RuntimeError("Failed docker build for {}".format(tag_name))
if push:
_do_push(container, faasm_ver)
def start(ctx, workers=2, sgx=FAASM_SGX_MODE_DISABLED):
"""
Start the local dev cluster
"""
# This env makes sure we mount our local setup into the containers, rather
# than using the prebuilt binaries
env = copy(os.environ)
env["FAASM_BUILD_DIR"] = join(PROJ_ROOT, "dev/faasm/build")
env["FAASM_BUILD_MOUNT"] = "/build/faasm"
env["FAASM_LOCAL_MOUNT"] = "/usr/local/faasm"
faasm_ver = get_faasm_version()
if sgx == FAASM_SGX_MODE_SIM:
env["FAASM_CLI_IMAGE"] = "faasm/cli-sgx-sim:{}".format(faasm_ver)
env["FAASM_WORKER_IMAGE"] = "faasm/worker-sgx-sim:{}".format(faasm_ver)
env["WASM_VM"] = "sgx"
elif sgx == FAASM_SGX_MODE_HARDWARE:
env["FAASM_CLI_IMAGE"] = "faasm/cli-sgx:{}".format(faasm_ver)
env["FAASM_WORKER_IMAGE"] = "faasm/worker-sgx:{}".format(faasm_ver)
env["WASM_VM"] = "sgx"
cmd = [
"docker compose up -d",
"--scale worker={}".format(workers),
"nginx",
]
cmd = " ".join(cmd)
print(cmd)
run(cmd, shell=True, check=True, cwd=PROJ_ROOT, env=env)
def create_release(ctx):
"""
Create a draft release on Github
"""
# Get the head of master
r = _get_repo()
b = r.get_branch(branch="master")
head = b.commit
faasm_ver = get_faasm_version()
# Create a tag from the head
tag_name = _tag_name(faasm_ver)
r.create_git_tag(
tag_name,
"Release {}\n".format(faasm_ver),
head.sha,
"commit",
)
r.create_git_release(
tag_name,
"Faasm {}".format(faasm_ver),
"Release {}\n".format(faasm_ver),
draft=True,
)
def bump(ctx, ver=None):
"""
Increase the version (defaults to bumping a single minor version)
"""
old_ver = get_faasm_version()
if ver:
new_ver = ver
else:
# Just bump the last minor version part
new_ver_parts = old_ver.split(".")
new_ver_minor = int(new_ver_parts[-1]) + 1
new_ver_parts[-1] = str(new_ver_minor)
new_ver = ".".join(new_ver_parts)
# Replace version in all files
for f in VERSIONED_FILES:
sed_cmd = "sed -i 's/{}/{}/g' {}".format(old_ver, new_ver, f)
run(sed_cmd, shell=True, check=True)
# Replace version in dirs
for d in VERSIONED_DIRS:
sed_cmd = [
"find {}".format(d),
"-type f",
"-exec sed -i -e 's/{}/{}/g'".format(old_ver, new_ver),
"{} \;",
]
sed_cmd = " ".join(sed_cmd)
print(sed_cmd)
run(sed_cmd, shell=True, check=True)
def push(ctx, c):
"""
Push container images
"""
faasm_ver = get_faasm_version()
_check_valid_containers(c)
for container in c:
_do_push(container, faasm_ver)
def _deploy_knative_fn(
name,
image,
replicas,
concurrency,
annotations,
extra_env=None,
shell_env=None,
):
faasm_ver = get_faasm_version()
image = "{}:{}".format(image, faasm_ver)
cmd = [
"kn",
"service",
"create",
name,
"--image",
image,
"--namespace",
"faasm",
"--force",
]
cmd.extend(
{
"--min-scale={}".format(replicas),
"--max-scale={}".format(replicas),
"--concurrency-limit={}".format(concurrency)
if concurrency
else "",
}
)
# Add annotations
for annotation in annotations:
cmd.append("--annotation {}".format(annotation))
# Add standard environment
for key, value in KNATIVE_ENV.items():
cmd.append("--env {}={}".format(key, value))
# Add extra environment
extra_env = extra_env if extra_env else {}
for key, value in extra_env.items():
cmd.append("--env {}={}".format(key, value))
cmd_string = " ".join(cmd)
print(cmd_string)
shell_env_dict = os.environ.copy()
if shell_env:
shell_env_dict.update(shell_env)
call(cmd_string, shell=True, env=shell_env_dict)
def pull(ctx, c):
"""
Pull container images
"""
faasm_ver = get_faasm_version()
_check_valid_containers(c)
for container in c:
res = call("docker pull faasm/{}:{}".format(container, faasm_ver), shell=True, cwd=PROJ_ROOT)
if res != 0:
raise RuntimeError("Failed docker pull for {}".format(container))
def build_native(ctx, user, func, host=False, clean=False, nopush=False):
"""
Build a native Knative container for the given function
"""
if host:
build_dir = join(PROJ_ROOT, "build", "knative_native")
target = "{}-knative".format(func)
clean_dir(build_dir, clean)
cmd = [
"cmake",
"-DCMAKE_CXX_COMPILER=/usr/bin/clang++",
"-DCMAKE_C_COMPILER=/usr/bin/clang",
"-DFAASM_BUILD_TYPE=knative-native",
"-DCMAKE_BUILD_TYPE=Debug",
PROJ_ROOT
]
call(" ".join(cmd), cwd=build_dir, shell=True)
make_cmd = "cmake --build . --target {} -- -j".format(target)
call(make_cmd, cwd=build_dir, shell=True)
else:
# Build the container
version = get_faasm_version()
tag_name = "{}:{}".format(_native_image_name(func), version)
cmd = [
"docker",
"build",
"--no-cache" if clean else "",
"-t", tag_name,
"--build-arg", "FAASM_VERSION={}".format(version),
"--build-arg", "FAASM_USER={}".format(user),
"--build-arg", "FAASM_FUNC={}".format(func),
"-f", "docker/knative-native.dockerfile",
"."
]
env = copy(os.environ)
env["DOCKER_BUILDKIT"] = "1"
cmd_string = " ".join(cmd)
print(cmd_string)
res = call(cmd_string, shell=True, cwd=PROJ_ROOT)
if res != 0:
print("Building container failed")
return 1
# Push the container
if not nopush:
cmd = "docker push {}".format(tag_name)
call(cmd, shell=True, cwd=PROJ_ROOT)
def _get_release():
version = get_faasm_version()
r = _get_repo()
rels = r.get_releases()
tag_name = _tag_name(version)
rel = rels[0]
if rel.tag_name != tag_name:
print("Expected latest release to have tag {} but had {}".format(
tag_name, rel.tag_name))
exit(1)
return rel
def pull(ctx, c):
"""
Pull container images
"""
faasm_ver = get_faasm_version()
_check_valid_containers(c)
for container in c:
run(
"docker pull faasm/{}:{}".format(container, faasm_ver),
shell=True,
check=True,
cwd=PROJ_ROOT,
)
def create(ctx):
"""
Create a draft release on Github
"""
# Work out the tag
faasm_ver = get_faasm_version()
tag_name = _tag_name(faasm_ver)
# Create a release in github from this tag
r = _get_repo()
r.create_git_release(
tag_name,
"Faasm {}".format(faasm_ver),
"Release {}\n".format(faasm_ver),
draft=True,
)
def _do_knative_native_local(img_name):
faasm_ver = get_faasm_version()
img_name = "{}:{}".format(img_name, faasm_ver)
# Run on host network for access to Redis
cmd = [
"docker", "run", "--network=host", "--env LOG_LEVEL=debug",
"--env STATE_MODE=redis", "--env FAASM_INVOKE_HOST=127.0.0.1",
"--env FAASM_INVOKE_PORT=8080", "--env HOST_TYPE=knative",
"--env FUNCTION_STORAGE=fileserver",
"--env FILESERVER_URL=http://127.0.0.1:8002", img_name
]
cmd_string = " ".join(cmd)
print(cmd_string)
call(cmd_string, shell=True, cwd=PROJ_ROOT)
def build(ctx, c, nocache=False, push=False):
"""
Build latest version of container images
"""
# Use buildkit for nicer logging
shell_env = copy(os.environ)
shell_env["DOCKER_BUILDKIT"] = "1"
_check_valid_containers(c)
faasm_ver = get_faasm_version()
for container_name in c:
# Prepare dockerfile and tag name
dockerfile = join("docker", CONTAINER_NAME2FILE_MAP[container_name])
tag_name = "faasm/{}:{}".format(container_name, faasm_ver)
# Prepare build arguments
build_args = {"FAASM_VERSION": faasm_ver}
if container_name.endswith(SGX_HW_CONTAINER_SUFFIX):
build_args["FAASM_SGX_MODE"] = FAASM_SGX_MODE_HARDWARE
build_args["FAASM_SGX_PARENT_SUFFIX"] = SGX_HW_CONTAINER_SUFFIX
elif container_name.endswith(SGX_SIMULATION_CONTAINER_SUFFIX):
build_args["FAASM_SGX_MODE"] = FAASM_SGX_MODE_SIM
build_args[
"FAASM_SGX_PARENT_SUFFIX"] = SGX_SIMULATION_CONTAINER_SUFFIX
else:
build_args["FAASM_SGX_MODE"] = FAASM_SGX_MODE_DISABLED
# Prepare docker command
cmd = [
"docker build {}".format("--no-cache" if nocache else ""),
"-t {}".format(tag_name),
"{}".format(" ".join([
"--build-arg {}={}".format(arg, build_args[arg])
for arg in build_args
])),
"-f {} .".format(dockerfile),
]
docker_cmd = " ".join(cmd)
print(docker_cmd)
# Build (and push) docker image
run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT, env=shell_env)
if push:
_do_push(container_name, faasm_ver)
def _do_knative_native_local(img_name):
version = get_faasm_version()
img_name = "{}:{}".format(img_name, version)
cmd = [
"docker", "run",
"-p 8080:8080",
"--env LOG_LEVEL=debug",
"--env FAASM_INVOKE_HOST=0.0.0.0",
"--env FAASM_INVOKE_PORT=8080",
"--env HOST_TYPE=knative",
img_name
]
cmd_string = " ".join(cmd)
print(cmd_string)
call(cmd_string, shell=True, cwd=PROJ_ROOT)
def delete_old(ctx):
"""
Deletes old Docker images
"""
faasm_ver = get_faasm_version()
dock = docker.from_env()
images = dock.images.list()
for image in images:
for t in image.tags:
if not t.startswith("faasm/"):
continue
tag_ver = t.split(":")[-1]
if version.parse(tag_ver) < version.parse(faasm_ver):
print("Removing old image: {}".format(t))
dock.images.remove(t, force=True)
def get_release_body():
"""
Generate body for release with detailed changelog
"""
docker_cmd = [
"docker run -t -v",
"{}:/app/".format(PROJ_ROOT),
"orhunp/git-cliff:latest",
"--config cliff.toml",
"--repository .",
"{}..v{}".format(_get_release().tag_name, get_faasm_version()),
]
cmd = " ".join(docker_cmd)
print("Generating release body...")
print(cmd)
result = run(cmd, shell=True, stdout=PIPE, stderr=PIPE)
return result.stdout.decode("utf-8")
def _get_artifact_url(name, version=None):
version = version if version else get_faasm_version()
url = "https://github.com/lsds/Faasm/releases/download/v{}/{}".format(
version, name)
return url
def get_runtime_tar_name(version=None):
version = version if version else get_faasm_version()
return "faasm-runtime-root-{}.tar.gz".format(version)
def get_sysroot_tar_name(version=None):
version = version if version else get_faasm_version()
return "faasm-sysroot-{}.tar.gz".format(version)
def get_toolchain_tar_name(version=None):
version = version if version else get_faasm_version()
return "faasm-toolchain-{}.tar.gz".format(version)
def _get_tag():
faasm_ver = get_faasm_version()
tag_name = _tag_name(faasm_ver)
return tag_name
