def build(ctx, c, nocache=False, push=False): """ Build container images """ # ----------------------- # NOTE - to allow container-specific dockerignore files, we need to switch on DOCKER_BUILDKIT=1 # this might change in future: # # https://stackoverflow.com/questions/40904409/how-to-specify-different-dockerignore-files-for-different-builds-in-the-same-pr # https://github.com/moby/moby/issues/12886#issuecomment-480575928 # ----------------------- shell_env = copy(os.environ) shell_env["DOCKER_BUILDKIT"] = "1" _check_valid_containers(c) version = get_faasm_version() for container in c: # Check if we need to template a special dockerignore file # It seems the dockerignore file needs to be at <dockerfile_path>.dockerignore dockerfile = join("docker", "{}.dockerfile".format(container)) dockerignore_template_path = join( "docker", "{}.dockerignore.j2".format(container)) dockerignore_target_path = "{}.dockerignore".format(dockerfile) if exists(join(PROJ_ROOT, dockerignore_template_path)): print("Templating new dockerignore file for {}".format(container)) jinja_env = jinja2.Environment( loader=jinja2.FileSystemLoader(PROJ_ROOT)) template = jinja_env.get_template(dockerignore_template_path) with open(join(PROJ_ROOT, dockerignore_target_path), "w") as fh: fh.write(template.render()) tag_name = "faasm/{}:{}".format(container, version) if nocache: no_cache_str = "--no-cache" else: no_cache_str = "" faasm_ver = get_faasm_version() cmd = "docker build {} -t {} --build-arg FAASM_VERSION={} -f {} .".format( no_cache_str, tag_name, faasm_ver, dockerfile) print(cmd) res = call(cmd, shell=True, cwd=PROJ_ROOT, env=shell_env) if res != 0: raise RuntimeError("Failed docker build for {}".format(tag_name)) if push: _do_push(container, version)
def _do_container_build(name, nocache=False, push=False): tag_name = _get_docker_tag(name) faasm_ver = get_faasm_version() if nocache: no_cache_str = "--no-cache" else: no_cache_str = "" dockerfile = join(PROJ_ROOT, "docker", "{}.dockerfile".format(name)) build_cmd = [ "docker build", no_cache_str, "-t {}".format(tag_name), "-f {}".format(dockerfile), "--build-arg FAASM_VERSION={}".format(faasm_ver), ".", ] build_cmd = " ".join(build_cmd) print(build_cmd) run(build_cmd, shell=True, check=True, env={"DOCKER_BUILDKIT": "1"}) if push: _do_push(name)
def copy_release_bundles(ctx, old_ver): new_ver = get_faasm_version() print("Copying sysroot {} -> {}".format(old_ver, new_ver)) copy_object_in_s3(MISC_S3_BUCKET, _get_sysroot_tar_name(old_ver), _get_sysroot_tar_name(new_ver), public=True) print("Copying runtime root {} -> {}".format(old_ver, new_ver)) copy_object_in_s3(MISC_S3_BUCKET, _get_runtime_tar_name(old_ver), _get_runtime_tar_name(new_ver), public=True) print("Copying toolchain {} -> {}".format(old_ver, new_ver)) copy_object_in_s3(MISC_S3_BUCKET, _get_toolchain_tar_name(old_ver), _get_toolchain_tar_name(new_ver), public=True) print("\nObjects in bucket:") objs = list_files_s3(MISC_S3_BUCKET, "") for o in objs: print(o)
def docker_push(ctx, c): version = get_faasm_version() _check_valid_containers(c) for container in c: _do_push(container, version)
def docker_pull(ctx, c): version = get_faasm_version() _check_valid_containers(c) for container in c: res = call("docker pull faasm/{}:{}".format(container, version), shell=True, cwd=PROJ_ROOT) if res != 0: raise RuntimeError("Failed docker pull for {}".format(container))
def push(ctx, c): """ Push container images """ version = get_faasm_version() _check_valid_containers(c) for container in c: _do_push(container, version)
def _do_knative_native_local(img_name): version = get_faasm_version() img_name = "{}:{}".format(img_name, version) cmd = [ "docker", "run", "-p 8080:8080", "--env LOG_LEVEL=debug", "--env FAASM_INVOKE_HOST=0.0.0.0", "--env FAASM_INVOKE_PORT=8080", "--env HOST_TYPE=knative", img_name ] cmd_string = " ".join(cmd) print(cmd_string) call(cmd_string, shell=True, cwd=PROJ_ROOT)
def _get_release(): version = get_faasm_version() r = _get_repo() rels = r.get_releases() tag_name = _tag_name(version) rel = rels[0] if rel.tag_name != tag_name: print("Expected latest release to have tag {} but had {}".format( tag_name, rel.tag_name)) exit(1) return rel
def _deploy_knative_fn(name, image, replicas, concurrency, annotations, extra_env=None, shell_env=None): version = get_faasm_version() image = "{}:{}".format(image, version) cmd = [ "kn", "service", "create", name, "--image", image, "--namespace", "faasm", "--force", ] cmd.extend({ "--min-scale={}".format(replicas), "--max-scale={}".format(replicas), "--concurrency-limit={}".format(concurrency), }) # Add annotations for annotation in annotations: cmd.append("--annotation {}".format(annotation)) # Add standard environment for key, value in KNATIVE_ENV.items(): cmd.append("--env {}={}".format(key, value)) # Add extra environment extra_env = extra_env if extra_env else {} for key, value in extra_env.items(): cmd.append("--env {}={}".format(key, value)) cmd_string = " ".join(cmd) print(cmd_string) shell_env_dict = os.environ.copy() if shell_env: shell_env_dict.update(shell_env) call(cmd_string, shell=True, env=shell_env_dict)
def build_knative_native(ctx, user, func, host=False, clean=False, nopush=False): if host: build_dir = join(PROJ_ROOT, "build", "knative_native") target = "{}-knative".format(func) clean_dir(build_dir, clean) cmd = [ "cmake", "-DCMAKE_CXX_COMPILER=/usr/bin/clang++", "-DCMAKE_C_COMPILER=/usr/bin/clang", "-DFAASM_BUILD_TYPE=knative-native", "-DCMAKE_BUILD_TYPE=Debug", "-DFAASM_AWS_SUPPORT=OFF", PROJ_ROOT ] call(" ".join(cmd), cwd=build_dir, shell=True) make_cmd = "cmake --build . --target {} -- -j".format(target) call(make_cmd, cwd=build_dir, shell=True) else: # Build the container version = get_faasm_version() tag_name = "{}:{}".format(_native_image_name(func), version) cmd = [ "docker", "build", "--no-cache" if clean else "", "-t", tag_name, "--build-arg", "FAASM_VERSION={}".format(version), "--build-arg", "FAASM_USER={}".format(user), "--build-arg", "FAASM_FUNC={}".format(func), "-f", "docker/knative-native.dockerfile", "." ] env = copy(os.environ) env["DOCKER_BUILDKIT"] = "1" cmd_string = " ".join(cmd) print(cmd_string) res = call(cmd_string, shell=True, cwd=PROJ_ROOT) if res != 0: print("Building container failed") return 1 # Push the container if not nopush: cmd = "docker push {}".format(tag_name) call(cmd, shell=True, cwd=PROJ_ROOT)
def gh_create_release(ctx): # Get the head of master r = _get_repo() b = r.get_branch(branch="master") head = b.commit version = get_faasm_version() # Create a tag from the head tag_name = _tag_name(version) r.create_git_tag( tag_name, "Release {}\n".format(version), head.sha, "commit", ) r.create_git_release(tag_name, "Faasm {}".format(version), "Release {}\n".format(version), draft=True)
def _do_deploy_knative_native(func_name, image_name, replicas): faasm_config = get_faasm_config() if not faasm_config.has_section("Kubernetes"): print("Must have faasm config set up with kubernetes section") return 1 version = get_faasm_version() # Host and port required for chaining native functions invoke_host, invoke_port = get_kubernetes_host_port() _deploy_knative_fn( func_name, image_name, replicas, 1, NATIVE_WORKER_ANNOTATIONS, extra_env={ "COLD_START_DELAY_MS": "500", "FAASM_INVOKE_HOST": invoke_host, "FAASM_INVOKE_PORT": invoke_port, }, )
def get_sysroot_tar_name(version=None): version = version if version else get_faasm_version() return "faasm-sysroot-{}.tar.gz".format(version)
def _get_tar_name(): ver = get_faasm_version() return "faasm-runtime-root-{}.tar.gz".format(ver)
def _get_toolchain_tar_name(): ver = get_faasm_version() return "faasm-toolchain-{}.tar.gz".format(ver)
def _get_artifact_url(name, version=None): version = version if version else get_faasm_version() url = "https://github.com/lsds/Faasm/releases/download/v{}/{}".format(version, name) return url
def get_runtime_tar_name(version=None): version = version if version else get_faasm_version() return "faasm-runtime-root-{}.tar.gz".format(version)
def get_toolchain_tar_name(version=None): version = version if version else get_faasm_version() return "faasm-toolchain-{}.tar.gz".format(version)
def _get_sysroot_tar_name(): ver = get_faasm_version() return "faasm-sysroot-{}.tar.gz".format(ver)