def build(ctx, c, nocache=False, push=False): """ Build latest version of container images """ # Use buildkit for nicer logging shell_env = copy(os.environ) shell_env["DOCKER_BUILDKIT"] = "1" _check_valid_containers(c) faasm_ver = get_faasm_version() for container in c: # Check if we need to template a special dockerignore file It seems the # dockerignore file needs to be at <dockerfile_path>.dockerignore dockerfile = join("docker", "{}.dockerfile".format(container)) tag_name = "faasm/{}:{}".format(container, faasm_ver) faasm_ver = get_faasm_version() cmd = "docker build {} -t {} --build-arg FAASM_VERSION={} -f {} .".format( "--no-cache" if nocache else "", tag_name, faasm_ver, dockerfile ) print(cmd) run(cmd, shell=True, check=True, cwd=PROJ_ROOT, env=shell_env) if push: _do_push(container, faasm_ver)
def build(ctx, c, nocache=False, push=False): """ Build container images """ # ----------------------- # NOTE - to allow container-specific dockerignore files, we need to switch on DOCKER_BUILDKIT=1 # this might change in future: # # https://stackoverflow.com/questions/40904409/how-to-specify-different-dockerignore-files-for-different-builds-in-the-same-pr # https://github.com/moby/moby/issues/12886#issuecomment-480575928 # ----------------------- shell_env = copy(os.environ) shell_env["DOCKER_BUILDKIT"] = "1" _check_valid_containers(c) faasm_ver = get_faasm_version() for container in c: # Check if we need to template a special dockerignore file # It seems the dockerignore file needs to be at <dockerfile_path>.dockerignore dockerfile = join("docker", "{}.dockerfile".format(container)) dockerignore_template_path = join( "docker", "{}.dockerignore.j2".format(container) ) dockerignore_target_path = "{}.dockerignore".format(dockerfile) if exists(join(PROJ_ROOT, dockerignore_template_path)): print("Templating new dockerignore file for {}".format(container)) jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(PROJ_ROOT)) template = jinja_env.get_template(dockerignore_template_path) with open(join(PROJ_ROOT, dockerignore_target_path), "w") as fh: fh.write(template.render()) tag_name = "faasm/{}:{}".format(container, faasm_ver) if nocache: no_cache_str = "--no-cache" else: no_cache_str = "" faasm_ver = get_faasm_version() cmd = "docker build {} -t {} --build-arg FAASM_VERSION={} -f {} .".format( no_cache_str, tag_name, faasm_ver, dockerfile ) print(cmd) res = call(cmd, shell=True, cwd=PROJ_ROOT, env=shell_env) if res != 0: raise RuntimeError("Failed docker build for {}".format(tag_name)) if push: _do_push(container, faasm_ver)
def start(ctx, workers=2, sgx=FAASM_SGX_MODE_DISABLED): """ Start the local dev cluster """ # This env makes sure we mount our local setup into the containers, rather # than using the prebuilt binaries env = copy(os.environ) env["FAASM_BUILD_DIR"] = join(PROJ_ROOT, "dev/faasm/build") env["FAASM_BUILD_MOUNT"] = "/build/faasm" env["FAASM_LOCAL_MOUNT"] = "/usr/local/faasm" faasm_ver = get_faasm_version() if sgx == FAASM_SGX_MODE_SIM: env["FAASM_CLI_IMAGE"] = "faasm/cli-sgx-sim:{}".format(faasm_ver) env["FAASM_WORKER_IMAGE"] = "faasm/worker-sgx-sim:{}".format(faasm_ver) env["WASM_VM"] = "sgx" elif sgx == FAASM_SGX_MODE_HARDWARE: env["FAASM_CLI_IMAGE"] = "faasm/cli-sgx:{}".format(faasm_ver) env["FAASM_WORKER_IMAGE"] = "faasm/worker-sgx:{}".format(faasm_ver) env["WASM_VM"] = "sgx" cmd = [ "docker compose up -d", "--scale worker={}".format(workers), "nginx", ] cmd = " ".join(cmd) print(cmd) run(cmd, shell=True, check=True, cwd=PROJ_ROOT, env=env)
def create_release(ctx): """ Create a draft release on Github """ # Get the head of master r = _get_repo() b = r.get_branch(branch="master") head = b.commit faasm_ver = get_faasm_version() # Create a tag from the head tag_name = _tag_name(faasm_ver) r.create_git_tag( tag_name, "Release {}\n".format(faasm_ver), head.sha, "commit", ) r.create_git_release( tag_name, "Faasm {}".format(faasm_ver), "Release {}\n".format(faasm_ver), draft=True, )
def bump(ctx, ver=None): """ Increase the version (defaults to bumping a single minor version) """ old_ver = get_faasm_version() if ver: new_ver = ver else: # Just bump the last minor version part new_ver_parts = old_ver.split(".") new_ver_minor = int(new_ver_parts[-1]) + 1 new_ver_parts[-1] = str(new_ver_minor) new_ver = ".".join(new_ver_parts) # Replace version in all files for f in VERSIONED_FILES: sed_cmd = "sed -i 's/{}/{}/g' {}".format(old_ver, new_ver, f) run(sed_cmd, shell=True, check=True) # Replace version in dirs for d in VERSIONED_DIRS: sed_cmd = [ "find {}".format(d), "-type f", "-exec sed -i -e 's/{}/{}/g'".format(old_ver, new_ver), "{} \;", ] sed_cmd = " ".join(sed_cmd) print(sed_cmd) run(sed_cmd, shell=True, check=True)
def push(ctx, c): """ Push container images """ faasm_ver = get_faasm_version() _check_valid_containers(c) for container in c: _do_push(container, faasm_ver)
def _deploy_knative_fn( name, image, replicas, concurrency, annotations, extra_env=None, shell_env=None, ): faasm_ver = get_faasm_version() image = "{}:{}".format(image, faasm_ver) cmd = [ "kn", "service", "create", name, "--image", image, "--namespace", "faasm", "--force", ] cmd.extend( { "--min-scale={}".format(replicas), "--max-scale={}".format(replicas), "--concurrency-limit={}".format(concurrency) if concurrency else "", } ) # Add annotations for annotation in annotations: cmd.append("--annotation {}".format(annotation)) # Add standard environment for key, value in KNATIVE_ENV.items(): cmd.append("--env {}={}".format(key, value)) # Add extra environment extra_env = extra_env if extra_env else {} for key, value in extra_env.items(): cmd.append("--env {}={}".format(key, value)) cmd_string = " ".join(cmd) print(cmd_string) shell_env_dict = os.environ.copy() if shell_env: shell_env_dict.update(shell_env) call(cmd_string, shell=True, env=shell_env_dict)
def pull(ctx, c): """ Pull container images """ faasm_ver = get_faasm_version() _check_valid_containers(c) for container in c: res = call("docker pull faasm/{}:{}".format(container, faasm_ver), shell=True, cwd=PROJ_ROOT) if res != 0: raise RuntimeError("Failed docker pull for {}".format(container))
def build_native(ctx, user, func, host=False, clean=False, nopush=False): """ Build a native Knative container for the given function """ if host: build_dir = join(PROJ_ROOT, "build", "knative_native") target = "{}-knative".format(func) clean_dir(build_dir, clean) cmd = [ "cmake", "-DCMAKE_CXX_COMPILER=/usr/bin/clang++", "-DCMAKE_C_COMPILER=/usr/bin/clang", "-DFAASM_BUILD_TYPE=knative-native", "-DCMAKE_BUILD_TYPE=Debug", PROJ_ROOT ] call(" ".join(cmd), cwd=build_dir, shell=True) make_cmd = "cmake --build . --target {} -- -j".format(target) call(make_cmd, cwd=build_dir, shell=True) else: # Build the container version = get_faasm_version() tag_name = "{}:{}".format(_native_image_name(func), version) cmd = [ "docker", "build", "--no-cache" if clean else "", "-t", tag_name, "--build-arg", "FAASM_VERSION={}".format(version), "--build-arg", "FAASM_USER={}".format(user), "--build-arg", "FAASM_FUNC={}".format(func), "-f", "docker/knative-native.dockerfile", "." ] env = copy(os.environ) env["DOCKER_BUILDKIT"] = "1" cmd_string = " ".join(cmd) print(cmd_string) res = call(cmd_string, shell=True, cwd=PROJ_ROOT) if res != 0: print("Building container failed") return 1 # Push the container if not nopush: cmd = "docker push {}".format(tag_name) call(cmd, shell=True, cwd=PROJ_ROOT)
def _get_release(): version = get_faasm_version() r = _get_repo() rels = r.get_releases() tag_name = _tag_name(version) rel = rels[0] if rel.tag_name != tag_name: print("Expected latest release to have tag {} but had {}".format( tag_name, rel.tag_name)) exit(1) return rel
def pull(ctx, c): """ Pull container images """ faasm_ver = get_faasm_version() _check_valid_containers(c) for container in c: run( "docker pull faasm/{}:{}".format(container, faasm_ver), shell=True, check=True, cwd=PROJ_ROOT, )
def create(ctx): """ Create a draft release on Github """ # Work out the tag faasm_ver = get_faasm_version() tag_name = _tag_name(faasm_ver) # Create a release in github from this tag r = _get_repo() r.create_git_release( tag_name, "Faasm {}".format(faasm_ver), "Release {}\n".format(faasm_ver), draft=True, )
def _do_knative_native_local(img_name): faasm_ver = get_faasm_version() img_name = "{}:{}".format(img_name, faasm_ver) # Run on host network for access to Redis cmd = [ "docker", "run", "--network=host", "--env LOG_LEVEL=debug", "--env STATE_MODE=redis", "--env FAASM_INVOKE_HOST=127.0.0.1", "--env FAASM_INVOKE_PORT=8080", "--env HOST_TYPE=knative", "--env FUNCTION_STORAGE=fileserver", "--env FILESERVER_URL=http://127.0.0.1:8002", img_name ] cmd_string = " ".join(cmd) print(cmd_string) call(cmd_string, shell=True, cwd=PROJ_ROOT)
def build(ctx, c, nocache=False, push=False): """ Build latest version of container images """ # Use buildkit for nicer logging shell_env = copy(os.environ) shell_env["DOCKER_BUILDKIT"] = "1" _check_valid_containers(c) faasm_ver = get_faasm_version() for container_name in c: # Prepare dockerfile and tag name dockerfile = join("docker", CONTAINER_NAME2FILE_MAP[container_name]) tag_name = "faasm/{}:{}".format(container_name, faasm_ver) # Prepare build arguments build_args = {"FAASM_VERSION": faasm_ver} if container_name.endswith(SGX_HW_CONTAINER_SUFFIX): build_args["FAASM_SGX_MODE"] = FAASM_SGX_MODE_HARDWARE build_args["FAASM_SGX_PARENT_SUFFIX"] = SGX_HW_CONTAINER_SUFFIX elif container_name.endswith(SGX_SIMULATION_CONTAINER_SUFFIX): build_args["FAASM_SGX_MODE"] = FAASM_SGX_MODE_SIM build_args[ "FAASM_SGX_PARENT_SUFFIX"] = SGX_SIMULATION_CONTAINER_SUFFIX else: build_args["FAASM_SGX_MODE"] = FAASM_SGX_MODE_DISABLED # Prepare docker command cmd = [ "docker build {}".format("--no-cache" if nocache else ""), "-t {}".format(tag_name), "{}".format(" ".join([ "--build-arg {}={}".format(arg, build_args[arg]) for arg in build_args ])), "-f {} .".format(dockerfile), ] docker_cmd = " ".join(cmd) print(docker_cmd) # Build (and push) docker image run(docker_cmd, shell=True, check=True, cwd=PROJ_ROOT, env=shell_env) if push: _do_push(container_name, faasm_ver)
def _do_knative_native_local(img_name): version = get_faasm_version() img_name = "{}:{}".format(img_name, version) cmd = [ "docker", "run", "-p 8080:8080", "--env LOG_LEVEL=debug", "--env FAASM_INVOKE_HOST=0.0.0.0", "--env FAASM_INVOKE_PORT=8080", "--env HOST_TYPE=knative", img_name ] cmd_string = " ".join(cmd) print(cmd_string) call(cmd_string, shell=True, cwd=PROJ_ROOT)
def delete_old(ctx): """ Deletes old Docker images """ faasm_ver = get_faasm_version() dock = docker.from_env() images = dock.images.list() for image in images: for t in image.tags: if not t.startswith("faasm/"): continue tag_ver = t.split(":")[-1] if version.parse(tag_ver) < version.parse(faasm_ver): print("Removing old image: {}".format(t)) dock.images.remove(t, force=True)
def get_release_body(): """ Generate body for release with detailed changelog """ docker_cmd = [ "docker run -t -v", "{}:/app/".format(PROJ_ROOT), "orhunp/git-cliff:latest", "--config cliff.toml", "--repository .", "{}..v{}".format(_get_release().tag_name, get_faasm_version()), ] cmd = " ".join(docker_cmd) print("Generating release body...") print(cmd) result = run(cmd, shell=True, stdout=PIPE, stderr=PIPE) return result.stdout.decode("utf-8")
def _get_artifact_url(name, version=None): version = version if version else get_faasm_version() url = "https://github.com/lsds/Faasm/releases/download/v{}/{}".format( version, name) return url
def get_runtime_tar_name(version=None): version = version if version else get_faasm_version() return "faasm-runtime-root-{}.tar.gz".format(version)
def get_sysroot_tar_name(version=None): version = version if version else get_faasm_version() return "faasm-sysroot-{}.tar.gz".format(version)
def get_toolchain_tar_name(version=None): version = version if version else get_faasm_version() return "faasm-toolchain-{}.tar.gz".format(version)
def _get_tag(): faasm_ver = get_faasm_version() tag_name = _tag_name(faasm_ver) return tag_name