def ibm_deploy_worker(ctx, update=False): config = get_faasm_config() redis_host = config["IBM"]["redis_host_private"] api_key = config["IBM"]["api_key"] # Note that concurrency here is _intra_ container, i.e. how many concurrent # invocations can each container support cmd = [ "ibmcloud", "fn", "action", "update" if update else "create", "--param FUNCTION_STORAGE ibm", "--param REDIS_QUEUE_HOST {}".format(redis_host), "--param REDIS_STATE_HOST {}".format(redis_host), "--param IBM_API_KEY {}".format(api_key), "--param CGROUP_MODE off", "--param NETNS_MODE off", "--memory 1024", "--timeout 30000", "--concurrency 20", "faasm-worker", "--docker", "faasm/ibm-worker:0.1.3" ] cmd_string = " ".join(cmd) print(cmd_string) call(cmd_string, shell=True) print("Done")
def _do_redis_command(sub_cmd, local, docker, knative, ibm): if local: cmd = [ "redis-cli", sub_cmd ] elif docker: cmd = [ "docker-compose", "exec", "redis-queue", "redis-cli", sub_cmd ] elif ibm: faasm_conf = get_faasm_config() cmd = [ "redis-cli", "-h {}".format(faasm_conf["IBM"]["redis_host_public"]), sub_cmd ] elif knative: cmd = [ "kubectl", "exec", "-n faasm", "redis-queue", "--", "redis-cli", sub_cmd ] else: cmd = ["redis-cli", sub_cmd] cmd_string = " ".join(cmd) print(cmd_string) ret_code = call(" ".join(cmd), shell=True, cwd=PROJ_ROOT) if ret_code != 0: print("Command failed: {}".format(cmd_string))
def ibm_k8s_add_knative(ctx): faasm_conf = get_faasm_config() print("Enabling knative") cmd = [ "ibmcloud", "ks", "cluster-addon-enable", "knative", "--cluster", faasm_conf["IBM"]["k8s_cluster_id"], ] call(" ".join(cmd), shell=True)
def ibm_k8s_config(ctx): faasm_conf = get_faasm_config() print("Getting cluster config") cmd = [ "ibmcloud", "ks", "cluster", "config", "--cluster", faasm_conf["IBM"]["k8s_cluster_id"] ] call(" ".join(cmd), shell=True)
def get_kubernetes_upload_host(knative, host): faasm_conf = get_faasm_config() if knative: if not faasm_conf.has_section("Faasm"): host = host if host else "localhost" else: host = faasm_conf["Faasm"]["upload_host"] return host
def _get_github_instance(): conf = get_faasm_config() if not conf.has_section("Github") or not conf.has_option("Github", "access_token"): print("Must set up Github config with access token") token = conf["Github"]["access_token"] g = Github(token) return g
def ibm_login(ctx): config = get_faasm_config() api_key = config["IBM"]["api_key"] email = config["IBM"]["email"] cmd = ["ibmcloud", "login", "--apikey", api_key] call(" ".join(cmd), shell=True) cmd = ["ibmcloud", "target", "-o", email, "-s", "dev"] call(" ".join(cmd), shell=True)
def get_invoke_host_port(host_in=None, port_in=None): faasm_config = get_faasm_config() if not host_in and faasm_config.has_section("Faasm"): host = faasm_config["Faasm"].get("invoke_host", "localhost") port = faasm_config["Faasm"].get("invoke_port", 8080) else: host = host_in if host_in else "127.0.0.1" port = port_in if port_in else 8080 return host, port
def get_upload_host_port(host_in, port_in): faasm_config = get_faasm_config() if not host_in and faasm_config.has_section("Faasm"): host = faasm_config["Faasm"].get("upload_host", "127.0.0.1") port = faasm_config["Faasm"].get("upload_port", 8002) else: host = host_in if host_in else "127.0.0.1" port = port_in if port_in else 8002 return host, port
def _get_config_value(env_var, key, default_value): """ Get the config value, allowing an environment variable to take precedence """ faasm_config = get_faasm_config() env_value = environ.get(env_var, None) if env_value: return env_value if faasm_config.has_section("Faasm"): conf_value = faasm_config["Faasm"].get(key, default_value) return conf_value return default_value
def _do_deploy_knative_native(func_name, image_name, replicas): faasm_config = get_faasm_config() if not faasm_config.has_section("Faasm"): print("Must have faasm config set up with Faasm section") return 1 # Host and port required for chaining native functions invoke_host, invoke_port = get_invoke_host_port() _deploy_knative_fn( func_name, image_name, replicas, 1, NATIVE_WORKER_ANNOTATIONS, extra_env={ "COLD_START_DELAY_MS": "1000", "FAASM_INVOKE_HOST": invoke_host, "FAASM_INVOKE_PORT": invoke_port, }, )
def deploy(ctx, replicas=DEFAULT_REPLICAS, local=False, ibm=False, gke=False): """ Deploy Faasm to knative """ faasm_conf = get_faasm_config() shell_env = {} if ibm: # IBM requires specifying custom kubeconfig shell_env["KUBECONFIG"] = get_ibm_kubeconfig() extra_env = { "FUNCTION_STORAGE": "ibm", "IBM_API_KEY": faasm_conf["IBM"]["api_key"], } else: extra_env = { "FUNCTION_STORAGE": "fileserver", "FILESERVER_URL": "http://upload:8002", } # Deploy the other K8s stuff (e.g. redis) _kubectl_apply(join(COMMON_CONF, "namespace.yml"), env=shell_env) _kubectl_apply(COMMON_CONF, env=shell_env) _kubectl_apply(BARE_METAL_CONF) if local: _kubectl_apply(LOCAL_CONF) else: _kubectl_apply(BARE_METAL_REMOTE_CONF) _deploy_knative_fn( FAASM_WORKER_NAME, FAASM_WORKER_IMAGE, replicas, FAASM_WORKER_CONCURRENCY, FAASM_WORKER_ANNOTATIONS, extra_env=extra_env, shell_env=shell_env )
def invoke_impl(user, func, host=None, port=None, input=None, py=False, asynch=False, knative=True, native=False, ibm=False, poll=False, cmdline=None, mpi_world_size=None, debug=False, poll_interval_ms=1000): faasm_config = get_faasm_config() # Provider-specific stuff if ibm: host = faasm_config["IBM"]["k8s_subdomain"] port = 8080 elif knative: host, port = get_invoke_host_port() # Defaults host = host if host else "127.0.0.1" port = port if port else 8080 # Polling always requires asynch if poll: asynch = True # Create URL and message url = "http://{}".format(host) if not port == "80": url += ":{}".format(port) if py: msg = { "user": PYTHON_USER, "function": PYTHON_FUNC, "async": asynch, "py_user": user, "py_func": func, "python": True, } else: msg = { "user": user, "function": func, "async": asynch, } if input: msg["input_data"] = input if cmdline: msg["cmdline"] = cmdline if mpi_world_size: msg["mpi_world_size"] = mpi_world_size # IBM-specific message format if ibm: faasm_conf = get_faasm_config() msg.update({ "IBM_API_KEY": faasm_conf["IBM"]["api_key"], "REDIS_QUEUE_HOST": faasm_conf["IBM"]["redis_host_public"], "REDIS_STATE_HOST": faasm_conf["IBM"]["redis_host_public"], }) # Message needs to be nested msg = { "value": msg, } # IBM must call init first if ibm: do_post("http://{}:{}/init/".format(host, port), msg, json=True) # Knative must pass custom headers if knative and native: if py: headers = _get_knative_headers("python") else: headers = _get_knative_headers(func) elif knative: headers = _get_knative_headers("worker") else: headers = {} if asynch: # Submit initial asynch call asynch_result = do_post(url, msg, headers=headers, quiet=True, json=True) try: call_id = int(asynch_result) except ValueError: raise RuntimeError("Could not parse async response to int: {}".format(asynch_result)) if not poll: # Return the call ID if we're not polling return call_id else: if not knative: raise RuntimeError("Poll only supported for knative") print("\n---- Polling {} ----".format(call_id)) # Poll status until we get success/ failure result = None output = None count = 0 while result != STATUS_SUCCESS: count += 1 interval = float(poll_interval_ms) / 1000 sleep(interval) result, output = status_call_impl(user, func, call_id, host, port, quiet=True, native=native) print("\nPOLL {} - {}".format(count, result)) print("\n---- Finished {} ----\n".format(call_id)) print(output) if result == STATUS_SUCCESS: prefix = "SUCCESS:" success = True else: prefix = "FAILED:" success = False output = output.replace(prefix, "") return success, output else: if ibm or knative: return do_post(url, msg, headers=headers, json=True, debug=debug) else: raise RuntimeError("Must specify knative or ibm")
def is_kubernetes(): faasm_conf = get_faasm_config() return faasm_conf.has_section("Faasm")
def get_ibm_kubeconfig(): # NOTE: we assume LON02 datacentre here faasm_config = get_faasm_config() return join(HOME_DIR, ".bluemix/plugins/container-service/clusters", faasm_config["IBM"]["k8s_cluster_id"], "kube-config-lon02-faasm.yml")
def invoke_impl(user, func, host=None, port=None, input=None, py=False, async=False, knative=True, native=False, ibm=False, poll=False, cmdline=None, mpi_world_size=None, debug=False, poll_interval_ms=1000): faasm_config = get_faasm_config() # Provider-specific stuff if ibm: host = faasm_config["IBM"]["k8s_subdomain"] port = 8080 elif knative: host, port = get_invoke_host_port() # Defaults host = host if host else "127.0.0.1" port = port if port else 8080 # Polling always requires async if poll: async = True
def ibm_ssh_redis(ctx): config = get_faasm_config() cmd = ["ssh", "root@{}".format(config["IBM"]["redis_host_public"])] call(" ".join(cmd), shell=True)