def write_serve_logs(self, endpoint, logs, out_dir):
name = self.workspace
log_dir = build_dir(out_dir, name, 'logs')
# save to file
with open(os.path.join(log_dir, f"{endpoint}.log"), 'w') as dst:
dst.write(logs)
def write_build_logs(self, job_id, logs, out_dir):
name = self.workspace
log_dir = build_dir(out_dir, name, 'logs')
with open(os.path.join(log_dir, f"build-serve-{job_id}.log"),
'w') as dst:
dst.write(logs)
def write_train_logs(self, job_id, logs, out_dir):
name = self.workspace
log_dir = build_dir(out_dir, name, 'logs')
# save to file
with open(os.path.join(log_dir, f"train-{job_id}.log"), 'w') as dst:
dst.write(logs)
def build(self, provider, env, local_backend=False, verbose=False):
env_state = EnvironmentState.initialize(provider, env)
if not os.path.exists(env_state.build_dir):
build_dir(env_state.build_dir)
auth_token = uuid.uuid4()
extra_vars = self._get_vars(provider, env_state.build_dir, auth_token)
self.tf_service.cd_dir(env_state.build_dir)
self.tf_service.set_verbose(verbose)
self._tf_init(env_state,
provider,
env,
local_backend,
destroying=False)
self.tf_service.plan(env_state.build_dir, extra_vars)
self.tf_service.apply(env_state.build_dir, extra_vars)
self.tf_service.execute()
# check if the deployed successfully
# Refresh environment states after terraform service operations
env_state = EnvironmentState.initialize(provider, env)
if env_state.if_tfstate_exists:
url, kubeconfig = self._parse_config(env_state.build_dir)
current_context = provider if provider in [
DOCKER, MINIKUBE
] else f"{provider}_{env}"
self.state_service.set(DEFAULT, user=USER)
self._set_context_list(current_context)
self._set_active_context(current_context)
self.state_service.set(current_context)
self.state_service.set_section(current_context,
BACKEND,
url=url,
token=auth_token)
self.state_service.set_section(current_context,
INFRASTRUCTURE,
kubeconfig=kubeconfig)
self.state_service.write()
return True, env_state
return False, env_state
def build(self, provider, env, local_backend=False, verbose=False):
dir_build = self._set_build_dir(provider, env)
build_dir(dir_build)
extra_vars = self._get_vars(provider, dir_build)
self.tf_service.cd_dir(dir_build)
self.tf_service.set_verbose(verbose)
directory = self._tf_init(provider,
env,
local_backend,
destroying=False)
self.tf_service.plan(directory, extra_vars)
self.tf_service.apply(directory, extra_vars)
self.tf_service.execute()
url, kubeconfig = self._parse_config(dir_build)
self.state_service.set(BACKEND, url=url, token=uuid.uuid4())
self.state_service.set(INFRASTRUCTURE, kubeconfig=kubeconfig)
self.state_service.write()
def provenance(self, out_dir, endpoint):
base_url = self.url
name = self.workspace
# build output directory (default = workspace)
prov_dir = build_dir(out_dir, name, 'provenance')
# GET /inference/<name>/<endpoint>/provenance
r = requests.get(f"{base_url}/inference/{name}/{endpoint}/provenance")
if r.status_code < 300:
out_fid = os.path.join(prov_dir, f"{endpoint}")
return out_fid, r.json()
elif 400 <= r.status_code < 500:
err = Error.from_dict(r.json())
raise RequestError(err.message)
else:
raise RequestError(r.text)
def provenance(self, out_dir, model_id):
base_url = self.url
name = self.workspace
# build output directory (default = workspace)
prov_dir = build_dir(out_dir, name, 'provenance')
# GET /train/<name>/<model_id>/provenance
r = requests.get(f"{base_url}/train/{name}/{model_id}/provenance",
headers={"X-Token": self.token})
if r.status_code < 300:
out_fid = os.path.join(prov_dir, f"model-{model_id}")
return out_fid, r.json()
elif 400 <= r.status_code < 500:
err = Error.from_dict(r.json())
raise RequestError(err.message)
else:
raise RequestError(r.text)
def test_build_dir():
t = TemporaryDirectory()
path = build_dir(t.name, "something", "something")
assert os.path.exists(path)
t.cleanup()
