def test_fetch_action_arg_is_custom(json_load_mock, isfile_mock, open_mock):
isfile_mock.return_value = True
custom = is_custom('deploy:')
open_mock.assert_called_once()
isfile_mock.assert_called_once()
assert custom
def _undeploy_jobs(self, namespace, jobs, all_jobs=False):
"""undeploy the jobs passed to us
jobs: 1 or more jobs to undeploy
NOTE: right now there's no case in which some template has both
custom and not custom jobs because we check for custom job
by if there's a Makefile in the top level of the project
"""
# simplify logic by `looping` over all jobs even if there's just 1
if not isinstance(jobs, list):
jobs = [jobs]
# custom jobs require looping over all of them and calling
# `make undeploy` on each job
recursive_delete = False if files.is_custom('undeploy:') else True
if recursive_delete:
process_helpers.run([
"kubectl", "--namespace", namespace, "delete", "-f", "k8s",
"--recursive"
],
raise_on_failure=True)
# TODO: have this not be in a loop
for job in jobs:
self.remove_job_dir(os.path.join('k8s', job))
else:
for job in jobs:
self._custom_undeploy(job)
self.remove_job_dir(os.path.join('k8s', job))
def _deploy_new_container(self):
"""Substitutes image, app, run data into k8s-template selected.
Can also launch user into interactive shell with --interactive flag
"""
app_name = self.config['name']
self.namespace = self.config['namespace']
remote_container_name = files.fetch_action_arg(
'push', 'last_remote_container')
if remote_container_name is None:
raise ValueError("No image found to deploy with. Run a plain "
"`mlt deploy` to fix this. Most common reason "
"for this is a --no-push was used before "
"any image was available to use.")
print("Deploying {}".format(remote_container_name))
kubernetes_helpers.ensure_namespace_exists(self.namespace)
app_run_id = str(uuid.uuid4())
"""
we'll keep track of the number of containers that would be deployed
so we know if we should exec into 1 or not (only auto-exec if 1 made)
if we have replicas (with value > 1) then we automatically won't
go into most recent pod, because there will be > 1 container made
if we find > 1 container regardless of replica, same logic applies
"""
self._replicas_found = False
self._total_containers = 0
# deploy our normal template sub logic, then if `deploy` in Makefile
# add whatever custom stuff is desired
self._default_deploy(app_name=app_name,
app_run_id=app_run_id,
remote_container_name=remote_container_name)
if files.is_custom("deploy:"):
# execute the custom deploy code
self._custom_deploy(app_name=app_name,
app_run_id=app_run_id,
remote_container_name=remote_container_name)
self._update_app_run_id(app_run_id)
print("\nInspect created objects by running:\n"
"$ kubectl get --namespace={} all\n"
"or \n$ mlt status\n".format(self.namespace))
if self.args["--interactive"] and not self._replicas_found \
and self._total_containers == 1:
self._exec_into_pod(self._get_most_recent_podname())
elif self.args["--interactive"]:
print("More than one container created."
".\nCall `kubectl exec -it {{pod_name_here}} "
"--namespace={} /bin/bash` on a `Running` pod NAME "
"below.\nIf no pods are running yet, run `mlt status` "
"occasionally, or `watch -n1 mlt status` to watch until "
"pods are `Running`.\n".format(self.namespace))
for line in self._get_pods_by_start_time():
print(line)
def _undeploy_job(self, namespace, job_name):
"""undeploy the given job name"""
job_dir = "k8s/{}".format(job_name)
if files.is_custom('undeploy:'):
self._custom_undeploy(job_name)
else:
process_helpers.run(
["kubectl", "--namespace", namespace, "delete", "-f",
job_dir, "--recursive"],
raise_on_failure=True)
self.remove_job_dir(job_dir)
def _undeploy_jobs(self, namespace, jobs, all_jobs=False):
"""undeploy the jobs passed to us
jobs: 1 or more jobs to undeploy
NOTE: right now there's no case in which some template has both
custom and not custom jobs because we check for custom job
by if there's a Makefile in the top level of the project
"""
# simplify logic by `looping` over all jobs even if there's just 1
if not isinstance(jobs, list):
jobs = [jobs]
# custom jobs require looping over all of them and calling
# `make undeploy` on each job
recursive_delete = False if files.is_custom('undeploy:') else True
if recursive_delete:
folder_to_delete = 'k8s'
if not all_jobs:
# only way all_jobs won't be false is if there's
# a --job-name flag passed or there's only 1 job to undeploy
if len(jobs) != 1:
error_handling.throw_error(
"There should be only 1 job to undeploy, "
"something went wrong. Please file a bug on "
"https://github.com/IntelAI/mlt")
folder_to_delete = os.path.join(folder_to_delete, jobs[0])
process_helpers.run([
"kubectl", "--namespace", namespace, "delete", "-f",
folder_to_delete, "--recursive"
],
raise_on_failure=True)
# TODO: have this not be in a loop
for job in jobs:
self.remove_job_dir(os.path.join('k8s', job))
else:
for job in jobs:
self._custom_undeploy(job, namespace)
self.remove_job_dir(os.path.join('k8s', job))
def _display_status(self, job, namespace):
"""detects what kind of job was deployed and calls the correct
status display function
"""
status_options = {
"job": self._generic_status,
"tfjob": self._crd_status,
"pytorchjob": self._crd_status,
# experiments have yaml templates but also a bash script to call
"experiment": self._custom_status
}
# if we have more than 1 k8 object created and types don't match
# go with a custom job type since we won't know what kubectl call
# to make to get status from everything
# also, if `status:` in Makefile we'll assume it's custom always
job_types, all_same_job_type = files.get_job_kinds()
if (job_types and not all_same_job_type) or files.is_custom('status:'):
job_types = "custom"
elif job_types:
job_types = job_types.pop()
try:
status_options.get(job_types, self._custom_status)(
job, namespace, job_types)
except subprocess.CalledProcessError as e:
if "No rule to make target `status'" in str(e.output):
# TODO: when we have a template updating capability, add a
# note recommending that he user update's their template to
# get the status command
error_msg = "This app does not support the `mlt status` " + \
"command. No `status` target was found in the Makefile."
else:
error_msg = "Error while getting app status: {}".format(
e.output)
error_handling.throw_error(error_msg)
