def test_is_k8s_stoppable(self):
assert LifeCycle.is_k8s_stoppable(None) is False
for status in LifeCycle.VALUES:
cond = (LifeCycle.is_running(status)
or LifeCycle.is_unschedulable(status)
or LifeCycle.is_warning(status=status)
or LifeCycle.is_unknown(status=status))
if cond:
assert LifeCycle.is_k8s_stoppable(status) is True
else:
assert LifeCycle.is_k8s_stoppable(status) is False
def test_is_k8s_stoppable(self):
assert LifeCycle.is_k8s_stoppable(None) is False
for status in V1Statuses.allowable_values:
cond = (LifeCycle.is_running(status)
or LifeCycle.is_unschedulable(status)
or LifeCycle.is_warning(status=status)
or LifeCycle.is_unknown(status=status))
if cond:
assert LifeCycle.is_k8s_stoppable(status) is True
else:
assert LifeCycle.is_k8s_stoppable(status) is False
def runs_stop(run_id: int,
run: Optional[BaseRun],
update_status=False,
message=None) -> bool:
run = get_run(run_id=run_id, run=run)
if not run:
return True
stopped = True
should_stop = (LifeCycle.is_k8s_stoppable(run.status)
or run.status == V1Statuses.STOPPING)
if run.is_managed and should_stop:
in_cluster = conf.get(K8S_IN_CLUSTER)
if in_cluster and (run.is_service or run.is_job):
stopped = manager.stop(
run_uuid=run.uuid.hex,
run_kind=run.kind,
namespace=conf.get(K8S_NAMESPACE),
in_cluster=in_cluster,
)
if not stopped:
return False
if not update_status:
return True
new_run_stop_status(run=run, message=message)
return True
def runs_stop(
run_id: int,
run: Optional[BaseRun],
update_status=False,
message=None,
clean=False,
) -> bool:
run = get_run(run_id=run_id, run=run)
if not run:
return True
stopped = True
should_stop = (LifeCycle.is_k8s_stoppable(run.status)
or run.status == V1Statuses.STOPPING)
def _clean():
try:
manager.clean(
run_uuid=run.uuid.hex,
run_kind=run.kind,
namespace=conf.get(K8S_NAMESPACE),
in_cluster=in_cluster,
)
except (PolyaxonK8SError, ApiException) as e:
_logger.warning(
"Something went wrong, the run `%s` could not be stopped, error %s",
run.uuid,
e,
)
return False
if run.is_managed and should_stop:
in_cluster = conf.get(K8S_IN_CLUSTER)
if in_cluster and (run.is_service or run.is_job):
if clean:
_clean()
try:
stopped = manager.stop(
run_uuid=run.uuid.hex,
run_kind=run.kind,
namespace=conf.get(K8S_NAMESPACE),
in_cluster=in_cluster,
)
except (PolyaxonK8SError, ApiException) as e:
_logger.warning(
"Something went wrong, the run `%s` could not be stopped, error %s",
run.uuid,
e,
)
return False
if not stopped:
return False
if not update_status:
return True
new_run_stop_status(run=run, message=message)
return True
