def list_pods(namespace: str, label_selector: str = '') -> List[K8SPod]:
config.load_kube_config()
v1 = client.CoreV1Api()
pods_list: V1PodList = v1.list_namespaced_pod(namespace=namespace, label_selector=label_selector)
pods: List[V1Pod] = pods_list.items
k8s_pods: List[K8SPod] = []
for pod in pods:
pod_name: str = pod.metadata.name
pod_status: str = pod.status.phase
k8s_pod_status = PodStatus(pod_status.upper())
pod_labels = pod.metadata.labels
k8s_pod = K8SPod(namespace=namespace, name=pod_name, status=k8s_pod_status, labels=pod_labels)
k8s_pods.append(k8s_pod)
return k8s_pods
def cancel_pods_mode(namespace: str,
run_name: str = None,
pod_ids: str = None,
pod_status: str = None):
namespace_pods = k8s_pods.list_pods(namespace=namespace)
runs_only_pods = [pod for pod in namespace_pods if 'runName' in pod.labels]
filtered_pods = runs_only_pods
if run_name:
run_name_match_pods = []
for pod in runs_only_pods:
if re.match(run_name, pod.labels['runName']):
run_name_match_pods.append(pod)
filtered_pods = run_name_match_pods
if pod_ids:
pod_ids_match_pods = []
pod_ids_array = pod_ids.split(',')
for pod in filtered_pods:
if pod.name in pod_ids_array:
pod_ids_match_pods.append(pod)
filtered_pods = pod_ids_match_pods
if pod_status:
status_filtered_pods = []
try:
converted_pod_status = PodStatus(pod_status.upper())
except ValueError:
handle_error(user_msg=Texts.BAD_POD_STATUS_PASSED.format(
status_passed=pod_status,
available_statuses=PodStatus.all_members()))
exit(1)
return
for pod in filtered_pods:
if pod.status == converted_pod_status:
status_filtered_pods.append(pod)
filtered_pods = status_filtered_pods
if not filtered_pods:
handle_error(user_msg=Texts.LACK_OF_PODS_ERROR_MSG)
exit(1)
click.echo(
Texts.WILL_BE_PURGED_LIST_HEADER.format(
experiment_name_plural='pods',
operation_word=Texts.DELETE_OPERATION["deleted"]))
for pod in filtered_pods:
click.echo(f" - {pod.name}")
if not click.confirm(
Texts.CONFIRM_CANCEL_MSG.format(
experiment_name_plural='pods',
operation_word=Texts.DELETE_OPERATION["deletion"])):
handle_error(user_msg=Texts.CANCELLATION_ABORTED_MSG.format(
experiment_name_plural='pods',
operation_word=Texts.DELETE_OPERATION["deletion"]))
exit(0)
deleted_pods = []
not_deleted_pods = []
for pod in filtered_pods:
click.echo(Texts.CANCELING_PODS_MSG.format(pod_name=pod.name))
try:
pod.delete()
deleted_pods.append(pod)
except Exception:
handle_error(logger, Texts.OTHER_POD_CANCELLING_ERROR_MSG)
not_deleted_pods.append(pod)
if deleted_pods:
click.echo(
Texts.SUCCESSFULLY_CANCELLED_LIST_HEADER.format(
experiment_name_plural='pods',
operation_word=Texts.DELETE_OPERATION["deleted"]))
for pod in deleted_pods:
click.echo(f" - {pod.name}")
if not_deleted_pods:
click.echo(
Texts.FAILED_TO_CANCEL_LIST_HEADER.format(
experiment_name_plural='pods',
operation_word=Texts.DELETE_OPERATION["deleted"]))
for pod in not_deleted_pods:
click.echo(f" - {pod.name}")
sys.exit(1)
experiment_name_plural = 'experiments'
@click.command(help=Texts.HELP,
short_help=Texts.SHORT_HELP,
cls=AliasCmd,
alias='c',
options_metavar='[options]')
@click.argument("name", required=False, metavar="[name]")
@click.option('-m', '--match', help=Texts.HELP_M)
@click.option('-p', '--purge', help=Texts.HELP_P, is_flag=True)
@click.option('-i', '--pod-ids', help=Texts.HELP_I)
@click.option(
'-s',
'--pod-status',
help=Texts.HELP_S.format(available_statuses=PodStatus.all_members()))
@common_options(admin_command=False)
@pass_state
def cancel(state: State,
name: str,
match: str,
purge: bool,
pod_ids: str,
pod_status: str,
listed_runs_kinds: List[RunKinds] = None):
"""
Cancels chosen experiments based on a name provided as a parameter.
"""
if not listed_runs_kinds:
listed_runs_kinds = [RunKinds.TRAINING, RunKinds.JUPYTER]