def get_pod_container(self,
volume_mounts,
env_vars=None,
command=None,
args=None,
resources=None):
"""Pod job container for task."""
env_vars = get_list(env_vars)
env_vars += get_job_env_vars(
log_level=self.log_level,
outputs_path=get_job_outputs_path(job_name=self.job_name),
logs_path=get_job_logs_path(job_name=self.job_name),
data_path=get_job_data_path(job_name=self.job_name),
project_data_path=get_project_data_path(project_name=self.project_name)
)
env_vars += [
get_env_var(name=constants.CONFIG_MAP_JOB_INFO_KEY_NAME, value=json.dumps(self.labels)),
]
if resources:
env_vars += get_resources_env_vars(resources=resources)
ports = [client.V1ContainerPort(container_port=port) for port in self.ports]
return client.V1Container(name=self.job_container_name,
image=self.job_docker_image,
command=command,
args=args,
ports=ports,
env=env_vars,
resources=get_resources(resources),
volume_mounts=volume_mounts)
def get_pod_container(self,
volume_mounts,
env_vars=None,
command=None,
args=None,
resources=None):
"""Pod job container for task."""
env_vars = get_list(env_vars)
env_vars += get_job_env_vars(
log_level=self.log_level,
outputs_path=get_job_outputs_path(job_name=self.job_name),
logs_path=get_job_logs_path(job_name=self.job_name),
data_path=get_job_data_path(job_name=self.job_name),
project_data_path=get_project_data_path(project_name=self.project_name)
)
env_vars += [
get_env_var(name=constants.CONFIG_MAP_JOB_INFO_KEY_NAME, value=json.dumps(self.labels)),
]
if resources:
env_vars += get_resources_env_vars(resources=resources)
ports = [client.V1ContainerPort(container_port=port) for port in self.ports]
return client.V1Container(name=self.job_container_name,
image=self.job_docker_image,
command=command,
args=args,
ports=ports,
env=env_vars,
resources=get_resources(resources),
volume_mounts=volume_mounts)
def get_pod_container(self,
volume_mounts,
env_vars=None,
command=None,
args=None,
persistence_outputs=None,
persistence_data=None,
outputs_refs_jobs=None,
outputs_refs_experiments=None,
secret_refs=None,
configmap_refs=None,
resources=None,
ephemeral_token=None):
"""Pod job container for task."""
assert self.cluster_def is not None
# Env vars preparations
env_vars = to_list(env_vars, check_none=True)
outputs_path = get_experiment_outputs_path(
persistence_outputs=persistence_outputs,
experiment_name=self.experiment_name,
original_name=self.original_name,
cloning_strategy=self.cloning_strategy)
env_vars += get_job_env_vars(
persistence_outputs=persistence_outputs,
outputs_path=outputs_path,
persistence_data=persistence_data,
log_level=self.log_level,
logs_path=get_experiment_logs_path(self.experiment_name,
temp=False),
outputs_refs_jobs=outputs_refs_jobs,
outputs_refs_experiments=outputs_refs_experiments,
ephemeral_token=ephemeral_token,
)
env_vars += [
get_env_var(name=constants.CONFIG_MAP_CLUSTER_KEY_NAME,
value=json.dumps(self.cluster_def)),
get_env_var(name=constants.CONFIG_MAP_DECLARATIONS_KEY_NAME,
value=self.declarations),
get_env_var(name=constants.CONFIG_MAP_EXPERIMENT_INFO_KEY_NAME,
value=json.dumps(self.experiment_labels)),
]
env_vars += get_resources_env_vars(resources=resources)
# Env from configmap and secret refs
env_from = get_pod_env_from(secret_refs=secret_refs,
configmap_refs=configmap_refs)
ports = [
client.V1ContainerPort(container_port=port) for port in self.ports
]
return client.V1Container(name=self.job_container_name,
image=self.job_docker_image,
command=command,
args=args,
ports=ports,
env=env_vars,
env_from=env_from,
resources=get_resources(resources),
volume_mounts=volume_mounts)
def test_get_resources_env_vars(self):
env_vars = get_resources_env_vars(None)
assert any(item.name == 'NVIDIA_VISIBLE_DEVICES' and item.value == 'none'
for item in env_vars)
resources = MagicMock()
resources.gpu = None
env_vars = get_resources_env_vars(resources)
assert any(item.name == 'NVIDIA_VISIBLE_DEVICES' and item.value == 'none'
for item in env_vars)
resources = MagicMock()
resources.gpu.limits = '0'
env_vars = get_resources_env_vars(resources)
assert any(item.name == 'NVIDIA_VISIBLE_DEVICES' and item.value == 'none'
for item in env_vars)
def get_project_pod_spec(volume_mounts,
volumes,
image,
command,
args,
ports,
env_vars=None,
env_from=None,
container_name=None,
resources=None,
node_selector=None,
affinity=None,
tolerations=None,
image_pull_policy=None,
restart_policy=None,
use_service_account=False):
"""Pod spec to be used to create pods for project: tensorboard, notebooks."""
env_vars = get_list(env_vars)
volume_mounts = get_list(volume_mounts)
volumes = get_list(volumes)
gpu_volume_mounts, gpu_volumes = get_gpu_volumes_def(resources)
volume_mounts += gpu_volume_mounts
volumes += gpu_volumes
ports = [client.V1ContainerPort(container_port=port) for port in ports]
env_vars += get_resources_env_vars(resources=resources)
containers = [
client.V1Container(name=container_name,
image=image,
image_pull_policy=image_pull_policy,
command=command,
args=args,
ports=ports,
env=env_vars,
env_from=env_from,
resources=get_resources(resources),
volume_mounts=volume_mounts)
]
service_account_name = None
if use_service_account and settings.K8S_RBAC_ENABLED:
service_account_name = settings.K8S_SERVICE_ACCOUNT_NAME
return client.V1PodSpec(restart_policy=restart_policy,
service_account_name=service_account_name,
containers=containers,
volumes=volumes,
node_selector=node_selector,
affinity=affinity,
tolerations=tolerations)
def get_project_pod_spec(volume_mounts,
volumes,
image,
command,
args,
ports,
env_vars=None,
env_from=None,
container_name=None,
resources=None,
node_selector=None,
affinity=None,
tolerations=None,
image_pull_policy=None,
restart_policy=None,
service_account_name=None):
"""Pod spec to be used to create pods for project: tensorboard, notebooks."""
env_vars = to_list(env_vars, check_none=True)
volume_mounts = to_list(volume_mounts, check_none=True)
volumes = to_list(volumes, check_none=True)
gpu_volume_mounts, gpu_volumes = get_gpu_volumes_def(resources)
volume_mounts += gpu_volume_mounts
volumes += gpu_volumes
ports = [client.V1ContainerPort(container_port=port) for port in ports]
env_vars += get_resources_env_vars(resources=resources)
containers = [
client.V1Container(name=container_name,
image=image,
image_pull_policy=image_pull_policy,
command=command,
args=args,
ports=ports,
env=env_vars,
env_from=env_from,
resources=get_resources(resources),
volume_mounts=volume_mounts)
]
if service_account_name and not conf.get('K8S_RBAC_ENABLED'):
service_account_name = None
return client.V1PodSpec(restart_policy=restart_policy,
service_account_name=service_account_name,
containers=containers,
volumes=volumes,
node_selector=node_selector,
affinity=affinity,
tolerations=tolerations)
def get_pod_container(self,
volume_mounts,
persistence_outputs,
persistence_data,
outputs_refs_jobs=None,
outputs_refs_experiments=None,
secret_refs=None,
configmap_refs=None,
env_vars=None,
command=None,
args=None,
resources=None):
"""Pod job container for task."""
# Env vars preparation
env_vars = to_list(env_vars, check_none=True)
logs_path = stores.get_job_logs_path(job_name=self.job_name,
temp=False)
outputs_path = stores.get_job_outputs_path(
persistence=persistence_outputs, job_name=self.job_name)
env_vars += get_job_env_vars(
log_level=self.log_level,
persistence_outputs=persistence_outputs,
outputs_path=outputs_path,
persistence_data=persistence_data,
logs_path=logs_path,
outputs_refs_jobs=outputs_refs_jobs,
outputs_refs_experiments=outputs_refs_experiments)
env_vars += [
get_env_var(name=constants.CONFIG_MAP_JOB_INFO_KEY_NAME,
value=json.dumps(self.labels)),
]
env_vars += get_resources_env_vars(resources=resources)
# Env from configmap and secret refs
env_from = get_pod_env_from(secret_refs=secret_refs,
configmap_refs=configmap_refs)
ports = [
client.V1ContainerPort(container_port=port) for port in self.ports
]
return client.V1Container(name=self.job_container_name,
image=self.job_docker_image,
command=command,
args=args,
ports=ports or None,
env=env_vars,
env_from=env_from,
resources=get_resources(resources),
volume_mounts=volume_mounts)
def get_pod_container(self,
volume_mounts,
env_vars=None,
command=None,
args=None,
persistence_outputs=None,
persistence_data=None,
outputs_refs_jobs=None,
outputs_refs_experiments=None,
resources=None):
"""Pod job container for task."""
assert self.cluster_def is not None
env_vars = get_list(env_vars)
outputs_path = get_experiment_outputs_path(
persistence_outputs=persistence_outputs,
experiment_name=self.experiment_name,
original_name=self.original_name,
cloning_strategy=self.cloning_strategy)
env_vars += get_job_env_vars(
log_level=self.log_level,
outputs_path=outputs_path,
data_paths=get_data_paths(persistence_data),
logs_path=get_experiment_logs_path(self.experiment_name),
outputs_refs_jobs=outputs_refs_jobs,
outputs_refs_experiments=outputs_refs_experiments)
env_vars += [
get_env_var(name=constants.CONFIG_MAP_CLUSTER_KEY_NAME,
value=json.dumps(self.cluster_def)),
get_env_var(name=constants.CONFIG_MAP_DECLARATIONS_KEY_NAME,
value=self.declarations),
get_env_var(name=constants.CONFIG_MAP_EXPERIMENT_INFO_KEY_NAME,
value=json.dumps(self.experiment_labels)),
]
env_vars += get_resources_env_vars(resources=resources)
ports = [
client.V1ContainerPort(container_port=port) for port in self.ports
]
return client.V1Container(name=self.job_container_name,
image=self.job_docker_image,
command=command,
args=args,
ports=ports,
env=env_vars,
resources=get_resources(resources),
volume_mounts=volume_mounts)
def get_pod_container(self,
volume_mounts,
persistence_outputs=None,
persistence_data=None,
outputs_refs_jobs=None,
outputs_refs_experiments=None,
secret_refs=None,
config_map_refs=None,
env_vars=None,
command=None,
args=None,
resources=None,
ports=None,
ephemeral_token=None):
"""Pod job container for task."""
self._pod_container_checks()
# Env vars preparations
env_vars = to_list(env_vars, check_none=True)
env_vars += self._get_container_pod_env_vars(
persistence_outputs=persistence_outputs,
persistence_data=persistence_data,
outputs_refs_jobs=outputs_refs_jobs,
outputs_refs_experiments=outputs_refs_experiments,
ephemeral_token=ephemeral_token)
env_vars += get_resources_env_vars(resources=resources)
env_vars += get_kv_env_vars(self._get_kv_env_vars(None))
# Env from config_map and secret refs
env_from = get_pod_env_from(secret_refs=secret_refs,
config_map_refs=config_map_refs)
def get_ports():
_ports = to_list(ports) if ports else []
return [
client.V1ContainerPort(container_port=port) for port in _ports
] or None
return client.V1Container(name=self.job_container_name,
image=self.job_docker_image,
command=command,
args=args,
ports=get_ports(),
env=env_vars,
env_from=env_from,
resources=get_resources(resources),
volume_mounts=volume_mounts)
def get_pod_container(self,
volume_mounts,
env_vars=None,
command=None,
args=None,
resources=None):
"""Pod job container for task."""
assert self.cluster_def is not None
env_vars = get_list(env_vars)
outputs_path = get_experiment_outputs_path(
experiment_name=self.experiment_name,
original_name=self.original_name,
cloning_strategy=self.cloning_strategy)
env_vars += get_job_env_vars(
log_level=self.log_level,
outputs_path=outputs_path,
logs_path=get_experiment_logs_path(self.experiment_name),
data_path=get_experiment_data_path(self.experiment_name),
project_data_path=get_project_data_path(project_name=self.project_name)
)
env_vars += [
get_env_var(name=constants.CONFIG_MAP_CLUSTER_KEY_NAME,
value=json.dumps(self.cluster_def)),
get_env_var(name=constants.CONFIG_MAP_DECLARATIONS_KEY_NAME,
value=self.declarations),
get_env_var(name=constants.CONFIG_MAP_EXPERIMENT_INFO_KEY_NAME,
value=json.dumps(self.experiment_labels)),
]
if resources:
env_vars += get_resources_env_vars(resources=resources)
ports = [client.V1ContainerPort(container_port=port) for port in self.ports]
return client.V1Container(name=self.job_container_name,
image=self.job_docker_image,
command=command,
args=args,
ports=ports,
env=env_vars,
resources=get_resources(resources),
volume_mounts=volume_mounts)
def get_pod_container(volume_mounts,
image,
command,
args,
ports,
env_vars=None,
env_from=None,
container_name=None,
resources=None,
image_pull_policy=None):
env_vars = to_list(env_vars, check_none=True)
env_vars += get_resources_env_vars(resources=resources)
ports = [client.V1ContainerPort(container_port=port) for port in ports]
return client.V1Container(name=container_name,
image=image,
image_pull_policy=image_pull_policy,
command=command,
args=args,
ports=ports,
env=env_vars,
env_from=env_from,
resources=get_resources(resources),
volume_mounts=volume_mounts)
