def get_task_pod_spec(self,
volume_mounts,
volumes,
persistence_outputs=None,
persistence_data=None,
outputs_refs_jobs=None,
outputs_refs_experiments=None,
env_vars=None,
command=None,
args=None,
resources=None,
node_selector=None,
affinity=None,
tolerations=None,
restart_policy='OnFailure'):
"""Pod spec to be used to create pods for tasks: master, worker, ps."""
volume_mounts = get_list(volume_mounts)
volumes = get_list(volumes)
gpu_volume_mounts, gpu_volumes = get_gpu_volumes_def(resources)
volume_mounts += gpu_volume_mounts
volumes += gpu_volumes
pod_container = self.get_pod_container(volume_mounts=volume_mounts,
persistence_outputs=persistence_outputs,
persistence_data=persistence_data,
outputs_refs_jobs=outputs_refs_jobs,
outputs_refs_experiments=outputs_refs_experiments,
env_vars=env_vars,
command=command,
args=args,
resources=resources)
containers = [pod_container]
if self.use_sidecar:
sidecar_container = self.get_sidecar_container()
containers.append(sidecar_container)
node_selector = get_node_selector(
node_selector=node_selector,
default_node_selector=settings.NODE_SELECTOR_JOBS)
affinity = get_affinity(
affinity=affinity,
default_affinity=settings.AFFINITY_JOBS)
tolerations = get_tolerations(
tolerations=tolerations,
default_tolerations=settings.TOLERATIONS_JOBS)
service_account_name = None
if settings.K8S_RBAC_ENABLED:
service_account_name = settings.K8S_SERVICE_ACCOUNT_NAME
return client.V1PodSpec(
restart_policy=restart_policy,
service_account_name=service_account_name,
init_containers=to_list(self.get_init_container(persistence_outputs)),
containers=containers,
volumes=volumes,
node_selector=node_selector,
affinity=affinity,
tolerations=tolerations)
def get_task_pod_spec(self,
volume_mounts,
volumes,
resource_name,
persistence_outputs=None,
persistence_data=None,
outputs_refs_jobs=None,
outputs_refs_experiments=None,
env_vars=None,
command=None,
args=None,
resources=None,
ports=None,
secret_refs=None,
configmap_refs=None,
ephemeral_token=None,
node_selector=None,
affinity=None,
tolerations=None,
restart_policy='OnFailure'):
"""Pod spec to be used to create pods for tasks: master, worker, ps."""
volume_mounts = to_list(volume_mounts, check_none=True)
volumes = to_list(volumes, check_none=True)
gpu_volume_mounts, gpu_volumes = get_gpu_volumes_def(resources)
volume_mounts += gpu_volume_mounts
volumes += gpu_volumes
pod_container = self.get_pod_container(volume_mounts=volume_mounts,
persistence_outputs=persistence_outputs,
persistence_data=persistence_data,
outputs_refs_jobs=outputs_refs_jobs,
outputs_refs_experiments=outputs_refs_experiments,
secret_refs=secret_refs,
configmap_refs=configmap_refs,
resources=resources,
env_vars=env_vars,
command=command,
args=args,
ports=ports,
ephemeral_token=ephemeral_token)
containers = [pod_container]
if self.use_sidecar:
sidecar_container = self.get_sidecar_container(resource_name=resource_name)
containers.append(sidecar_container)
node_selector = self._get_node_selector(node_selector=node_selector)
affinity = self._get_affinity(affinity=affinity)
tolerations = self._get_tolerations(tolerations=tolerations)
service_account_name = self._get_service_account_name()
return client.V1PodSpec(
restart_policy=restart_policy,
service_account_name=service_account_name,
init_containers=to_list(self.get_init_container(persistence_outputs), check_none=True),
containers=containers,
volumes=volumes,
node_selector=node_selector,
tolerations=tolerations,
affinity=affinity)
def get_task_pod_spec(self,
task_type,
task_idx,
volume_mounts,
volumes,
env_vars=None,
command=None,
args=None,
sidecar_args=None,
resources=None,
node_selector=None,
restart_policy='OnFailure'):
"""Pod spec to be used to create pods for tasks: master, worker, ps."""
volume_mounts = get_list(volume_mounts)
volumes = get_list(volumes)
gpu_volume_mounts, gpu_volumes = get_gpu_volumes_def(resources)
volume_mounts += gpu_volume_mounts
volumes += gpu_volumes
# Add job information
env_vars = get_list(env_vars)
env_vars.append(
client.V1EnvVar(name=constants.CONFIG_MAP_TASK_INFO_KEY_NAME,
value=json.dumps({
'type': task_type,
'index': task_idx
})))
pod_container = self.get_pod_container(volume_mounts=volume_mounts,
env_vars=env_vars,
command=command,
args=args,
resources=resources)
containers = [pod_container]
if self.use_sidecar:
sidecar_container = self.get_sidecar_container(task_type=task_type,
task_idx=task_idx,
args=sidecar_args)
containers.append(sidecar_container)
if not node_selector:
node_selector = settings.NODE_SELECTORS_EXPERIMENTS
node_selector = json.loads(
node_selector) if node_selector else None
service_account_name = None
if settings.K8S_RBAC_ENABLED:
service_account_name = settings.K8S_SERVICE_ACCOUNT_NAME
return client.V1PodSpec(restart_policy=restart_policy,
service_account_name=service_account_name,
init_containers=to_list(
self.get_init_container()),
containers=containers,
volumes=volumes,
node_selector=node_selector)
def get_task_pod_spec(self,
task_type,
task_idx,
volume_mounts,
volumes,
env_vars=None,
command=None,
args=None,
sidecar_args=None,
resources=None,
node_selector=None,
restart_policy='OnFailure'):
"""Pod spec to be used to create pods for tasks: master, worker, ps."""
volume_mounts = get_list(volume_mounts)
volumes = get_list(volumes)
gpu_volume_mounts, gpu_volumes = get_gpu_volumes_def(resources)
volume_mounts += gpu_volume_mounts
volumes += gpu_volumes
# Add job information
env_vars = get_list(env_vars)
env_vars.append(
client.V1EnvVar(
name=constants.CONFIG_MAP_TASK_INFO_KEY_NAME,
value=json.dumps({'type': task_type, 'index': task_idx})
)
)
pod_container = self.get_pod_container(volume_mounts=volume_mounts,
env_vars=env_vars,
command=command,
args=args,
resources=resources)
containers = [pod_container]
if self.use_sidecar:
sidecar_container = self.get_sidecar_container(task_type=task_type,
task_idx=task_idx,
args=sidecar_args)
containers.append(sidecar_container)
if not node_selector:
node_selector = settings.NODE_SELECTORS_EXPERIMENTS
node_selector = json.loads(node_selector) if node_selector else None
service_account_name = None
if settings.K8S_RBAC_ENABLED:
service_account_name = settings.K8S_SERVICE_ACCOUNT_NAME
return client.V1PodSpec(restart_policy=restart_policy,
service_account_name=service_account_name,
init_containers=to_list(self.get_init_container()),
containers=containers,
volumes=volumes,
node_selector=node_selector)
def get_project_pod_spec(volume_mounts,
volumes,
image,
command,
args,
ports,
env_vars=None,
env_from=None,
container_name=None,
resources=None,
node_selector=None,
affinity=None,
tolerations=None,
image_pull_policy=None,
restart_policy=None,
use_service_account=False):
"""Pod spec to be used to create pods for project: tensorboard, notebooks."""
env_vars = get_list(env_vars)
volume_mounts = get_list(volume_mounts)
volumes = get_list(volumes)
gpu_volume_mounts, gpu_volumes = get_gpu_volumes_def(resources)
volume_mounts += gpu_volume_mounts
volumes += gpu_volumes
ports = [client.V1ContainerPort(container_port=port) for port in ports]
env_vars += get_resources_env_vars(resources=resources)
containers = [
client.V1Container(name=container_name,
image=image,
image_pull_policy=image_pull_policy,
command=command,
args=args,
ports=ports,
env=env_vars,
env_from=env_from,
resources=get_resources(resources),
volume_mounts=volume_mounts)
]
service_account_name = None
if use_service_account and settings.K8S_RBAC_ENABLED:
service_account_name = settings.K8S_SERVICE_ACCOUNT_NAME
return client.V1PodSpec(restart_policy=restart_policy,
service_account_name=service_account_name,
containers=containers,
volumes=volumes,
node_selector=node_selector,
affinity=affinity,
tolerations=tolerations)
def get_project_pod_spec(volume_mounts,
volumes,
image,
command,
args,
ports,
env_vars=None,
env_from=None,
container_name=None,
resources=None,
node_selector=None,
affinity=None,
tolerations=None,
image_pull_policy=None,
restart_policy=None,
service_account_name=None):
"""Pod spec to be used to create pods for project: tensorboard, notebooks."""
env_vars = to_list(env_vars, check_none=True)
volume_mounts = to_list(volume_mounts, check_none=True)
volumes = to_list(volumes, check_none=True)
gpu_volume_mounts, gpu_volumes = get_gpu_volumes_def(resources)
volume_mounts += gpu_volume_mounts
volumes += gpu_volumes
ports = [client.V1ContainerPort(container_port=port) for port in ports]
env_vars += get_resources_env_vars(resources=resources)
containers = [
client.V1Container(name=container_name,
image=image,
image_pull_policy=image_pull_policy,
command=command,
args=args,
ports=ports,
env=env_vars,
env_from=env_from,
resources=get_resources(resources),
volume_mounts=volume_mounts)
]
if service_account_name and not conf.get('K8S_RBAC_ENABLED'):
service_account_name = None
return client.V1PodSpec(restart_policy=restart_policy,
service_account_name=service_account_name,
containers=containers,
volumes=volumes,
node_selector=node_selector,
affinity=affinity,
tolerations=tolerations)
def get_task_pod_spec(self,
volume_mounts,
volumes,
env_vars=None,
command=None,
args=None,
resources=None,
node_selector=None,
restart_policy='OnFailure'):
"""Pod spec to be used to create pods for tasks: master, worker, ps."""
volume_mounts = get_list(volume_mounts)
volumes = get_list(volumes)
gpu_volume_mounts, gpu_volumes = get_gpu_volumes_def(resources)
volume_mounts += gpu_volume_mounts
volumes += gpu_volumes
pod_container = self.get_pod_container(volume_mounts=volume_mounts,
env_vars=env_vars,
command=command,
args=args,
resources=resources)
containers = [pod_container]
if self.use_sidecar:
sidecar_container = self.get_sidecar_container()
containers.append(sidecar_container)
if not node_selector:
node_selector = settings.NODE_SELECTORS_EXPERIMENTS
node_selector = json.loads(
node_selector) if node_selector else None
service_account_name = None
if settings.K8S_RBAC_ENABLED:
service_account_name = settings.K8S_SERVICE_ACCOUNT_NAME
return client.V1PodSpec(restart_policy=restart_policy,
service_account_name=service_account_name,
init_containers=to_list(
self.get_init_container()),
containers=containers,
volumes=volumes,
node_selector=node_selector)
def get_project_pod_spec(volume_mounts,
volumes,
image,
command,
args,
ports,
env_vars=None,
container_name=None,
resources=None,
node_selector=None,
restart_policy=None,
use_service_account=False):
"""Pod spec to be used to create pods for project: tensorboard, notebooks."""
volume_mounts = get_list(volume_mounts)
volumes = get_list(volumes)
gpu_volume_mounts, gpu_volumes = get_gpu_volumes_def(resources)
volume_mounts += gpu_volume_mounts
volumes += gpu_volumes
ports = [client.V1ContainerPort(container_port=port) for port in ports]
containers = [client.V1Container(name=container_name,
image=image,
command=command,
args=args,
ports=ports,
env=env_vars,
resources=get_resources(resources),
volume_mounts=volume_mounts)]
service_account_name = None
if use_service_account and settings.K8S_RBAC_ENABLED:
service_account_name = settings.K8S_SERVICE_ACCOUNT_NAME
return client.V1PodSpec(restart_policy=restart_policy,
service_account_name=service_account_name,
containers=containers,
volumes=volumes,
node_selector=node_selector)
def get_task_pod_spec(self,
volume_mounts,
volumes,
env_vars=None,
command=None,
args=None,
resources=None,
node_selector=None,
restart_policy='OnFailure'):
"""Pod spec to be used to create pods for tasks: master, worker, ps."""
volume_mounts = get_list(volume_mounts)
volumes = get_list(volumes)
gpu_volume_mounts, gpu_volumes = get_gpu_volumes_def(resources)
volume_mounts += gpu_volume_mounts
volumes += gpu_volumes
pod_container = self.get_pod_container(volume_mounts=volume_mounts,
env_vars=env_vars,
command=command,
args=args,
resources=resources)
containers = [pod_container]
if self.use_sidecar:
sidecar_container = self.get_sidecar_container()
containers.append(sidecar_container)
if not node_selector:
node_selector = settings.NODE_SELECTORS_EXPERIMENTS
node_selector = json.loads(node_selector) if node_selector else None
service_account_name = None
if settings.K8S_RBAC_ENABLED:
service_account_name = settings.K8S_SERVICE_ACCOUNT_NAME
return client.V1PodSpec(restart_policy=restart_policy,
service_account_name=service_account_name,
init_containers=to_list(self.get_init_container()),
containers=containers,
volumes=volumes,
node_selector=node_selector)
def get_task_pod_spec(self,
task_type,
task_idx,
volume_mounts,
volumes,
env_vars=None,
command=None,
args=None,
sidecar_args=None,
persistence_outputs=None,
persistence_data=None,
outputs_refs_jobs=None,
outputs_refs_experiments=None,
resources=None,
node_selector=None,
affinity=None,
tolerations=None,
restart_policy='OnFailure'):
"""Pod spec to be used to create pods for tasks: master, worker, ps."""
volume_mounts = get_list(volume_mounts)
volumes = get_list(volumes)
gpu_volume_mounts, gpu_volumes = get_gpu_volumes_def(resources)
volume_mounts += gpu_volume_mounts
volumes += gpu_volumes
# Add job information
env_vars = get_list(env_vars)
env_vars.append(
client.V1EnvVar(
name=constants.CONFIG_MAP_TASK_INFO_KEY_NAME,
value=json.dumps({'type': task_type, 'index': task_idx})
)
)
pod_container = self.get_pod_container(volume_mounts=volume_mounts,
env_vars=env_vars,
command=command,
args=args,
persistence_outputs=persistence_outputs,
persistence_data=persistence_data,
outputs_refs_jobs=outputs_refs_jobs,
outputs_refs_experiments=outputs_refs_experiments,
resources=resources)
containers = [pod_container]
if self.use_sidecar:
sidecar_container = self.get_sidecar_container(task_type=task_type,
task_idx=task_idx,
args=sidecar_args)
containers.append(sidecar_container)
node_selector = get_node_selector(
node_selector=node_selector,
default_node_selector=settings.NODE_SELECTOR_EXPERIMENTS)
affinity = get_affinity(
affinity=affinity,
default_affinity=settings.AFFINITY_EXPERIMENTS)
tolerations = get_tolerations(
tolerations=tolerations,
default_tolerations=settings.TOLERATIONS_EXPERIMENTS)
service_account_name = None
if settings.K8S_RBAC_ENABLED:
service_account_name = settings.K8S_SERVICE_ACCOUNT_NAME
return client.V1PodSpec(
restart_policy=restart_policy,
service_account_name=service_account_name,
init_containers=to_list(self.get_init_container(persistence_outputs)),
containers=containers,
volumes=volumes,
node_selector=node_selector,
tolerations=tolerations,
affinity=affinity)
