def test_default_volumes(self):
assert (get_pod_volumes(
contexts=None,
artifacts_store=None,
init_connections=None,
connection_by_names=None,
secrets=None,
config_maps=None,
volumes=None,
) == [])
assert (get_pod_volumes(
contexts=None,
artifacts_store=None,
init_connections=[],
connection_by_names={},
secrets=[],
config_maps=[],
volumes=[],
) == [])
assert (get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=False,
shm=False,
auth=False,
collect_artifacts=False,
collect_logs=False,
)),
artifacts_store=None,
init_connections=[],
connection_by_names={},
secrets=[],
config_maps=[],
volumes=[],
) == [])
assert get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=True,
shm=True,
auth=True,
collect_artifacts=False,
collect_logs=False,
)),
artifacts_store=None,
init_connections=[],
connection_by_names={},
secrets=[],
config_maps=[],
volumes=[],
) == [
get_shm_context_volume(),
get_configs_context_volume(),
get_docker_context_volume(),
]
def test_multiple_resources(self):
assert (
get_pod_volumes(
contexts=None,
artifacts_store=None,
init_connections=[],
connection_by_names={},
connections=[],
secrets=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
config_maps=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
volumes=[],
)
== []
)
# Make the resources requested
self.non_mount_resource1.is_requested = True
self.non_mount_resource2.is_requested = True
self.mount_resource1.is_requested = True
self.mount_resource2.is_requested = True
assert get_pod_volumes(
contexts=None,
artifacts_store=None,
init_connections=[],
connection_by_names={},
connections=[],
secrets=[
self.non_mount_resource1,
self.non_mount_resource2,
self.mount_resource1,
self.mount_resource2,
],
config_maps=[
self.non_mount_resource1,
self.non_mount_resource2,
self.mount_resource1,
self.mount_resource2,
],
volumes=[],
) == [
get_volume_from_secret(secret=self.mount_resource1),
get_volume_from_secret(secret=self.mount_resource2),
get_volume_from_config_map(config_map=self.mount_resource1),
get_volume_from_config_map(config_map=self.mount_resource2),
]
def assert_config_map(config_map, connection, results):
dummy_connection = (
V1ConnectionType(
name="connection",
kind=V1ConnectionKind.S3,
schema=None,
config_map=config_map.schema,
)
if connection
else None
)
connection_by_names = {"connection": dummy_connection} if connection else {}
connections = ["connection"] if connection else []
assert (
get_pod_volumes(
contexts=None,
artifacts_store=None,
init_connections=[],
connections=connections,
connection_by_names=connection_by_names,
secrets=[],
config_maps=[config_map],
volumes=[],
)
== results
)
def assert_config_map(config_map, results):
assert (get_pod_volumes(
contexts=None,
artifacts_store=None,
init_connections=[],
connection_by_names={},
secrets=[],
config_maps=[config_map],
volumes=[],
) == results)
def assert_single_init_artifacts_store(store, results):
assert (get_pod_volumes(
contexts=None,
artifacts_store=None,
init_connections=[V1Init(connection=store.name)],
connection_by_names={store.name: store},
secrets=[],
config_maps=[],
volumes=[],
) == results)
def assert_artifacts_store(store, results):
assert (get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=False,
shm=False,
auth=False,
collect_artifacts=True,
collect_logs=False,
)),
artifacts_store=store,
init_connections=[],
connection_by_names={},
secrets=[],
config_maps=[],
volumes=[],
) == results)
def test_passing_volumes(self):
assert get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=False,
shm=False,
auth=False,
collect_artifacts=False,
collect_logs=False,
)),
artifacts_store=None,
init_connections=[],
connection_by_names={},
secrets=[],
config_maps=[],
volumes=[self.vol1, self.vol2, self.vol3],
) == [self.vol1, self.vol2, self.vol3]
def get_replica_resource(
self,
environment: V1Environment,
plugins: V1Plugins,
contexts: PluginsContextsSpec,
volumes: List[k8s_schemas.V1Volume],
init: List[V1Init],
sidecars: List[k8s_schemas.V1Container],
container: k8s_schemas.V1Container,
artifacts_store: V1ConnectionType,
connections: List[str],
connection_by_names: Dict[str, V1ConnectionType],
secrets: Optional[Iterable[V1K8sResourceType]],
config_maps: Optional[Iterable[V1K8sResourceType]],
default_sa: str = None,
ports: List[int] = None,
num_replicas: int = None,
) -> ReplicaSpec:
volumes = volumes or []
init = init or []
sidecars = sidecars or []
connections = connections or []
environment = environment or V1Environment()
environment.service_account_name = (environment.service_account_name
or default_sa
or settings.AGENT_CONFIG.runs_sa)
init_connections = self.filter_connections_from_init(init=init)
volumes = get_pod_volumes(
contexts=contexts,
artifacts_store=artifacts_store,
init_connections=init_connections,
connections=connections,
connection_by_names=connection_by_names,
secrets=secrets,
config_maps=config_maps,
volumes=volumes,
)
init_containers = self.get_init_containers(
polyaxon_init=self.polyaxon_init,
contexts=contexts,
artifacts_store=artifacts_store,
init_connections=init_connections,
init_containers=self.filter_containers_from_init(init=init),
connection_by_names=connection_by_names,
)
sidecar_containers = self.get_sidecar_containers(
polyaxon_sidecar=self.polyaxon_sidecar,
contexts=contexts,
artifacts_store=artifacts_store,
sidecar_containers=sidecars,
)
main_container = self.get_main_container(
main_container=container,
contexts=contexts,
artifacts_store=artifacts_store,
connections=connections,
init_connections=init_connections,
connection_by_names=connection_by_names,
log_level=plugins.log_level,
secrets=secrets,
config_maps=config_maps,
ports=ports,
)
labels = self.get_labels(version=pkg.VERSION,
labels=environment.labels)
return ReplicaSpec(
volumes=volumes,
init_containers=init_containers,
sidecar_containers=sidecar_containers,
main_container=main_container,
labels=labels,
environment=environment,
num_replicas=num_replicas,
)
def test_all_volumes_and_artifacts_store(self):
connection_by_names = {
self.s3_store.name: self.s3_store,
self.gcs_store.name: self.gcs_store,
self.az_store.name: self.az_store,
self.claim_store.name: self.claim_store,
self.host_path_store.name: self.host_path_store,
}
init_connections = [
V1Init(connection=self.s3_store.name, path="/test-1"),
V1Init(connection=self.gcs_store.name, path="/test-2"),
V1Init(connection=self.az_store.name, path="/test-3"),
V1Init(connection=self.claim_store.name, path="/test-4"),
V1Init(connection=self.host_path_store.name, path="/test-5"),
]
pod_volumes = get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=True,
shm=True,
auth=True,
collect_artifacts=False,
collect_logs=False,
)
),
artifacts_store=None,
init_connections=init_connections,
connection_by_names=connection_by_names,
connections=[],
secrets=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
config_maps=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
volumes=[self.vol1, self.vol2, self.vol3],
)
# 3: 3 context requested constant contexts
# 3: 3 volumes
# 5: 5 managed contexts + 2 mount volumes
# 1: 1 secret
assert len(pod_volumes) == 3 + 3 + 7 + 1
pod_volumes = get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=True,
shm=True,
auth=True,
collect_artifacts=False,
collect_logs=False,
)
),
artifacts_store=None,
init_connections=init_connections,
connections=list(connection_by_names.keys()),
connection_by_names=connection_by_names,
secrets=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
config_maps=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
volumes=[self.vol1, self.vol2, self.vol3],
)
# 3: 3 context requested constant contexts
# 3: 3 volumes
# 7: 5 managed contexts + 2 mount volumes
assert len(pod_volumes) == 3 + 3 + 7 + 1
pod_volumes = get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=True,
shm=True,
auth=True,
collect_artifacts=True,
collect_logs=True,
)
),
artifacts_store=self.host_path_store,
init_connections=[
V1Init(connection=self.s3_store.name),
V1Init(connection=self.gcs_store.name),
V1Init(connection=self.az_store.name),
],
connections=list(connection_by_names.keys()),
connection_by_names=connection_by_names,
secrets=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
config_maps=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
volumes=[self.vol1, self.vol2, self.vol3],
)
# 4: 4 context requested constant contexts / init volumes contexts
# 3: 3 volumes
# 2: 2 managed volumes
assert len(pod_volumes) == 4 + 2 + 3 + 1
# Enable requesting resources
self.mount_resource1.is_requested = True
self.mount_resource2.is_requested = True
pod_volumes = get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=True,
shm=True,
auth=True,
collect_artifacts=True,
collect_logs=True,
)
),
artifacts_store=self.host_path_store,
init_connections=[
V1Init(connection=self.s3_store.name),
V1Init(connection=self.gcs_store.name),
V1Init(connection=self.az_store.name),
],
connections=list(connection_by_names.keys()),
connection_by_names=connection_by_names,
secrets=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
config_maps=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
volumes=[self.vol1, self.vol2, self.vol3],
)
# 4: 4 context requested constant contexts / init volumes contexts
# 3: 3 volumes
# 2: 2 managed volumes
# 4: 4 mount resources (secrets + configs)
assert len(pod_volumes) == 4 + 2 + 3 + 4
def test_all_volumes_and_init_in_the_same_context(self):
connection_by_names = {
self.s3_store.name: self.s3_store,
self.gcs_store.name: self.gcs_store,
self.az_store.name: self.az_store,
self.claim_store.name: self.claim_store,
self.host_path_store.name: self.host_path_store,
}
# Test all init are in the same context
pod_volumes = get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=True,
shm=True,
auth=True,
collect_artifacts=True,
collect_logs=True,
)
),
artifacts_store=self.claim_store,
init_connections=[
V1Init(connection=self.s3_store.name),
V1Init(connection=self.gcs_store.name),
V1Init(connection=self.az_store.name),
V1Init(connection=self.claim_store.name),
V1Init(connection=self.host_path_store.name),
],
connections=[],
connection_by_names=connection_by_names,
secrets=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
config_maps=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
volumes=[self.vol1, self.vol2, self.vol3],
)
# 1: logs/output contexts (same volume) / 1 managed contexts
# 3: 3 context requested constant contexts
# 3: 3 volumes
# 7: 2 mount volumes
# 1: 1 mount secret
assert len(pod_volumes) == 1 + 3 + 3 + 2 + 1
# Test all init are in the same context
pod_volumes = get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=True,
shm=True,
auth=True,
collect_artifacts=True,
collect_logs=True,
)
),
artifacts_store=self.claim_store,
init_connections=[
V1Init(connection=self.s3_store.name),
V1Init(connection=self.gcs_store.name),
V1Init(connection=self.az_store.name),
V1Init(connection=self.claim_store.name),
V1Init(connection=self.host_path_store.name),
],
connections=list(connection_by_names.keys()),
connection_by_names=connection_by_names,
secrets=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
config_maps=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
volumes=[self.vol1, self.vol2, self.vol3],
)
# 1: logs/output contexts (same volume) / 1 managed contexts
# 3: 3 context requested constant contexts
# 3: 3 volumes
# 7: 2 mount volumes
# 4: 1 mount resources (secrets + configs)
assert len(pod_volumes) == 1 + 3 + 3 + 2 + 1
# Enable requesting resources
self.mount_resource1.is_requested = True
self.mount_resource2.is_requested = True
# Test all init are in the same context and requested values
pod_volumes = get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=True,
shm=True,
auth=True,
collect_artifacts=True,
collect_logs=True,
)
),
artifacts_store=self.claim_store,
init_connections=[
V1Init(connection=self.s3_store.name),
V1Init(connection=self.gcs_store.name),
V1Init(connection=self.az_store.name),
V1Init(connection=self.claim_store.name),
V1Init(connection=self.host_path_store.name),
],
connections=list(connection_by_names.keys()),
connection_by_names=connection_by_names,
secrets=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
config_maps=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
volumes=[self.vol1, self.vol2, self.vol3],
)
# 1: logs/output contexts (same volume) / 1 managed contexts
# 3: 3 context requested constant contexts
# 3: 3 volumes
# 7: 2 mount volumes
# 4: 4 mount resources (secrets + configs)
assert len(pod_volumes) == 1 + 3 + 3 + 2 + 4
def test_multi_connections(self):
connection_by_names = {
self.s3_store.name: self.s3_store,
self.gcs_store.name: self.gcs_store,
self.az_store.name: self.az_store,
self.claim_store.name: self.claim_store,
self.host_path_store.name: self.host_path_store,
}
init_connections = [
V1Init(connection=self.s3_store.name, path="/test-1"),
V1Init(connection=self.gcs_store.name, path="/test-2"),
V1Init(connection=self.az_store.name, path="/test-3"),
V1Init(connection=self.claim_store.name, path="/test-4"),
V1Init(connection=self.host_path_store.name, path="/test-5"),
]
assert (
len(
get_pod_volumes(
contexts=None,
artifacts_store=None,
init_connections=[],
connection_by_names=connection_by_names,
connections=[],
secrets=[],
config_maps=[],
volumes=[],
)
)
== 2
)
# test all inits are mounted to the same context and a single secret requested for all
assert (
len(
get_pod_volumes(
contexts=None,
artifacts_store=None,
init_connections=[
V1Init(connection=self.s3_store.name),
V1Init(connection=self.gcs_store.name),
V1Init(connection=self.az_store.name),
V1Init(connection=self.claim_store.name),
V1Init(connection=self.host_path_store.name),
],
connection_by_names=connection_by_names,
connections=[],
secrets=[],
config_maps=[],
volumes=[],
)
)
== 4
)
assert (
len(
get_pod_volumes(
contexts=None,
artifacts_store=None,
init_connections=init_connections,
connection_by_names=connection_by_names,
connections=[],
secrets=[],
config_maps=[],
volumes=[],
)
)
== 8
)
assert (
len(
get_pod_volumes(
contexts=None,
artifacts_store=None,
init_connections=init_connections,
connection_by_names=connection_by_names,
connections=[],
secrets=[],
config_maps=[],
volumes=[],
)
)
== 8
)
assert (
len(
get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=True,
shm=True,
auth=True,
collect_artifacts=True,
collect_logs=True,
)
),
artifacts_store=self.claim_store,
init_connections=init_connections,
connection_by_names=connection_by_names,
connections=[],
secrets=[],
config_maps=[],
volumes=[],
)
)
== 12
)
assert (
len(
get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=True,
shm=True,
auth=True,
collect_artifacts=True,
collect_logs=True,
)
),
artifacts_store=self.claim_store,
init_connections=init_connections,
connection_by_names=connection_by_names,
connections=list(connection_by_names.keys()),
secrets=[],
config_maps=[],
volumes=[],
)
)
== 12
)
def test_all_volumes(self):
connection_by_names = {
self.s3_store.name: self.s3_store,
self.gcs_store.name: self.gcs_store,
self.az_store.name: self.az_store,
self.claim_store.name: self.claim_store,
self.host_path_store.name: self.host_path_store,
}
# Test all init are in the same context
assert (
len(
get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=True,
shm=True,
auth=True,
collect_artifacts=True,
collect_logs=True,
)),
artifacts_store=self.claim_store,
init_connections=[
V1Init(connection=self.s3_store.name),
V1Init(connection=self.gcs_store.name),
V1Init(connection=self.az_store.name),
V1Init(connection=self.claim_store.name),
V1Init(connection=self.host_path_store.name),
],
connection_by_names=connection_by_names,
secrets=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
config_maps=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
volumes=[self.vol1, self.vol2, self.vol3],
))
# 1: logs/output contexts (same volume) / 1 managed contexts
# 3: 3 context requested constant contexts
# 3: 3 volumes
# 7: 2 mount volumes
# 4: 4 mount resources (secrets + configs)
== 1 + 3 + 3 + 2 + 4)
init_connections = [
V1Init(connection=self.s3_store.name, path="/test-1"),
V1Init(connection=self.gcs_store.name, path="/test-2"),
V1Init(connection=self.az_store.name, path="/test-3"),
V1Init(connection=self.claim_store.name, path="/test-4"),
V1Init(connection=self.host_path_store.name, path="/test-5"),
]
assert (
len(
get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=True,
shm=True,
auth=True,
collect_artifacts=True,
collect_logs=True,
)),
artifacts_store=self.claim_store,
init_connections=init_connections,
connection_by_names=connection_by_names,
secrets=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
config_maps=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
volumes=[self.vol1, self.vol2, self.vol3],
))
# 1: logs/output contexts (same volume)
# 3: 3 context requested constant contexts
# 3: 3 volumes
# 7: 5 managed contexts + 2 mount volumes
# 4: 4 mount resources (secrets + configs)
== 1 + 3 + 3 + 7 + 4)
assert (
len(
get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=True,
shm=True,
auth=True,
collect_artifacts=True,
collect_logs=True,
)),
artifacts_store=self.s3_store,
init_connections=init_connections,
connection_by_names=connection_by_names,
secrets=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
config_maps=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
volumes=[self.vol1, self.vol2, self.vol3],
))
# 1: logs/output contexts (same volume)
# 3: 3 context requested constant contexts
# 3: 3 volumes
# 7: 5 managed contexts + 2 mount volumes
# 4: 4 mount resources (secrets + configs)
== 1 + 3 + 3 + 7 + 4)
assert (
len(
get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=True,
shm=True,
auth=True,
collect_artifacts=False,
collect_logs=False,
)),
artifacts_store=None,
init_connections=init_connections,
connection_by_names=connection_by_names,
secrets=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
config_maps=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
volumes=[self.vol1, self.vol2, self.vol3],
))
# 3: 3 context requested constant contexts
# 3: 3 volumes
# 7: 5 managed contexts + 2 mount volumes
# 4: 4 mount resources (secrets + configs)
== 3 + 3 + 7 + 4)
assert (
len(
get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=True,
shm=True,
auth=True,
collect_artifacts=False,
collect_logs=False,
)),
artifacts_store=None,
init_connections=init_connections,
connection_by_names=connection_by_names,
secrets=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
config_maps=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
volumes=[self.vol1, self.vol2, self.vol3],
))
# 3: 3 context requested constant contexts
# 3: 3 volumes
# 7: 5 managed contexts + 2 mount volumes
# 4: 4 mount resources (secrets + configs)
== 3 + 3 + 7 + 4)
assert (
len(
get_pod_volumes(
contexts=PluginsContextsSpec.from_config(
V1Plugins(
docker=True,
shm=True,
auth=True,
collect_artifacts=True,
collect_logs=True,
)),
artifacts_store=self.host_path_store,
init_connections=[
V1Init(connection=self.s3_store.name),
V1Init(connection=self.gcs_store.name),
V1Init(connection=self.az_store.name),
],
connection_by_names=connection_by_names,
secrets=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
config_maps=[
self.non_mount_resource1,
self.non_mount_resource1,
self.mount_resource1,
self.mount_resource2,
],
volumes=[self.vol1, self.vol2, self.vol3],
))
# 4: 4 context requested constant contexts / init volumes contexts
# 3: 3 volumes
# 2: 2 managed volumes
# 4: 4 mount resources (secrets + configs)
== 4 + 2 + 3 + 4)