def test_get_artifacts_path_container_with_none_values(self):
with self.assertRaises(PolypodException):
get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(),
artifacts_store=None,
run_path="",
auto_resume=True,
)
def test_get_artifacts_path_container_with_non_managed_mount_store(self):
store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/claim/path",
volume_claim="claim"),
)
container = get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"),
artifacts_store=store,
run_path="run_uid",
auto_resume=True,
)
init_args = init_artifact_context_args("run_uid")
init_args.append(
get_volume_args(
store=store,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
artifacts=V1ArtifactsType(dirs=["run_uid"]),
))
assert container == get_base_store_container(
container=k8s_schemas.V1Container(name="init"),
container_name=generate_container_name(
INIT_ARTIFACTS_CONTAINER_PREFIX, "default", False),
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"),
store=store,
env=[],
env_from=[],
volume_mounts=[get_artifacts_context_mount()],
args=[" ".join(init_args)],
)
container = get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"),
artifacts_store=store,
run_path="run_uid",
auto_resume=False,
)
init_args = init_artifact_context_args("run_uid")
assert container == get_base_store_container(
container=k8s_schemas.V1Container(name="init"),
container_name=generate_container_name(
INIT_ARTIFACTS_CONTAINER_PREFIX, "default", False),
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"),
store=store,
env=[],
env_from=[],
volume_mounts=[get_artifacts_context_mount()],
args=[" ".join(init_args)],
)
def test_get_artifacts_path_container_with_bucket_store(self):
store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
)
container = get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"),
artifacts_store=store,
run_path="run_uid",
auto_resume=True,
)
init_args = init_artifact_context_args("run_uid")
init_args.append(
get_volume_args(
store=store,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
artifacts=V1ArtifactsType(dirs=["run_uid"]),
))
assert container == get_base_store_container(
container=k8s_schemas.V1Container(name="default"),
container_name=INIT_ARTIFACTS_CONTAINER.format("default"),
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"),
store=store,
env=[],
env_from=[],
volume_mounts=[get_artifacts_context_mount()],
args=[" ".join(init_args)],
)
def test_get_artifacts_path_container_with_managed_mount_store(self):
store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/claim/path", volume_claim="claim"),
)
container = get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"
),
artifacts_store=store,
run_path="run_uid",
clean=True,
)
init_args = init_artifact_context_args("run_uid")
init_args.append(
get_artifacts_store_args(
artifacts_path=get_path(store.store_path, "run_uid"), clean=True
)
)
assert container == get_base_store_container(
container=k8s_schemas.V1Container(name="default"),
container_name=INIT_ARTIFACTS_CONTAINER.format("default"),
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"
),
store=store,
env=[],
env_from=[],
volume_mounts=[get_artifacts_context_mount()],
args=[" ".join(init_args)],
is_artifact_store=True,
)
def get_init_containers(
self,
polyaxon_init: V1PolyaxonInitContainer,
contexts: PluginsContextsSpec,
artifacts_store: V1ConnectionType,
init_connections: List[V1Init],
init_containers: List[k8s_schemas.V1Container],
connection_by_names: Dict[str, V1ConnectionType],
) -> List[k8s_schemas.V1Container]:
init_containers = [
ensure_container_name(container=c, prefix=INIT_PREFIX)
for c in to_list(init_containers, check_none=True)
]
init_connections = to_list(init_connections, check_none=True)
containers = []
# Add auth context
if contexts and contexts.auth:
containers.append(
get_auth_context_container(
polyaxon_init=polyaxon_init,
env=self.get_auth_service_env_vars(
external_host=contexts.external_host),
))
# Add outputs
if contexts and contexts.collect_artifacts:
containers += to_list(
get_artifacts_path_container(
polyaxon_init=polyaxon_init,
artifacts_store=artifacts_store,
run_path=self.run_path,
auto_resume=contexts.auto_resume,
),
check_none=True,
)
containers += self.handle_init_connections(
polyaxon_init=polyaxon_init,
artifacts_store=artifacts_store,
init_connections=init_connections,
connection_by_names=connection_by_names,
contexts=contexts,
)
init_containers = containers + init_containers
return [sanitize_container(c) for c in init_containers]
def test_get_init_containers_with_store_outputs(self):
store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3://foo"),
)
# No context
containers = self.converter.get_init_containers(
contexts=None,
artifacts_store=store,
init_connections=[],
init_containers=[],
connection_by_names={},
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == []
# With context
containers = self.converter.get_init_containers(
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=True,
collect_logs=False,
auth=True)),
artifacts_store=store,
init_connections=[],
init_containers=[],
connection_by_names={},
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_auth_context_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
env=self.converter.get_auth_service_env_vars(),
),
get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
artifacts_store=store,
run_path=self.converter.run_path,
auto_resume=True,
),
]
def get_init_containers(
self,
polyaxon_init: V1PolyaxonInitContainer,
contexts: PluginsContextsSpec,
artifacts_store: V1ConnectionType,
init_connections: List[V1Init],
init_containers: List[k8s_schemas.V1Container],
connection_by_names: Dict[str, V1ConnectionType],
is_resume: bool = False,
) -> List[k8s_schemas.V1Container]:
init_containers = init_containers or []
init_connections = init_connections or []
containers = []
# Add auth context
if contexts and contexts.auth:
containers.append(
get_auth_context_container(
polyaxon_init=polyaxon_init,
env=self.get_auth_service_env_vars()))
# Add outputs
if contexts and contexts.collect_artifacts:
containers += to_list(
get_artifacts_path_container(
polyaxon_init=polyaxon_init,
artifacts_store=artifacts_store,
run_path=self.run_path,
clean=not is_resume,
),
check_none=True,
)
containers += self.handle_init_connections(
polyaxon_init=polyaxon_init,
init_connections=init_connections,
connection_by_names=connection_by_names,
contexts=contexts,
)
return containers + init_containers
def test_get_init_containers_with_claim_outputs(self):
store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/claim/path",
volume_claim="claim",
read_only=True),
)
# No context to enable the outputs
containers = self.converter.get_init_containers(
contexts=None,
artifacts_store=store.name,
init_connections=None,
connection_by_names={},
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == []
# Enable outputs
containers = self.converter.get_init_containers(
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=True, collect_logs=False)),
artifacts_store=store,
connection_by_names={},
init_connections=None,
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
artifacts_store=store,
run_path=self.converter.run_path,
auto_resume=True,
),
]
# Use store for init
containers = self.converter.get_init_containers(
contexts=None,
artifacts_store=None,
connection_by_names={store.name: store},
init_connections=[V1Init(connection=store.name)],
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_store_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
connection=store,
artifacts=None,
env=self.converter.get_init_service_env_vars(),
)
]
# Use store for init and outputs
containers = self.converter.get_init_containers(
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=True, collect_logs=False)),
artifacts_store=store,
init_connections=[V1Init(connection=store.name)],
connection_by_names={store.name: store},
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
artifacts_store=store,
run_path=self.converter.run_path,
auto_resume=True,
),
get_store_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
connection=store,
artifacts=None,
env=self.converter.get_init_service_env_vars(),
),
]
# Add Store
store1 = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3://foo"),
secret=None,
)
containers = self.converter.get_init_containers(
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=True,
collect_logs=False,
auth=True)),
artifacts_store=store,
init_connections=[
V1Init(
connection=store.name,
artifacts=V1ArtifactsType(files=["/foo", "/bar"]),
),
V1Init(
connection=store1.name,
artifacts=V1ArtifactsType(files=["/foo", "/bar"]),
),
],
connection_by_names={
store.name: store,
store1.name: store1
},
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_auth_context_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
env=self.converter.get_auth_service_env_vars(),
),
get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
artifacts_store=store,
run_path=self.converter.run_path,
auto_resume=True,
),
get_store_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
connection=store,
artifacts=V1ArtifactsType(files=["/foo", "/bar"]),
env=self.converter.get_init_service_env_vars(),
),
get_store_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
connection=store1,
artifacts=V1ArtifactsType(files=["/foo", "/bar"]),
env=self.converter.get_init_service_env_vars(),
),
]
