def test_get_store_container_mount_stores(self):
# Managed store
store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(
mount_path="/tmp", volume_claim="test", read_only=True
),
)
container = get_store_container(
polyaxon_init=V1PolyaxonInitContainer(
image="foo/foo", image_tag="foo", image_pull_policy="IfNotPresent"
),
connection=store,
artifacts=None,
)
mount_path = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name)
assert container.name == INIT_ARTIFACTS_CONTAINER.format(store.name)
assert container.image == "foo/foo:foo"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["/bin/sh", "-c"]
assert container.args == [
get_volume_args(store=store, mount_path=mount_path, artifacts=None)
]
assert container.env == get_connection_env_var(connection=store, secret=None)
assert container.env_from == []
assert container.resources is not None
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name),
),
get_mount_from_store(store=store),
]
def test_get_store_container_bucket_stores(self):
mount_path = "/test-path"
resource1 = V1K8sResourceType(
name="non_mount_test1",
schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]),
is_requested=False,
)
store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
secret=resource1.schema,
)
container = get_store_container(
polyaxon_init=V1PolyaxonInitContainer(
image="foo/foo", image_tag="", image_pull_policy="IfNotPresent"
),
connection=store,
artifacts=None,
mount_path=mount_path,
)
assert container.name == generate_container_name(
INIT_ARTIFACTS_CONTAINER_PREFIX, store.name
)
assert container.image == "foo/foo"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["/bin/sh", "-c"]
assert container.args == [
get_volume_args(store=store, mount_path=mount_path, artifacts=None)
]
assert container.env is not None
assert container.env_from == []
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=get_volume_name(mount_path), mount_path=mount_path
)
]
def handle_init_connections(
self,
polyaxon_init: V1PolyaxonInitContainer,
artifacts_store: V1ConnectionType,
init_connections: List[V1Init],
connection_by_names: Dict[str, V1ConnectionType],
contexts: PluginsContextsSpec,
) -> List[k8s_schemas.V1Container]:
containers = []
external_host = contexts.external_host if contexts else False
# Prepare connections that Polyaxon can init automatically
for init_connection in init_connections:
if init_connection.connection:
connection_spec = connection_by_names.get(
init_connection.connection)
# Handling ssh with git
if (V1ConnectionKind.is_ssh(connection_spec.kind)
and init_connection.git):
connection_spec.schema.patch_git(init_connection.git)
containers.append(
get_git_init_container(
polyaxon_init=polyaxon_init,
connection=connection_spec,
container=init_connection.container,
env=self.get_init_service_env_vars(
external_host=external_host),
mount_path=init_connection.path,
contexts=contexts,
track=True,
))
elif V1ConnectionKind.is_git(connection_spec.kind):
if init_connection.git: # Update the default schema
connection_spec.schema.patch(init_connection.git)
containers.append(
get_git_init_container(
polyaxon_init=polyaxon_init,
connection=connection_spec,
container=init_connection.container,
env=self.get_init_service_env_vars(
external_host=external_host),
mount_path=init_connection.path,
contexts=contexts,
track=True,
))
elif V1ConnectionKind.is_artifact(connection_spec.kind):
containers.append(
get_store_container(
polyaxon_init=polyaxon_init,
connection=connection_spec,
artifacts=init_connection.artifacts,
container=init_connection.container,
env=self.get_init_service_env_vars(
external_host=external_host),
mount_path=init_connection.path,
is_default_artifacts_store=artifacts_store and
init_connection.connection == artifacts_store.name,
))
else:
containers.append(
get_custom_init_container(
connection=connection_spec,
container=init_connection.container,
env=self.get_init_service_env_vars(
external_host=external_host),
mount_path=init_connection.path,
contexts=contexts,
))
else:
# artifacts init without connection should default to the artifactsStore
if init_connection.artifacts:
containers.append(
get_store_container(
polyaxon_init=polyaxon_init,
connection=artifacts_store,
artifacts=init_connection.artifacts,
container=init_connection.container,
env=self.get_init_service_env_vars(
external_host=external_host),
mount_path=init_connection.path,
is_default_artifacts_store=True,
))
# git init without connection
if init_connection.git:
git_name = init_connection.git.get_name()
containers.append(
get_git_init_container(
polyaxon_init=polyaxon_init,
connection=V1ConnectionType(
name=git_name,
kind=V1ConnectionKind.GIT,
schema=init_connection.git,
secret=None,
),
container=init_connection.container,
env=self.get_init_service_env_vars(
external_host=external_host),
mount_path=init_connection.path,
contexts=contexts,
track=False,
))
# Dockerfile initialization
if init_connection.dockerfile:
containers.append(
get_dockerfile_init_container(
polyaxon_init=polyaxon_init,
dockerfile_args=init_connection.dockerfile,
env=self.get_init_service_env_vars(
external_host=external_host),
mount_path=init_connection.path,
contexts=contexts,
run_path=self.run_path,
run_instance=self.run_instance,
))
# File initialization
if init_connection.file:
containers.append(
get_file_init_container(
polyaxon_init=polyaxon_init,
file_args=init_connection.file,
env=self.get_init_service_env_vars(
external_host=external_host),
mount_path=init_connection.path,
contexts=contexts,
run_path=self.run_path,
run_instance=self.run_instance,
))
return containers
def test_get_init_containers_with_claim_outputs(self):
store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/claim/path",
volume_claim="claim",
read_only=True),
)
# No context to enable the outputs
containers = self.converter.get_init_containers(
contexts=None,
artifacts_store=store.name,
init_connections=None,
connection_by_names={},
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == []
# Enable outputs
containers = self.converter.get_init_containers(
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=True, collect_logs=False)),
artifacts_store=store,
connection_by_names={},
init_connections=None,
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
artifacts_store=store,
run_path=self.converter.run_path,
auto_resume=True,
),
]
# Use store for init
containers = self.converter.get_init_containers(
contexts=None,
artifacts_store=None,
connection_by_names={store.name: store},
init_connections=[V1Init(connection=store.name)],
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_store_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
connection=store,
artifacts=None,
env=self.converter.get_init_service_env_vars(),
)
]
# Use store for init and outputs
containers = self.converter.get_init_containers(
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=True, collect_logs=False)),
artifacts_store=store,
init_connections=[V1Init(connection=store.name)],
connection_by_names={store.name: store},
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
artifacts_store=store,
run_path=self.converter.run_path,
auto_resume=True,
),
get_store_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
connection=store,
artifacts=None,
env=self.converter.get_init_service_env_vars(),
),
]
# Add Store
store1 = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3://foo"),
secret=None,
)
containers = self.converter.get_init_containers(
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=True,
collect_logs=False,
auth=True)),
artifacts_store=store,
init_connections=[
V1Init(
connection=store.name,
artifacts=V1ArtifactsType(files=["/foo", "/bar"]),
),
V1Init(
connection=store1.name,
artifacts=V1ArtifactsType(files=["/foo", "/bar"]),
),
],
connection_by_names={
store.name: store,
store1.name: store1
},
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_auth_context_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
env=self.converter.get_auth_service_env_vars(),
),
get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
artifacts_store=store,
run_path=self.converter.run_path,
auto_resume=True,
),
get_store_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
connection=store,
artifacts=V1ArtifactsType(files=["/foo", "/bar"]),
env=self.converter.get_init_service_env_vars(),
),
get_store_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
connection=store1,
artifacts=V1ArtifactsType(files=["/foo", "/bar"]),
env=self.converter.get_init_service_env_vars(),
),
]
def handle_init_connections(
self,
polyaxon_init: V1PolyaxonInitContainer,
init_connections: List[V1Init],
connection_by_names: Dict[str, V1ConnectionType],
contexts: PluginsContextsSpec,
) -> List[k8s_schemas.V1Container]:
containers = []
# Prepare connections that Polyaxon can inti automatically
for init_connection in init_connections:
if init_connection.connection:
connection_spec = connection_by_names.get(
init_connection.connection)
if init_connection.git: # Update the default schema
connection_spec.schema.patch(init_connection.git)
if connection_spec.kind == V1ConnectionKind.GIT:
containers.append(
get_git_init_container(
polyaxon_init=polyaxon_init,
connection=connection_spec,
container=init_connection.container,
env=self.get_init_service_env_vars(),
mount_path=init_connection.path,
contexts=contexts,
track=True,
))
if V1ConnectionKind.is_artifact(connection_spec.kind):
containers.append(
get_store_container(
polyaxon_init=polyaxon_init,
connection=connection_spec,
artifacts=init_connection.artifacts,
container=init_connection.container,
env=self.get_init_service_env_vars(),
mount_path=init_connection.path,
))
else:
# git init without connection
if init_connection.git:
git_name = init_connection.git.get_name()
containers.append(
get_git_init_container(
polyaxon_init=polyaxon_init,
connection=V1ConnectionType(
name=git_name,
kind=V1ConnectionKind.GIT,
schema=init_connection.git,
secret=None,
),
container=init_connection.container,
env=self.get_init_service_env_vars(),
mount_path=init_connection.path,
contexts=contexts,
track=False,
))
# Dockerfile initialization
if init_connection.dockerfile:
containers.append(
get_dockerfile_init_container(
polyaxon_init=polyaxon_init,
dockerfile_args=init_connection.dockerfile,
env=self.get_init_service_env_vars(),
mount_path=init_connection.path,
contexts=contexts,
run_path=self.run_path,
))
return containers
