def test_get_volume_args_az(self):
az_store = V1ConnectionType(
name="test_az",
kind=V1ConnectionKind.WASB,
schema=V1BucketConnection(bucket="Conwasb://[email protected]"),
)
path_to = "/path/to/"
path_from = os.path.join(az_store.store_path, "")
assert get_volume_args(az_store, path_to, None) == " ".join(
[
get_or_create_args(path=path_to),
cp_wasb_args(path_from=path_from, path_to=path_to, is_file=False),
]
)
az_store = V1ConnectionType(
name="test_az",
kind=V1ConnectionKind.WASB,
schema=V1BucketConnection(bucket="Conwasb://[email protected]"),
)
base_path = "/path/to/"
path_to1 = "/path/to/path1"
path_to2 = "/path/to/path2"
path_from1 = os.path.join(az_store.store_path, "path1")
path_from2 = os.path.join(az_store.store_path, "path2")
assert get_volume_args(
az_store,
"/path/to",
artifacts=V1ArtifactsType(files=["path1"], dirs=["path2"]),
) == " ".join(
[
get_or_create_args(path=base_path),
cp_wasb_args(path_from=path_from1, path_to=path_to1, is_file=True),
get_or_create_args(path=base_path),
cp_wasb_args(path_from=path_from2, path_to=path_to2, is_file=False),
]
)
def test_get_volume_args_s3(self):
s3_store = V1ConnectionType(
name="test_s3",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3//:foo"),
)
path_to = "/path-to/"
path_from = os.path.join(s3_store.store_path, "")
assert get_volume_args(s3_store, path_to, None) == "; ".join([
get_or_create_args(path=path_to),
cp_s3_args(path_from=path_from, path_to=path_to, is_file=False),
])
s3_store = V1ConnectionType(
name="test_s3",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3//:foo"),
)
base_path = "/path/to/"
path_to1 = "/path/to/path1"
path_to2 = "/path/to/path2"
path_from1 = os.path.join(s3_store.store_path, "path1")
path_from2 = os.path.join(s3_store.store_path, "path2")
assert get_volume_args(
s3_store,
"/path/to",
artifacts=V1ArtifactsType(files=["path1", "path2"])) == "; ".join([
get_or_create_args(path=base_path),
cp_s3_args(path_from=path_from1,
path_to=path_to1,
is_file=True),
get_or_create_args(path=base_path),
cp_s3_args(path_from=path_from2,
path_to=path_to2,
is_file=True),
])
def test_get_volume_args_gcs(self):
gcs_store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
)
path_to = "/path/to/"
path_from = os.path.join(gcs_store.store_path, "")
assert get_volume_args(gcs_store, path_to, None) == " ".join([
get_or_create_args(path=path_to),
cp_gcs_args(path_from=path_from, path_to=path_to, is_file=False),
])
gcs_store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="Congs//:foo"),
)
path_to1 = "/path/to/path1"
path_to2 = "/path/to/path2"
path_from1 = os.path.join(gcs_store.store_path, "path1")
path_from2 = os.path.join(gcs_store.store_path, "path2")
assert get_volume_args(
gcs_store,
"/path/to",
artifacts=V1ArtifactsType(dirs=["path1", "path2"])) == " ".join([
get_or_create_args(path=path_to1),
cp_gcs_args(path_from=path_from1,
path_to=path_to1,
is_file=False),
get_or_create_args(path=path_to2),
cp_gcs_args(path_from=path_from2,
path_to=path_to2,
is_file=False),
])
def test_get_init_containers_with_store_outputs(self):
store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3://foo"),
)
# No context
containers = self.converter.get_init_containers(
contexts=None,
artifacts_store=store,
init_connections=[],
init_containers=[],
connection_by_names={},
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == []
# With context
containers = self.converter.get_init_containers(
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=True,
collect_logs=False,
auth=True)),
artifacts_store=store,
init_connections=[],
init_containers=[],
connection_by_names={},
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_auth_context_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
env=self.converter.get_auth_service_env_vars(),
),
get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
artifacts_store=store,
run_path=self.converter.run_path,
auto_resume=True,
),
]
def test_get_volume_from_connection(self):
# No store
assert get_volume_from_connection(connection=None) is None
# Bucket store
store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3//:foo"),
)
assert get_volume_from_connection(connection=store) is None
# Claim store
store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(
mount_path="/tmp", volume_claim="test", read_only=True
),
)
volume = get_volume_from_connection(connection=store)
assert volume.name == store.name
assert volume.persistent_volume_claim.claim_name == store.schema.volume_claim
assert volume.persistent_volume_claim.read_only == store.schema.read_only
# Host path
store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.HOST_PATH,
schema=V1HostPathConnection(
mount_path="/tmp", host_path="/tmp", read_only=True
),
)
volume = get_volume_from_connection(connection=store)
assert volume.name == store.name
assert volume.host_path == k8s_schemas.V1HostPathVolumeSource(
path=store.schema.host_path
)
def test_get_artifacts_path_container_with_bucket_store(self):
store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
)
container = get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"
),
artifacts_store=store,
run_path="run_uid",
auto_resume=True,
)
init_args = init_artifact_context_args("run_uid")
init_args.append(
get_volume_args(
store=store,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
artifacts=V1ArtifactsType(dirs=["run_uid"]),
)
)
assert container == get_base_store_container(
container=k8s_schemas.V1Container(name="default"),
container_name=generate_container_name(
INIT_ARTIFACTS_CONTAINER_PREFIX, "default"
),
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"
),
store=store,
env=[],
env_from=[],
volume_mounts=[get_artifacts_context_mount()],
args=[" ".join(init_args)],
)
def test_get_store_container_bucket_stores(self):
mount_path = "/test-path"
resource1 = V1K8sResourceType(
name="non_mount_test1",
schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]),
is_requested=False,
)
store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
secret=resource1.schema,
)
container = get_store_container(
polyaxon_init=V1PolyaxonInitContainer(
image="foo/foo", image_tag="", image_pull_policy="IfNotPresent"
),
connection=store,
artifacts=None,
mount_path=mount_path,
)
assert container.name == generate_container_name(
INIT_ARTIFACTS_CONTAINER_PREFIX, store.name
)
assert container.image == "foo/foo"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["/bin/sh", "-c"]
assert container.args == [
get_volume_args(store=store, mount_path=mount_path, artifacts=None)
]
assert container.env is not None
assert container.env_from == []
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=get_volume_name(mount_path), mount_path=mount_path
)
]
def test_resolve_connections_with_invalid_config(self):
fpath = tempfile.mkdtemp()
AgentManager.CONFIG_PATH = fpath
secret1 = V1K8sResourceType(
name="secret1",
schema=V1K8sResourceSchema(name="secret1"),
is_requested=True,
)
secret2 = V1K8sResourceType(
name="secret2",
schema=V1K8sResourceSchema(name="secret2"),
is_requested=True,
)
connection1 = V1ConnectionType(
name="test_s3",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3//:foo"),
secret=secret1.schema,
)
connection2 = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gcs//:foo"),
secret=secret1.schema,
)
connection3 = V1ConnectionType(
name="test_wasb",
kind=V1ConnectionKind.WASB,
schema=V1BucketConnection(bucket="wasbs//:foo"),
secret=secret2.schema,
)
settings.AGENT_CONFIG = AgentConfig(
namespace="foo",
artifacts_store=connection1,
connections=[connection2, connection3],
)
resolver = BaseResolver(
run=None,
compiled_operation=self.compiled_operation,
owner_name="user",
project_name="p1",
project_uuid=None,
run_name="j1",
run_uuid=None,
run_path="test",
params=None,
)
resolver.resolve_connections()
assert resolver.namespace == "foo"
assert resolver.connection_by_names == {connection1.name: connection1}
assert resolver.artifacts_store == connection1
assert [s.schema
for s in resolver.secrets] == [secret1.schema, secret2.schema]
assert resolver.polyaxon_sidecar == get_default_sidecar_container()
assert resolver.polyaxon_init == get_default_init_container()
# Add run spec to resolve connections
compiled_operation = V1CompiledOperation.read({
"version": 1.1,
"kind": kinds.COMPILED_OPERATION,
"plugins": {
"auth": False,
"shm": False,
"collectLogs": False,
"collectArtifacts": False,
"collectResources": False,
},
"run": {
"kind": V1RunKind.JOB,
"container": {
"image": "test"
},
"connections": {connection3.name},
},
})
resolver = BaseResolver(
run=None,
compiled_operation=compiled_operation,
owner_name="user",
project_name="p1",
project_uuid=None,
run_name="j1",
run_uuid=None,
run_path="test",
params=None,
)
resolver.resolve_connections()
assert resolver.namespace == "foo"
assert resolver.connection_by_names == {
connection1.name: connection1,
connection3.name: connection3,
}
assert [s.schema
for s in resolver.secrets] == [secret1.schema, secret2.schema]
assert resolver.artifacts_store == connection1
assert resolver.polyaxon_sidecar == get_default_sidecar_container()
assert resolver.polyaxon_init == get_default_init_container()
# Add run spec to resolve connections
compiled_operation = V1CompiledOperation.read({
"version": 1.1,
"kind": kinds.COMPILED_OPERATION,
"plugins": {
"auth": False,
"shm": False,
"collectLogs": False,
"collectArtifacts": False,
"collectResources": False,
},
"run": {
"kind": V1RunKind.JOB,
"container": {
"image": "test"
},
"connections": {
connection1.name,
connection2.name,
connection3.name,
},
},
})
resolver = BaseResolver(
run=None,
compiled_operation=compiled_operation,
owner_name="user",
project_name="p1",
project_uuid=None,
run_name="j1",
run_uuid=None,
run_path="test",
params=None,
)
resolver.resolve_connections()
assert resolver.namespace == "foo"
assert resolver.connection_by_names == {
connection3.name: connection3,
connection2.name: connection2,
connection1.name: connection1,
}
assert [s.schema
for s in resolver.secrets] == [secret1.schema, secret2.schema]
assert resolver.artifacts_store == connection1
assert resolver.polyaxon_sidecar == get_default_sidecar_container()
assert resolver.polyaxon_init == get_default_init_container()
def setUp(self):
super().setUp()
# Secrets and config maps
self.non_mount_resource1 = V1K8sResourceType(
name="non_mount_test1",
schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]),
is_requested=False,
)
self.request_non_mount_resource1 = V1K8sResourceType(
name="request_non_mount_resource1",
schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]),
is_requested=True,
)
self.non_mount_resource2 = V1K8sResourceType(
name="non_mount_test2",
schema=V1K8sResourceSchema(name="ref"),
is_requested=False,
)
self.mount_resource1 = V1K8sResourceType(
name="mount_test1",
schema=V1K8sResourceSchema(name="ref",
items=["item1", "item2"],
mount_path="/tmp1"),
is_requested=False,
)
self.request_mount_resource2 = V1K8sResourceType(
name="mount_test1",
schema=V1K8sResourceSchema(name="ref", mount_path="/tmp2"),
is_requested=True,
)
# Connections
self.gcs_store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
secret=self.mount_resource1.schema,
)
self.s3_store = V1ConnectionType(
name="test_s3",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3//:foo"),
secret=self.non_mount_resource1.schema,
)
self.az_store = V1ConnectionType(
name="test_az",
kind=V1ConnectionKind.WASB,
schema=V1BucketConnection(
bucket="wasb://[email protected]"),
secret=self.non_mount_resource1.schema,
)
self.claim_store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/tmp",
volume_claim="test",
read_only=True),
)
self.host_path_store = V1ConnectionType(
name="test_path",
kind=V1ConnectionKind.HOST_PATH,
schema=V1HostPathConnection(mount_path="/tmp", host_path="/tmp"),
)
def test_get_sidecar_container_with_managed_bucket_outputs_logs_store_and_env_from(
self, ):
env_vars = [
get_env_var(name="key1", value="value1"),
get_env_var(name="key2", value="value2"),
]
resource1 = V1K8sResourceType(name="test1",
schema=V1K8sResourceSchema(name="ref"),
is_requested=False)
bucket_managed_store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
secret=resource1.schema,
)
# both logs and outputs
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
env=env_vars,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar",
image_tag="",
image_pull_policy="IfNotPresent",
sleep_interval=213,
sync_interval=212,
),
artifacts_store=bucket_managed_store,
contexts=PluginsContextsSpec.from_config(None, default_auth=True),
run_path="test",
)
assert sidecar.name == SIDECAR_CONTAINER
assert sidecar.image == "sidecar/sidecar"
assert sidecar.image_pull_policy == "IfNotPresent"
assert sidecar.command == ["polyaxon", "sidecar"]
assert sidecar.args == get_sidecar_args(
container_id=MAIN_JOB_CONTAINER,
sleep_interval=213,
sync_interval=212,
monitor_logs=False,
)
assert sidecar.env == get_sidecar_env_vars(
env_vars=env_vars,
container_id=MAIN_JOB_CONTAINER,
artifacts_store_name=bucket_managed_store.name,
) + get_items_from_secret(secret=resource1) + get_connection_env_var(
connection=bucket_managed_store, secret=resource1)
assert sidecar.env_from == [get_env_from_secret(secret=resource1)]
assert sidecar.resources == get_sidecar_resources()
assert sidecar.volume_mounts == [
get_auth_context_mount(read_only=True),
get_artifacts_context_mount(read_only=False),
]
# logs and no outputs
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
env=env_vars,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar",
image_tag="",
image_pull_policy="IfNotPresent",
sleep_interval=213,
sync_interval=212,
),
artifacts_store=bucket_managed_store,
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=False, auth=True)),
run_path="test",
)
assert sidecar.name == SIDECAR_CONTAINER
assert sidecar.image == "sidecar/sidecar"
assert sidecar.image_pull_policy == "IfNotPresent"
assert sidecar.command == ["polyaxon", "sidecar"]
assert sidecar.args == get_sidecar_args(
container_id=MAIN_JOB_CONTAINER,
sleep_interval=213,
sync_interval=212,
monitor_logs=False,
)
assert sidecar.env == get_sidecar_env_vars(
env_vars=env_vars,
container_id=MAIN_JOB_CONTAINER,
artifacts_store_name=bucket_managed_store.name,
) + get_items_from_secret(secret=resource1) + get_connection_env_var(
connection=bucket_managed_store, secret=resource1)
assert sidecar.env_from == [get_env_from_secret(secret=resource1)]
assert sidecar.resources == get_sidecar_resources()
assert sidecar.volume_mounts == [
get_auth_context_mount(read_only=True),
]
# outputs and no logs
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
env=env_vars,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar",
image_tag="",
image_pull_policy="IfNotPresent",
sleep_interval=213,
sync_interval=212,
),
artifacts_store=bucket_managed_store,
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_logs=False, auth=True)),
run_path="test",
)
assert sidecar.name == SIDECAR_CONTAINER
assert sidecar.image == "sidecar/sidecar"
assert sidecar.image_pull_policy == "IfNotPresent"
assert sidecar.command == ["polyaxon", "sidecar"]
assert sidecar.args == get_sidecar_args(
container_id=MAIN_JOB_CONTAINER,
sleep_interval=213,
sync_interval=212,
monitor_logs=False,
)
assert sidecar.env == get_sidecar_env_vars(
env_vars=env_vars,
container_id=MAIN_JOB_CONTAINER,
artifacts_store_name=bucket_managed_store.name,
) + get_items_from_secret(secret=resource1) + get_connection_env_var(
connection=bucket_managed_store, secret=resource1)
assert sidecar.env_from == [get_env_from_secret(secret=resource1)]
assert sidecar.resources == get_sidecar_resources()
assert sidecar.volume_mounts == [
get_auth_context_mount(read_only=True),
get_artifacts_context_mount(read_only=False),
]
def test_get_sidecar_container_auth_context(self, ):
env_vars = [
get_env_var(name="key1", value="value1"),
get_env_var(name="key2", value="value2"),
]
resource1 = V1K8sResourceType(
name="test1",
schema=V1K8sResourceSchema(name="ref1", items=["item1", "item2"]),
is_requested=False,
)
bucket_managed_store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
secret=resource1.schema,
)
# Default auth is included
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
env=env_vars,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar",
image_tag="",
image_pull_policy="IfNotPresent",
sleep_interval=213,
sync_interval=212,
),
artifacts_store=bucket_managed_store,
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
run_path="test",
)
assert sidecar.name == SIDECAR_CONTAINER
assert sidecar.image == "sidecar/sidecar"
assert sidecar.image_pull_policy == "IfNotPresent"
assert sidecar.command == ["polyaxon", "sidecar"]
assert sidecar.volume_mounts == [
get_auth_context_mount(read_only=True),
get_artifacts_context_mount(read_only=False),
]
# Nno auth
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
env=env_vars,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar",
image_tag="",
image_pull_policy="IfNotPresent",
sleep_interval=213,
sync_interval=-212,
),
artifacts_store=bucket_managed_store,
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=False)),
run_path="test",
)
assert sidecar.name == SIDECAR_CONTAINER
assert sidecar.image == "sidecar/sidecar"
assert sidecar.image_pull_policy == "IfNotPresent"
assert sidecar.command == ["polyaxon", "sidecar"]
assert sidecar.volume_mounts == [
get_artifacts_context_mount(read_only=False)
]
def test_get_requested_secrets(self):
expected = get_requested_secrets(secrets=[self.resource1],
connections=[self.s3_store])
assert [e.schema for e in expected] == [self.resource1.schema]
expected = get_requested_secrets(
secrets=[self.resource1, self.resource3],
connections=[self.s3_store])
assert [e.schema for e in expected] == [
self.resource3.schema,
self.resource1.schema,
]
expected = get_requested_secrets(
secrets=[self.resource2, self.resource3, self.resource4],
connections=[self.gcs_store],
)
assert [e.schema for e in expected] == [
self.resource3.schema,
self.resource4.schema,
self.resource2.schema,
]
expected = get_requested_secrets(
secrets=[self.resource1, self.resource2],
connections=[self.gcs_store])
assert [e.schema for e in expected] == [self.resource2.schema]
expected = get_requested_secrets(
secrets=[self.resource1, self.resource2],
connections=[self.s3_store, self.gcs_store],
)
assert [e.schema for e in expected] == [
self.resource1.schema,
self.resource2.schema,
]
expected = get_requested_secrets(
secrets=[self.resource1, self.resource2],
connections=[
self.s3_store,
self.gcs_store,
self.host_path_store,
self.claim_store,
],
)
assert [e.schema for e in expected] == [
self.resource1.schema,
self.resource2.schema,
]
new_az_store = V1ConnectionType(
name="test_az",
kind=V1ConnectionKind.WASB,
schema=V1BucketConnection(
bucket="Conwasb://[email protected]"),
secret=self.resource1,
)
expected = get_requested_secrets(
secrets=[self.resource1, self.resource2],
connections=[
self.s3_store,
self.gcs_store,
new_az_store,
self.host_path_store,
self.claim_store,
],
)
assert [e.schema for e in expected] == [
self.resource1.schema,
self.resource2.schema,
]
# Using a requested secret with same id
expected = get_requested_secrets(
secrets=[self.resource5, self.resource2],
connections=[
self.s3_store,
self.gcs_store,
new_az_store,
self.host_path_store,
self.claim_store,
],
)
assert [e.schema for e in expected] == [
self.resource5.schema,
self.resource2.schema,
]
def test_get_base_store_container_with_store_with_secret(self):
non_mount_resource1 = V1K8sResourceType(
name="resource",
schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]),
is_requested=False,
)
bucket_store_with_secret = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
secret=non_mount_resource1.schema,
)
container = get_base_store_container(
container=k8s_schemas.V1Container(name="init"),
container_name="init",
polyaxon_init=V1PolyaxonInitContainer(image_tag=""),
store=bucket_store_with_secret,
env=None,
env_from=None,
volume_mounts=None,
args=None,
)
assert container.name == "init"
assert container.image == "polyaxon/polyaxon-init"
assert container.image_pull_policy is None
assert container.command == ["/bin/sh", "-c"]
assert container.args is None
env = get_items_from_secret(
secret=non_mount_resource1) + get_connection_env_var(
connection=bucket_store_with_secret,
secret=non_mount_resource1)
assert container.env == env
assert container.env_from == []
assert container.resources is not None
assert container.volume_mounts == []
mount_resource1 = V1K8sResourceType(
name="resource",
schema=V1K8sResourceSchema(name="resource",
items=["item1", "item2"],
mount_path="/tmp1"),
is_requested=False,
)
bucket_store_with_secret.secret = mount_resource1.schema
container = get_base_store_container(
container=k8s_schemas.V1Container(name="init"),
container_name="init",
polyaxon_init=V1PolyaxonInitContainer(image_tag=""),
store=bucket_store_with_secret,
env=None,
env_from=None,
volume_mounts=None,
args=None,
)
assert container.name == "init"
assert container.image == "polyaxon/polyaxon-init"
assert container.image_pull_policy is None
assert container.command == ["/bin/sh", "-c"]
assert container.args is None
assert container.env == get_connection_env_var(
connection=bucket_store_with_secret, secret=mount_resource1)
assert container.env_from == []
assert container.resources is not None
assert container.volume_mounts == [
get_mount_from_resource(resource=mount_resource1)
]
def test_get_sidecars(self):
assert (self.converter.get_sidecar_containers(
contexts=None,
artifacts_store=None,
sidecar_containers=[],
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar", sleep_interval=12, sync_interval=-1),
) == [])
# Store with single path, no secret is passed and not required
store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3://foo"),
)
contexts = PluginsContextsSpec.from_config(
V1Plugins(collect_logs=True, collect_artifacts=True, auth=True))
assert self.converter.get_sidecar_containers(
contexts=contexts,
artifacts_store=store,
sidecar_containers=[],
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar", sleep_interval=12, sync_interval=12),
) == [
get_sidecar_container(
contexts=contexts,
env=self.converter.get_polyaxon_sidecar_service_env_vars(),
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar",
sleep_interval=12,
sync_interval=12),
artifacts_store=store,
run_path=self.converter.run_path,
)
]
secret1 = V1K8sResourceType(
name="test1",
schema=V1K8sResourceSchema(name="ref1", items=["item1", "item2"]),
is_requested=True,
)
store.secret = secret1.schema
polyaxon_sidecar = V1PolyaxonSidecarContainer(
image="sidecar/sidecar",
image_pull_policy=None,
sleep_interval=12,
sync_interval=-1,
)
assert self.converter.get_sidecar_containers(
contexts=contexts,
artifacts_store=store,
polyaxon_sidecar=polyaxon_sidecar,
sidecar_containers=[],
) == [
get_sidecar_container(
contexts=contexts,
env=self.converter.get_polyaxon_sidecar_service_env_vars(),
polyaxon_sidecar=polyaxon_sidecar,
artifacts_store=store,
run_path=self.converter.run_path,
)
]
def test_get_init_containers_with_claim_outputs(self):
store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/claim/path",
volume_claim="claim",
read_only=True),
)
# No context to enable the outputs
containers = self.converter.get_init_containers(
contexts=None,
artifacts_store=store.name,
init_connections=None,
connection_by_names={},
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == []
# Enable outputs
containers = self.converter.get_init_containers(
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=True, collect_logs=False)),
artifacts_store=store,
connection_by_names={},
init_connections=None,
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
artifacts_store=store,
run_path=self.converter.run_path,
auto_resume=True,
),
]
# Use store for init
containers = self.converter.get_init_containers(
contexts=None,
artifacts_store=None,
connection_by_names={store.name: store},
init_connections=[V1Init(connection=store.name)],
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_store_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
connection=store,
artifacts=None,
env=self.converter.get_init_service_env_vars(),
)
]
# Use store for init and outputs
containers = self.converter.get_init_containers(
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=True, collect_logs=False)),
artifacts_store=store,
init_connections=[V1Init(connection=store.name)],
connection_by_names={store.name: store},
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
artifacts_store=store,
run_path=self.converter.run_path,
auto_resume=True,
),
get_store_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
connection=store,
artifacts=None,
env=self.converter.get_init_service_env_vars(),
),
]
# Add Store
store1 = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3://foo"),
secret=None,
)
containers = self.converter.get_init_containers(
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=True,
collect_logs=False,
auth=True)),
artifacts_store=store,
init_connections=[
V1Init(
connection=store.name,
artifacts=V1ArtifactsType(files=["/foo", "/bar"]),
),
V1Init(
connection=store1.name,
artifacts=V1ArtifactsType(files=["/foo", "/bar"]),
),
],
connection_by_names={
store.name: store,
store1.name: store1
},
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_auth_context_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
env=self.converter.get_auth_service_env_vars(),
),
get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
artifacts_store=store,
run_path=self.converter.run_path,
auto_resume=True,
),
get_store_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
connection=store,
artifacts=V1ArtifactsType(files=["/foo", "/bar"]),
env=self.converter.get_init_service_env_vars(),
),
get_store_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
connection=store1,
artifacts=V1ArtifactsType(files=["/foo", "/bar"]),
env=self.converter.get_init_service_env_vars(),
),
]
def test_agent_config(self):
config_dict = {POLYAXON_KEYS_AGENT_ARTIFACTS_STORE: 12}
with self.assertRaises(ValidationError):
AgentConfig.from_dict(config_dict)
config_dict = {POLYAXON_KEYS_AGENT_ARTIFACTS_STORE: "some"}
with self.assertRaises(ValidationError):
AgentConfig.from_dict(config_dict)
config_dict = {
POLYAXON_KEYS_K8S_NAMESPACE: "foo",
POLYAXON_KEYS_AGENT_ARTIFACTS_STORE: {
"name": "some",
"kind": V1ConnectionKind.GCS,
"schema": V1BucketConnection(bucket="gs://test").to_dict(),
},
POLYAXON_KEYS_AGENT_CONNECTIONS: [],
POLYAXON_KEYS_AGENT_NOTIFICATION_CONNECTIONS: [],
}
config = AgentConfig.from_dict(config_dict)
assert config.to_light_dict() == config_dict
config_dict = {
POLYAXON_KEYS_K8S_NAMESPACE:
"foo",
POLYAXON_KEYS_AGENT_ARTIFACTS_STORE:
"some",
POLYAXON_KEYS_AGENT_NOTIFICATION_CONNECTIONS: [],
POLYAXON_KEYS_AGENT_CONNECTIONS: [{
"name":
"some",
"kind":
V1ConnectionKind.GCS,
"schema":
V1BucketConnection(bucket="gs://test").to_dict(),
"secretResource":
"some",
}],
}
with self.assertRaises(ValidationError):
AgentConfig.from_dict(config_dict)
config_dict = {
POLYAXON_KEYS_K8S_NAMESPACE:
"foo",
POLYAXON_KEYS_AGENT_ARTIFACTS_STORE: {
"name": "test",
"kind": V1ConnectionKind.GCS,
"schema": V1BucketConnection(bucket="gs://test").to_dict(),
"secret": V1K8sResourceSchema(name="some").to_dict(),
},
POLYAXON_KEYS_AGENT_NOTIFICATION_CONNECTIONS: [{
"name":
"slack",
"kind":
V1ConnectionKind.SLACK,
"secret":
V1K8sResourceSchema(name="some").to_dict(),
}],
POLYAXON_KEYS_AGENT_CONNECTIONS: [{
"name":
"some",
"kind":
V1ConnectionKind.GCS,
"schema":
V1BucketConnection(bucket="gs://test").to_dict(),
"secret":
V1K8sResourceSchema(name="some").to_dict(),
}],
}
config = AgentConfig.from_dict(config_dict)
assert config.to_light_dict() == config_dict