def test_host_path_connect_config(self):
config_dict = {}
with self.assertRaises(ValidationError):
V1HostPathConnection.from_dict(config_dict)
with self.assertRaises(ValidationError):
validate_connection(V1ConnectionKind.HOST_PATH, config_dict)
config_dict = {"host_path": "foo"}
with self.assertRaises(ValidationError):
V1HostPathConnection.from_dict(config_dict)
with self.assertRaises(ValidationError):
validate_connection(V1ConnectionKind.HOST_PATH, config_dict)
config_dict = {"hostPath": "foo", "mountPath": "foo", "readOnly": True}
config = V1HostPathConnection.from_dict(config_dict)
assert config.to_dict() == config_dict
with self.assertRaises(ValidationError):
validate_connection(V1ConnectionKind.S3, config_dict)
with self.assertRaises(ValidationError):
validate_connection(V1ConnectionKind.WASB, config_dict)
validate_connection(V1ConnectionKind.HOST_PATH, config_dict)
def test_get_mount_from_store(self):
# Non bucket stores
assert get_mount_from_store(store=None) is None
store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3//:foo"),
)
assert get_mount_from_store(store=store) is None
# Claim store
store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/tmp",
volume_claim="test",
read_only=True),
)
mount = get_mount_from_store(store=store)
assert mount.name == store.name
assert mount.mount_path == store.schema.mount_path
assert mount.read_only == store.schema.read_only
# Host path
store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.HOST_PATH,
schema=V1HostPathConnection(mount_path="/tmp",
host_path="/tmp",
read_only=True),
)
mount = get_mount_from_store(store=store)
assert mount.name == store.name
assert mount.mount_path == store.schema.mount_path
assert mount.read_only == store.schema.read_only
def setUp(self):
super().setUp()
self.s3_store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3//:foo"),
)
self.gcs_store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
)
self.az_store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.WASB,
schema=V1BucketConnection(
bucket="Conwasb://[email protected]"),
)
self.claim_store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(volume_claim="test",
mount_path="/tmp",
read_only=True),
)
self.host_path_store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.HOST_PATH,
schema=V1HostPathConnection(host_path="/tmp",
mount_path="/tmp",
read_only=True),
)
def test_get_volume_args_host(self):
host_path_store = V1ConnectionType(
name="test_path",
kind=V1ConnectionKind.HOST_PATH,
schema=V1HostPathConnection(
mount_path="/tmp", host_path="/tmp", read_only=True
),
)
path_to = "/path/to/"
path_from = os.path.join(host_path_store.store_path, "")
assert get_volume_args(host_path_store, path_to, None) == "; ".join(
[
get_or_create_args(path=path_to),
cp_copy_args(path_from=path_from, path_to=path_to, is_file=False),
]
)
host_path_store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.HOST_PATH,
schema=V1HostPathConnection(
mount_path="/tmp", host_path="/tmp", read_only=True
),
)
base_path = "/path/to/"
path_to1 = "/path/to/path1"
path_to2 = "/path/to/path2"
path_from1 = os.path.join(host_path_store.store_path, "path1")
path_from2 = os.path.join(host_path_store.store_path, "path2")
assert get_volume_args(
host_path_store,
"/path/to",
artifacts=V1ArtifactsType(dirs=["path1", "path2"]),
) == "; ".join(
[
get_or_create_args(path=base_path),
cp_copy_args(path_from=path_from1, path_to=path_to1, is_file=False),
get_or_create_args(path=base_path),
cp_copy_args(path_from=path_from2, path_to=path_to2, is_file=False),
]
)
def test_get_main_container_host_paths(self):
contexts = PluginsContextsSpec(
auth=True,
docker=False,
shm=False,
collect_logs=True,
collect_artifacts=True,
collect_resources=True,
auto_resume=True,
sync_statuses=True,
external_host=False,
sidecar=None,
)
volume_mounts = get_mounts(
use_auth_context=contexts.auth,
use_artifacts_context=False,
use_docker_context=contexts.docker,
use_shm_context=contexts.shm,
)
artifacts_store = V1ConnectionType(
name="plx-outputs",
kind=V1ConnectionKind.HOST_PATH,
schema=V1HostPathConnection(mount_path="/tmp/plx/outputs",
host_path="/tmp/plx/outputs"),
)
container = get_main_container(
container_id="test",
main_container=k8s_schemas.V1Container(name="main"),
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=True, collect_logs=True)),
volume_mounts=volume_mounts,
log_level=None,
artifacts_store=artifacts_store,
init=[],
connections=[],
connection_by_names={artifacts_store.name: artifacts_store},
secrets=[],
config_maps=[],
kv_env_vars=None,
env=None,
ports=None,
run_path="run_path",
)
assert container.volume_mounts == [
get_auth_context_mount(read_only=True),
get_artifacts_context_mount(read_only=False),
]
def set_store():
store_root = tempfile.mkdtemp()
settings.AGENT_CONFIG = AgentConfig(
artifacts_store=V1ConnectionType(
name="test",
kind=V1ConnectionKind.HOST_PATH,
schema=V1HostPathConnection(host_path=store_root,
mount_path=store_root),
secret=None,
),
connections=[],
)
settings.CLIENT_CONFIG.archive_root = tempfile.mkdtemp()
return store_root
def setUp(self):
super().setUp()
self.s3_store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.S3,
tags=["test", "foo"],
schema=V1BucketConnection(bucket="s3//:foo"),
)
self.gcs_store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.GCS,
tags=["test"],
schema=V1BucketConnection(bucket="gs//:foo"),
)
self.az_store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.WASB,
schema=V1BucketConnection(
bucket="Conwasb://[email protected]"),
)
self.claim_store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(volume_claim="test",
mount_path="/tmp",
read_only=True),
)
self.host_path_store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.HOST_PATH,
schema=V1HostPathConnection(host_path="/tmp",
mount_path="/tmp",
read_only=True),
)
self.custom_connection1 = V1ConnectionType(
name="db",
kind=V1ConnectionKind.POSTGRES,
)
self.custom_connection2 = V1ConnectionType(
name="ssh",
kind=V1ConnectionKind.SSH,
schema=V1CustomConnection(key1="val1", key2="val2"),
)
def test_get_sidecar_container_host_paths(self):
artifacts_store = V1ConnectionType(
name="plx-outputs",
kind=V1ConnectionKind.HOST_PATH,
schema=V1HostPathConnection(mount_path="/tmp/plx/outputs",
host_path="/tmp/plx/outputs"),
)
contexts = PluginsContextsSpec(
auth=True,
docker=False,
shm=False,
collect_logs=True,
collect_artifacts=True,
collect_resources=True,
auto_resume=True,
sync_statuses=True,
external_host=False,
sidecar=None,
)
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="foo",
image_pull_policy="sdf",
sleep_interval=2,
sync_interval=212,
),
env=[],
artifacts_store=artifacts_store,
contexts=contexts,
run_path="test",
)
assert sidecar.volume_mounts == [
get_auth_context_mount(read_only=True),
get_artifacts_context_mount(read_only=False),
get_mount_from_store(store=artifacts_store),
]
def test_get_volume_from_connection(self):
# No store
assert get_volume_from_connection(connection=None) is None
# Bucket store
store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3//:foo"),
)
assert get_volume_from_connection(connection=store) is None
# Claim store
store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(
mount_path="/tmp", volume_claim="test", read_only=True
),
)
volume = get_volume_from_connection(connection=store)
assert volume.name == store.name
assert volume.persistent_volume_claim.claim_name == store.schema.volume_claim
assert volume.persistent_volume_claim.read_only == store.schema.read_only
# Host path
store = V1ConnectionType(
name="test",
kind=V1ConnectionKind.HOST_PATH,
schema=V1HostPathConnection(
mount_path="/tmp", host_path="/tmp", read_only=True
),
)
volume = get_volume_from_connection(connection=store)
assert volume.name == store.name
assert volume.host_path == k8s_schemas.V1HostPathVolumeSource(
path=store.schema.host_path
)
def setUp(self):
super().setUp()
# Secrets and config maps
self.non_mount_resource1 = V1K8sResourceType(
name="non_mount_test1",
schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]),
is_requested=False,
)
self.non_mount_resource2 = V1K8sResourceType(
name="non_mount_test2",
schema=V1K8sResourceSchema(name="ref"),
is_requested=False,
)
self.mount_resource1 = V1K8sResourceType(
name="mount_test1",
schema=V1K8sResourceSchema(name="ref",
items=["item1", "item2"],
mount_path="/tmp1"),
is_requested=False,
)
self.mount_resource2 = V1K8sResourceType(
name="mount_test2",
schema=V1K8sResourceSchema(name="ref", mount_path="/tmp2"),
is_requested=False,
)
# Volumes
self.vol1 = get_volume(volume="vol1", claim_name="claim1")
self.vol2 = get_volume(volume="vol2", host_path="/path2")
self.vol3 = get_volume(volume="vol3")
# connections
self.s3_store = V1ConnectionType(
name="test_s3",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3//:foo"),
secret=self.mount_resource1.schema,
)
self.gcs_store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
secret=self.mount_resource1.schema,
)
self.az_store = V1ConnectionType(
name="test_az",
kind=V1ConnectionKind.WASB,
schema=V1BucketConnection(
bucket="wasb://[email protected]"),
secret=self.mount_resource1.schema,
)
self.claim_store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/tmp",
volume_claim="test",
read_only=True),
)
self.host_path_store = V1ConnectionType(
name="test_path",
kind=V1ConnectionKind.HOST_PATH,
schema=V1HostPathConnection(mount_path="/tmp",
host_path="/tmp",
read_only=True),
)
def test_get_sidecar_container_with_managed_mount_outputs_logs_store(self):
env_vars = [
get_env_var(name="key1", value="value1"),
get_env_var(name="key2", value="value2"),
]
mount_managed_store = V1ConnectionType(
name="test_path",
kind=V1ConnectionKind.HOST_PATH,
schema=V1HostPathConnection(mount_path="/tmp", host_path="/tmp"),
secret=None,
)
# logs and outputs
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
env=env_vars,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar",
image_tag="",
image_pull_policy="IfNotPresent",
sleep_interval=213,
sync_interval=212,
),
artifacts_store=mount_managed_store,
contexts=PluginsContextsSpec.from_config(None, default_auth=True),
run_path="test",
)
assert sidecar.name == SIDECAR_CONTAINER
assert sidecar.image == "sidecar/sidecar"
assert sidecar.image_pull_policy == "IfNotPresent"
assert sidecar.command == ["polyaxon", "sidecar"]
assert sidecar.args == get_sidecar_args(
container_id=MAIN_JOB_CONTAINER,
sleep_interval=213,
sync_interval=212,
monitor_logs=False,
)
assert sidecar.env == get_sidecar_env_vars(
env_vars=env_vars,
container_id=MAIN_JOB_CONTAINER,
artifacts_store_name=mount_managed_store.name,
) + get_connection_env_var(connection=mount_managed_store, secret=None)
assert sidecar.env_from == []
assert sidecar.resources == get_sidecar_resources()
assert sidecar.volume_mounts == [
get_auth_context_mount(read_only=True),
get_artifacts_context_mount(read_only=False),
get_mount_from_store(store=mount_managed_store),
]
# logs and no outputs
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
env=env_vars,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar",
image_tag="",
image_pull_policy="IfNotPresent",
sleep_interval=213,
sync_interval=212,
),
artifacts_store=mount_managed_store,
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=False, auth=True)),
run_path="test",
)
assert sidecar.name == SIDECAR_CONTAINER
assert sidecar.image == "sidecar/sidecar"
assert sidecar.image_pull_policy == "IfNotPresent"
assert sidecar.command == ["polyaxon", "sidecar"]
assert sidecar.args == get_sidecar_args(
container_id=MAIN_JOB_CONTAINER,
sleep_interval=213,
sync_interval=212,
monitor_logs=False,
)
assert sidecar.env == get_sidecar_env_vars(
env_vars=env_vars,
container_id=MAIN_JOB_CONTAINER,
artifacts_store_name=mount_managed_store.name,
) + get_connection_env_var(connection=mount_managed_store, secret=None)
assert sidecar.env_from == []
assert sidecar.resources == get_sidecar_resources()
assert sidecar.volume_mounts == [
get_auth_context_mount(read_only=True),
get_mount_from_store(store=mount_managed_store),
]
# outputs and no logs
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
env=env_vars,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar",
image_tag="",
image_pull_policy="IfNotPresent",
sleep_interval=213,
sync_interval=212,
),
artifacts_store=mount_managed_store,
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_logs=False, auth=True)),
run_path="test",
)
assert sidecar.name == SIDECAR_CONTAINER
assert sidecar.image == "sidecar/sidecar"
assert sidecar.image_pull_policy == "IfNotPresent"
assert sidecar.command == ["polyaxon", "sidecar"]
assert sidecar.args == get_sidecar_args(
container_id=MAIN_JOB_CONTAINER,
sleep_interval=213,
sync_interval=212,
monitor_logs=False,
)
assert sidecar.env == get_sidecar_env_vars(
env_vars=env_vars,
container_id=MAIN_JOB_CONTAINER,
artifacts_store_name=mount_managed_store.name,
) + get_connection_env_var(connection=mount_managed_store, secret=None)
assert sidecar.env_from == []
assert sidecar.resources == get_sidecar_resources()
assert sidecar.volume_mounts == [
get_auth_context_mount(read_only=True),
get_artifacts_context_mount(read_only=False),
get_mount_from_store(store=mount_managed_store),
]
def setUp(self):
super().setUp()
# Secrets
self.resource1 = V1K8sResourceType(
name="non_mount_test1",
schema=V1K8sResourceSchema(name="non_mount_test1",
items=["item1", "item2"]),
is_requested=False,
)
self.resource2 = V1K8sResourceType(
name="non_mount_test2",
schema=V1K8sResourceSchema(name="non_mount_test2"),
is_requested=False,
)
self.resource3 = V1K8sResourceType(
name="non_mount_test3",
schema=V1K8sResourceSchema(name="non_mount_test3",
items=["item1", "item2"]),
is_requested=True,
)
self.resource4 = V1K8sResourceType(
name="non_mount_test4",
schema=V1K8sResourceSchema(name="non_mount_test4"),
is_requested=True,
)
self.resource5 = V1K8sResourceType(
name="non_mount_test1",
schema=V1K8sResourceSchema(name="non_mount_test1"),
is_requested=True,
)
# Connections
self.s3_store = V1ConnectionType(
name="test_s3",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3//:foo"),
secret=self.resource1.schema,
)
self.gcs_store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gcs//:foo"),
secret=self.resource2.schema,
)
self.az_store = V1ConnectionType(
name="test_az",
kind=V1ConnectionKind.WASB,
schema=V1BucketConnection(
bucket="wasb://[email protected]"),
secret=self.resource3.schema,
)
self.claim_store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/tmp", volume_claim="test"),
)
self.host_path_store = V1ConnectionType(
name="test_path",
kind=V1ConnectionKind.HOST_PATH,
schema=V1HostPathConnection(mount_path="/tmp",
host_path="/tmp",
read_only=True),
)
def test_get_sidecar_container_override_sync(self):
artifacts_store = V1ConnectionType(
name="plx-outputs",
kind=V1ConnectionKind.HOST_PATH,
schema=V1HostPathConnection(mount_path="/tmp/plx/outputs",
host_path="/tmp/plx/outputs"),
)
contexts = PluginsContextsSpec(
auth=True,
docker=False,
shm=False,
collect_logs=True,
collect_artifacts=True,
collect_resources=True,
auto_resume=True,
sync_statuses=True,
external_host=False,
sidecar=None,
)
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="foo",
image_pull_policy="sdf",
sleep_interval=2,
sync_interval=212,
),
env=[],
artifacts_store=artifacts_store,
contexts=contexts,
run_path="test",
)
assert sidecar.args == [
"--container-id=polyaxon-main",
"--sleep-interval=2",
"--sync-interval=212",
]
contexts = PluginsContextsSpec(
auth=True,
docker=False,
shm=False,
collect_logs=True,
collect_artifacts=True,
collect_resources=True,
auto_resume=True,
sync_statuses=True,
external_host=False,
sidecar=V1PolyaxonSidecarContainer(
sleep_interval=-1,
sync_interval=-1,
),
)
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="foo",
image_pull_policy="sdf",
sleep_interval=2,
sync_interval=212,
),
env=[],
artifacts_store=artifacts_store,
contexts=contexts,
run_path="test",
)
assert sidecar.args == [
"--container-id=polyaxon-main",
"--sleep-interval=-1",
"--sync-interval=-1",
]
