def test_get_volume_args_s3(self):
s3_store = V1ConnectionType(
name="test_s3",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3//:foo"),
)
path_to = "/path-to/"
path_from = os.path.join(s3_store.store_path, "")
assert get_volume_args(s3_store, path_to, None) == "; ".join(
[
get_or_create_args(path=path_to),
cp_s3_args(path_from=path_from, path_to=path_to, is_file=False),
]
)
s3_store = V1ConnectionType(
name="test_s3",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3//:foo"),
)
base_path = "/path/to/"
path_to1 = "/path/to/path1"
path_to2 = "/path/to/path2"
path_from1 = os.path.join(s3_store.store_path, "path1")
path_from2 = os.path.join(s3_store.store_path, "path2")
assert get_volume_args(
s3_store, "/path/to", artifacts=V1ArtifactsType(files=["path1", "path2"])
) == "; ".join(
[
get_or_create_args(path=base_path),
cp_s3_args(path_from=path_from1, path_to=path_to1, is_file=True),
get_or_create_args(path=base_path),
cp_s3_args(path_from=path_from2, path_to=path_to2, is_file=True),
]
)
def get_artifacts_path_container(
polyaxon_init: V1PolyaxonInitContainer,
artifacts_store: V1ConnectionType,
run_path: str,
auto_resume: bool,
) -> Optional[k8s_schemas.V1Container]:
if not artifacts_store:
raise PolypodException("Init artifacts container requires a store.")
init_args = init_artifact_context_args(run_path=run_path)
if auto_resume:
init_args.append(
get_volume_args(
store=artifacts_store,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
artifacts=V1ArtifactsType(dirs=[run_path]),
))
container_name = generate_container_name(INIT_ARTIFACTS_CONTAINER_PREFIX,
DEFAULT)
container = k8s_schemas.V1Container(name=container_name)
return get_base_store_container(
container_name=container_name,
container=container,
polyaxon_init=polyaxon_init,
store=artifacts_store,
env=[],
env_from=[],
volume_mounts=[get_artifacts_context_mount()],
# If we are dealing with a volume we need to make sure the path exists for the user
# We also clean the path if this is not a resume run
args=[" ".join(init_args)],
)
def test_get_artifacts_path_container_with_managed_mount_store(self):
store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/claim/path",
volume_claim="claim"),
)
container = get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"),
artifacts_store=store,
run_path="run_uid",
auto_resume=True,
)
init_args = init_artifact_context_args("run_uid")
init_args.append(
get_volume_args(
store=store,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
artifacts=V1ArtifactsType(dirs=["run_uid"]),
))
assert container == get_base_store_container(
container=k8s_schemas.V1Container(name="default"),
container_name=INIT_ARTIFACTS_CONTAINER.format("default"),
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"),
store=store,
env=[],
env_from=[],
volume_mounts=[get_artifacts_context_mount()],
args=[" ".join(init_args)],
)
def test_get_artifacts_path_container_with_bucket_store(self):
store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
)
container = get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"),
artifacts_store=store,
run_path="run_uid",
auto_resume=True,
)
init_args = init_artifact_context_args("run_uid")
init_args.append(
get_volume_args(
store=store,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
artifacts=V1ArtifactsType(dirs=["run_uid"]),
))
assert container == get_base_store_container(
container=k8s_schemas.V1Container(name="default"),
container_name=generate_container_name(
INIT_ARTIFACTS_CONTAINER_PREFIX, "default", False),
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"),
store=store,
env=[],
env_from=[],
volume_mounts=[get_artifacts_context_mount()],
args=[" ".join(init_args)],
)
def test_get_store_container_mount_stores(self):
# Managed store
store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(
mount_path="/tmp", volume_claim="test", read_only=True
),
)
container = get_store_container(
polyaxon_init=V1PolyaxonInitContainer(
image="foo/foo", image_tag="foo", image_pull_policy="IfNotPresent"
),
connection=store,
artifacts=None,
)
mount_path = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name)
assert container.name == INIT_ARTIFACTS_CONTAINER.format(store.name)
assert container.image == "foo/foo:foo"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["/bin/sh", "-c"]
assert container.args == [
get_volume_args(store=store, mount_path=mount_path, artifacts=None)
]
assert container.env == get_connection_env_var(connection=store, secret=None)
assert container.env_from == []
assert container.resources is not None
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name),
),
get_mount_from_store(store=store),
]
def test_get_volume_args_host(self):
host_path_store = V1ConnectionType(
name="test_path",
kind=V1ConnectionKind.HOST_PATH,
schema=V1HostPathConnection(
mount_path="/tmp", host_path="/tmp", read_only=True
),
)
path_to = "/path/to/"
path_from = os.path.join(host_path_store.store_path, "")
assert get_volume_args(host_path_store, path_to, None) == "; ".join(
[
get_or_create_args(path=path_to),
cp_copy_args(path_from=path_from, path_to=path_to, is_file=False),
]
)
host_path_store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.HOST_PATH,
schema=V1HostPathConnection(
mount_path="/tmp", host_path="/tmp", read_only=True
),
)
base_path = "/path/to/"
path_to1 = "/path/to/path1"
path_to2 = "/path/to/path2"
path_from1 = os.path.join(host_path_store.store_path, "path1")
path_from2 = os.path.join(host_path_store.store_path, "path2")
assert get_volume_args(
host_path_store,
"/path/to",
artifacts=V1ArtifactsType(dirs=["path1", "path2"]),
) == "; ".join(
[
get_or_create_args(path=base_path),
cp_copy_args(path_from=path_from1, path_to=path_to1, is_file=False),
get_or_create_args(path=base_path),
cp_copy_args(path_from=path_from2, path_to=path_to2, is_file=False),
]
)
def test_get_volume_args_claim(self):
claim_store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/tmp",
volume_claim="test",
read_only=True),
)
path_to = "/path/to/"
path_from = os.path.join(claim_store.store_path, "")
assert get_volume_args(claim_store, path_to, None) == "; ".join([
get_or_create_args(path=path_to),
cp_copy_args(path_from=path_from, path_to=path_to, is_file=False),
])
claim_store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/tmp",
volume_claim="test",
read_only=True),
)
base_path = "/path/to/"
path_to1 = "/path/to/path1"
path_to2 = "/path/to/path2"
path_from1 = os.path.join(claim_store.store_path, "path1")
path_from2 = os.path.join(claim_store.store_path, "path2")
assert get_volume_args(
claim_store,
"/path/to",
artifacts=V1ArtifactsType(files=["path1", "path2"])) == "; ".join([
get_or_create_args(path=base_path),
cp_copy_args(path_from=path_from1,
path_to=path_to1,
is_file=True),
get_or_create_args(path=base_path),
cp_copy_args(path_from=path_from2,
path_to=path_to2,
is_file=True),
])
def test_get_volume_args_gcs(self):
gcs_store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
)
path_to = "/path/to/"
path_from = os.path.join(gcs_store.store_path, "")
assert get_volume_args(gcs_store, path_to, None) == " ".join([
get_or_create_args(path=path_to),
cp_gcs_args(path_from=path_from, path_to=path_to, is_file=False),
])
gcs_store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="Congs//:foo"),
)
base_path = "/path/to/"
path_to1 = "/path/to/path1"
path_to2 = "/path/to/path2"
path_from1 = os.path.join(gcs_store.store_path, "path1")
path_from2 = os.path.join(gcs_store.store_path, "path2")
assert get_volume_args(
gcs_store,
"/path/to",
artifacts=V1ArtifactsType(dirs=["path1", "path2"])) == " ".join([
get_or_create_args(path=base_path),
cp_gcs_args(path_from=path_from1,
path_to=path_to1,
is_file=False),
get_or_create_args(path=base_path),
cp_gcs_args(path_from=path_from2,
path_to=path_to2,
is_file=False),
])
def test_get_volume_args_az(self):
az_store = V1ConnectionType(
name="test_az",
kind=V1ConnectionKind.WASB,
schema=V1BucketConnection(bucket="Conwasb://[email protected]"),
)
path_to = "/path/to/"
path_from = os.path.join(az_store.store_path, "")
assert get_volume_args(az_store, path_to, None) == "; ".join(
[
get_or_create_args(path=path_to),
cp_wasb_args(path_from=path_from, path_to=path_to, is_file=False),
]
)
az_store = V1ConnectionType(
name="test_az",
kind=V1ConnectionKind.WASB,
schema=V1BucketConnection(bucket="Conwasb://[email protected]"),
)
base_path = "/path/to/"
path_to1 = "/path/to/path1"
path_to2 = "/path/to/path2"
path_from1 = os.path.join(az_store.store_path, "path1")
path_from2 = os.path.join(az_store.store_path, "path2")
assert get_volume_args(
az_store,
"/path/to",
artifacts=V1ArtifactsType(files=["path1"], dirs=["path2"]),
) == "; ".join(
[
get_or_create_args(path=base_path),
cp_wasb_args(path_from=path_from1, path_to=path_to1, is_file=True),
get_or_create_args(path=base_path),
cp_wasb_args(path_from=path_from2, path_to=path_to2, is_file=False),
]
)
def test_get_store_container_bucket_stores(self):
mount_path = "/test-path"
resource1 = V1K8sResourceType(
name="non_mount_test1",
schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]),
is_requested=False,
)
store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
secret=resource1.schema,
)
container = get_store_container(
polyaxon_init=V1PolyaxonInitContainer(
image="foo/foo", image_tag="", image_pull_policy="IfNotPresent"
),
connection=store,
artifacts=None,
mount_path=mount_path,
)
assert container.name == generate_container_name(
INIT_ARTIFACTS_CONTAINER_PREFIX, store.name
)
assert container.image == "foo/foo"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["/bin/sh", "-c"]
assert container.args == [
get_volume_args(store=store, mount_path=mount_path, artifacts=None)
]
assert container.env is not None
assert container.env_from == []
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=get_volume_name(mount_path), mount_path=mount_path
)
]