def test_get_volume_args_s3(self): s3_store = V1ConnectionType( name="test_s3", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3//:foo"), ) path_to = "/path-to/" path_from = os.path.join(s3_store.store_path, "") assert get_volume_args(s3_store, path_to, None) == "; ".join( [ get_or_create_args(path=path_to), cp_s3_args(path_from=path_from, path_to=path_to, is_file=False), ] ) s3_store = V1ConnectionType( name="test_s3", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3//:foo"), ) base_path = "/path/to/" path_to1 = "/path/to/path1" path_to2 = "/path/to/path2" path_from1 = os.path.join(s3_store.store_path, "path1") path_from2 = os.path.join(s3_store.store_path, "path2") assert get_volume_args( s3_store, "/path/to", artifacts=V1ArtifactsType(files=["path1", "path2"]) ) == "; ".join( [ get_or_create_args(path=base_path), cp_s3_args(path_from=path_from1, path_to=path_to1, is_file=True), get_or_create_args(path=base_path), cp_s3_args(path_from=path_from2, path_to=path_to2, is_file=True), ] )
def get_artifacts_path_container( polyaxon_init: V1PolyaxonInitContainer, artifacts_store: V1ConnectionType, run_path: str, auto_resume: bool, ) -> Optional[k8s_schemas.V1Container]: if not artifacts_store: raise PolypodException("Init artifacts container requires a store.") init_args = init_artifact_context_args(run_path=run_path) if auto_resume: init_args.append( get_volume_args( store=artifacts_store, mount_path=CONTEXT_MOUNT_ARTIFACTS, artifacts=V1ArtifactsType(dirs=[run_path]), )) container_name = generate_container_name(INIT_ARTIFACTS_CONTAINER_PREFIX, DEFAULT) container = k8s_schemas.V1Container(name=container_name) return get_base_store_container( container_name=container_name, container=container, polyaxon_init=polyaxon_init, store=artifacts_store, env=[], env_from=[], volume_mounts=[get_artifacts_context_mount()], # If we are dealing with a volume we need to make sure the path exists for the user # We also clean the path if this is not a resume run args=[" ".join(init_args)], )
def test_get_artifacts_path_container_with_managed_mount_store(self): store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(mount_path="/claim/path", volume_claim="claim"), ) container = get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer( image="init", image_pull_policy="IfNotPresent"), artifacts_store=store, run_path="run_uid", auto_resume=True, ) init_args = init_artifact_context_args("run_uid") init_args.append( get_volume_args( store=store, mount_path=CONTEXT_MOUNT_ARTIFACTS, artifacts=V1ArtifactsType(dirs=["run_uid"]), )) assert container == get_base_store_container( container=k8s_schemas.V1Container(name="default"), container_name=INIT_ARTIFACTS_CONTAINER.format("default"), polyaxon_init=V1PolyaxonInitContainer( image="init", image_pull_policy="IfNotPresent"), store=store, env=[], env_from=[], volume_mounts=[get_artifacts_context_mount()], args=[" ".join(init_args)], )
def test_get_artifacts_path_container_with_bucket_store(self): store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), ) container = get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer( image="init", image_pull_policy="IfNotPresent"), artifacts_store=store, run_path="run_uid", auto_resume=True, ) init_args = init_artifact_context_args("run_uid") init_args.append( get_volume_args( store=store, mount_path=CONTEXT_MOUNT_ARTIFACTS, artifacts=V1ArtifactsType(dirs=["run_uid"]), )) assert container == get_base_store_container( container=k8s_schemas.V1Container(name="default"), container_name=generate_container_name( INIT_ARTIFACTS_CONTAINER_PREFIX, "default", False), polyaxon_init=V1PolyaxonInitContainer( image="init", image_pull_policy="IfNotPresent"), store=store, env=[], env_from=[], volume_mounts=[get_artifacts_context_mount()], args=[" ".join(init_args)], )
def test_get_store_container_mount_stores(self): # Managed store store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection( mount_path="/tmp", volume_claim="test", read_only=True ), ) container = get_store_container( polyaxon_init=V1PolyaxonInitContainer( image="foo/foo", image_tag="foo", image_pull_policy="IfNotPresent" ), connection=store, artifacts=None, ) mount_path = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name) assert container.name == INIT_ARTIFACTS_CONTAINER.format(store.name) assert container.image == "foo/foo:foo" assert container.image_pull_policy == "IfNotPresent" assert container.command == ["/bin/sh", "-c"] assert container.args == [ get_volume_args(store=store, mount_path=mount_path, artifacts=None) ] assert container.env == get_connection_env_var(connection=store, secret=None) assert container.env_from == [] assert container.resources is not None assert container.volume_mounts == [ get_connections_context_mount( name=constants.CONTEXT_VOLUME_ARTIFACTS, mount_path=CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name), ), get_mount_from_store(store=store), ]
def test_get_volume_args_host(self): host_path_store = V1ConnectionType( name="test_path", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection( mount_path="/tmp", host_path="/tmp", read_only=True ), ) path_to = "/path/to/" path_from = os.path.join(host_path_store.store_path, "") assert get_volume_args(host_path_store, path_to, None) == "; ".join( [ get_or_create_args(path=path_to), cp_copy_args(path_from=path_from, path_to=path_to, is_file=False), ] ) host_path_store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection( mount_path="/tmp", host_path="/tmp", read_only=True ), ) base_path = "/path/to/" path_to1 = "/path/to/path1" path_to2 = "/path/to/path2" path_from1 = os.path.join(host_path_store.store_path, "path1") path_from2 = os.path.join(host_path_store.store_path, "path2") assert get_volume_args( host_path_store, "/path/to", artifacts=V1ArtifactsType(dirs=["path1", "path2"]), ) == "; ".join( [ get_or_create_args(path=base_path), cp_copy_args(path_from=path_from1, path_to=path_to1, is_file=False), get_or_create_args(path=base_path), cp_copy_args(path_from=path_from2, path_to=path_to2, is_file=False), ] )
def test_get_volume_args_claim(self): claim_store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(mount_path="/tmp", volume_claim="test", read_only=True), ) path_to = "/path/to/" path_from = os.path.join(claim_store.store_path, "") assert get_volume_args(claim_store, path_to, None) == "; ".join([ get_or_create_args(path=path_to), cp_copy_args(path_from=path_from, path_to=path_to, is_file=False), ]) claim_store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(mount_path="/tmp", volume_claim="test", read_only=True), ) base_path = "/path/to/" path_to1 = "/path/to/path1" path_to2 = "/path/to/path2" path_from1 = os.path.join(claim_store.store_path, "path1") path_from2 = os.path.join(claim_store.store_path, "path2") assert get_volume_args( claim_store, "/path/to", artifacts=V1ArtifactsType(files=["path1", "path2"])) == "; ".join([ get_or_create_args(path=base_path), cp_copy_args(path_from=path_from1, path_to=path_to1, is_file=True), get_or_create_args(path=base_path), cp_copy_args(path_from=path_from2, path_to=path_to2, is_file=True), ])
def test_get_volume_args_gcs(self): gcs_store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), ) path_to = "/path/to/" path_from = os.path.join(gcs_store.store_path, "") assert get_volume_args(gcs_store, path_to, None) == " ".join([ get_or_create_args(path=path_to), cp_gcs_args(path_from=path_from, path_to=path_to, is_file=False), ]) gcs_store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="Congs//:foo"), ) base_path = "/path/to/" path_to1 = "/path/to/path1" path_to2 = "/path/to/path2" path_from1 = os.path.join(gcs_store.store_path, "path1") path_from2 = os.path.join(gcs_store.store_path, "path2") assert get_volume_args( gcs_store, "/path/to", artifacts=V1ArtifactsType(dirs=["path1", "path2"])) == " ".join([ get_or_create_args(path=base_path), cp_gcs_args(path_from=path_from1, path_to=path_to1, is_file=False), get_or_create_args(path=base_path), cp_gcs_args(path_from=path_from2, path_to=path_to2, is_file=False), ])
def test_get_volume_args_az(self): az_store = V1ConnectionType( name="test_az", kind=V1ConnectionKind.WASB, schema=V1BucketConnection(bucket="Conwasb://[email protected]"), ) path_to = "/path/to/" path_from = os.path.join(az_store.store_path, "") assert get_volume_args(az_store, path_to, None) == "; ".join( [ get_or_create_args(path=path_to), cp_wasb_args(path_from=path_from, path_to=path_to, is_file=False), ] ) az_store = V1ConnectionType( name="test_az", kind=V1ConnectionKind.WASB, schema=V1BucketConnection(bucket="Conwasb://[email protected]"), ) base_path = "/path/to/" path_to1 = "/path/to/path1" path_to2 = "/path/to/path2" path_from1 = os.path.join(az_store.store_path, "path1") path_from2 = os.path.join(az_store.store_path, "path2") assert get_volume_args( az_store, "/path/to", artifacts=V1ArtifactsType(files=["path1"], dirs=["path2"]), ) == "; ".join( [ get_or_create_args(path=base_path), cp_wasb_args(path_from=path_from1, path_to=path_to1, is_file=True), get_or_create_args(path=base_path), cp_wasb_args(path_from=path_from2, path_to=path_to2, is_file=False), ] )
def test_get_store_container_bucket_stores(self): mount_path = "/test-path" resource1 = V1K8sResourceType( name="non_mount_test1", schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]), is_requested=False, ) store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), secret=resource1.schema, ) container = get_store_container( polyaxon_init=V1PolyaxonInitContainer( image="foo/foo", image_tag="", image_pull_policy="IfNotPresent" ), connection=store, artifacts=None, mount_path=mount_path, ) assert container.name == generate_container_name( INIT_ARTIFACTS_CONTAINER_PREFIX, store.name ) assert container.image == "foo/foo" assert container.image_pull_policy == "IfNotPresent" assert container.command == ["/bin/sh", "-c"] assert container.args == [ get_volume_args(store=store, mount_path=mount_path, artifacts=None) ] assert container.env is not None assert container.env_from == [] assert container.resources == get_init_resources() assert container.volume_mounts == [ get_connections_context_mount( name=get_volume_name(mount_path), mount_path=mount_path ) ]