def test_get_artifacts_path_container_with_none_values(self): with self.assertRaises(PolypodException): get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer(), artifacts_store=None, run_path="", auto_resume=True, )
def test_get_artifacts_path_container_with_non_managed_mount_store(self): store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(mount_path="/claim/path", volume_claim="claim"), ) container = get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer( image="init", image_pull_policy="IfNotPresent"), artifacts_store=store, run_path="run_uid", auto_resume=True, ) init_args = init_artifact_context_args("run_uid") init_args.append( get_volume_args( store=store, mount_path=CONTEXT_MOUNT_ARTIFACTS, artifacts=V1ArtifactsType(dirs=["run_uid"]), )) assert container == get_base_store_container( container=k8s_schemas.V1Container(name="init"), container_name=generate_container_name( INIT_ARTIFACTS_CONTAINER_PREFIX, "default", False), polyaxon_init=V1PolyaxonInitContainer( image="init", image_pull_policy="IfNotPresent"), store=store, env=[], env_from=[], volume_mounts=[get_artifacts_context_mount()], args=[" ".join(init_args)], ) container = get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer( image="init", image_pull_policy="IfNotPresent"), artifacts_store=store, run_path="run_uid", auto_resume=False, ) init_args = init_artifact_context_args("run_uid") assert container == get_base_store_container( container=k8s_schemas.V1Container(name="init"), container_name=generate_container_name( INIT_ARTIFACTS_CONTAINER_PREFIX, "default", False), polyaxon_init=V1PolyaxonInitContainer( image="init", image_pull_policy="IfNotPresent"), store=store, env=[], env_from=[], volume_mounts=[get_artifacts_context_mount()], args=[" ".join(init_args)], )
def test_get_artifacts_path_container_with_bucket_store(self): store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), ) container = get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer( image="init", image_pull_policy="IfNotPresent"), artifacts_store=store, run_path="run_uid", auto_resume=True, ) init_args = init_artifact_context_args("run_uid") init_args.append( get_volume_args( store=store, mount_path=CONTEXT_MOUNT_ARTIFACTS, artifacts=V1ArtifactsType(dirs=["run_uid"]), )) assert container == get_base_store_container( container=k8s_schemas.V1Container(name="default"), container_name=INIT_ARTIFACTS_CONTAINER.format("default"), polyaxon_init=V1PolyaxonInitContainer( image="init", image_pull_policy="IfNotPresent"), store=store, env=[], env_from=[], volume_mounts=[get_artifacts_context_mount()], args=[" ".join(init_args)], )
def test_get_artifacts_path_container_with_managed_mount_store(self): store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(mount_path="/claim/path", volume_claim="claim"), ) container = get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer( image="init", image_pull_policy="IfNotPresent" ), artifacts_store=store, run_path="run_uid", clean=True, ) init_args = init_artifact_context_args("run_uid") init_args.append( get_artifacts_store_args( artifacts_path=get_path(store.store_path, "run_uid"), clean=True ) ) assert container == get_base_store_container( container=k8s_schemas.V1Container(name="default"), container_name=INIT_ARTIFACTS_CONTAINER.format("default"), polyaxon_init=V1PolyaxonInitContainer( image="init", image_pull_policy="IfNotPresent" ), store=store, env=[], env_from=[], volume_mounts=[get_artifacts_context_mount()], args=[" ".join(init_args)], is_artifact_store=True, )
def get_init_containers( self, polyaxon_init: V1PolyaxonInitContainer, contexts: PluginsContextsSpec, artifacts_store: V1ConnectionType, init_connections: List[V1Init], init_containers: List[k8s_schemas.V1Container], connection_by_names: Dict[str, V1ConnectionType], ) -> List[k8s_schemas.V1Container]: init_containers = [ ensure_container_name(container=c, prefix=INIT_PREFIX) for c in to_list(init_containers, check_none=True) ] init_connections = to_list(init_connections, check_none=True) containers = [] # Add auth context if contexts and contexts.auth: containers.append( get_auth_context_container( polyaxon_init=polyaxon_init, env=self.get_auth_service_env_vars( external_host=contexts.external_host), )) # Add outputs if contexts and contexts.collect_artifacts: containers += to_list( get_artifacts_path_container( polyaxon_init=polyaxon_init, artifacts_store=artifacts_store, run_path=self.run_path, auto_resume=contexts.auto_resume, ), check_none=True, ) containers += self.handle_init_connections( polyaxon_init=polyaxon_init, artifacts_store=artifacts_store, init_connections=init_connections, connection_by_names=connection_by_names, contexts=contexts, ) init_containers = containers + init_containers return [sanitize_container(c) for c in init_containers]
def test_get_init_containers_with_store_outputs(self): store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3://foo"), ) # No context containers = self.converter.get_init_containers( contexts=None, artifacts_store=store, init_connections=[], init_containers=[], connection_by_names={}, polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), ) assert containers == [] # With context containers = self.converter.get_init_containers( contexts=PluginsContextsSpec.from_config( V1Plugins(collect_artifacts=True, collect_logs=False, auth=True)), artifacts_store=store, init_connections=[], init_containers=[], connection_by_names={}, polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), ) assert containers == [ get_auth_context_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), env=self.converter.get_auth_service_env_vars(), ), get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), artifacts_store=store, run_path=self.converter.run_path, auto_resume=True, ), ]
def get_init_containers( self, polyaxon_init: V1PolyaxonInitContainer, contexts: PluginsContextsSpec, artifacts_store: V1ConnectionType, init_connections: List[V1Init], init_containers: List[k8s_schemas.V1Container], connection_by_names: Dict[str, V1ConnectionType], is_resume: bool = False, ) -> List[k8s_schemas.V1Container]: init_containers = init_containers or [] init_connections = init_connections or [] containers = [] # Add auth context if contexts and contexts.auth: containers.append( get_auth_context_container( polyaxon_init=polyaxon_init, env=self.get_auth_service_env_vars())) # Add outputs if contexts and contexts.collect_artifacts: containers += to_list( get_artifacts_path_container( polyaxon_init=polyaxon_init, artifacts_store=artifacts_store, run_path=self.run_path, clean=not is_resume, ), check_none=True, ) containers += self.handle_init_connections( polyaxon_init=polyaxon_init, init_connections=init_connections, connection_by_names=connection_by_names, contexts=contexts, ) return containers + init_containers
def test_get_init_containers_with_claim_outputs(self): store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(mount_path="/claim/path", volume_claim="claim", read_only=True), ) # No context to enable the outputs containers = self.converter.get_init_containers( contexts=None, artifacts_store=store.name, init_connections=None, connection_by_names={}, init_containers=[], polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), ) assert containers == [] # Enable outputs containers = self.converter.get_init_containers( contexts=PluginsContextsSpec.from_config( V1Plugins(collect_artifacts=True, collect_logs=False)), artifacts_store=store, connection_by_names={}, init_connections=None, init_containers=[], polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), ) assert containers == [ get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), artifacts_store=store, run_path=self.converter.run_path, auto_resume=True, ), ] # Use store for init containers = self.converter.get_init_containers( contexts=None, artifacts_store=None, connection_by_names={store.name: store}, init_connections=[V1Init(connection=store.name)], init_containers=[], polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), ) assert containers == [ get_store_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), connection=store, artifacts=None, env=self.converter.get_init_service_env_vars(), ) ] # Use store for init and outputs containers = self.converter.get_init_containers( contexts=PluginsContextsSpec.from_config( V1Plugins(collect_artifacts=True, collect_logs=False)), artifacts_store=store, init_connections=[V1Init(connection=store.name)], connection_by_names={store.name: store}, init_containers=[], polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), ) assert containers == [ get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), artifacts_store=store, run_path=self.converter.run_path, auto_resume=True, ), get_store_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), connection=store, artifacts=None, env=self.converter.get_init_service_env_vars(), ), ] # Add Store store1 = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3://foo"), secret=None, ) containers = self.converter.get_init_containers( contexts=PluginsContextsSpec.from_config( V1Plugins(collect_artifacts=True, collect_logs=False, auth=True)), artifacts_store=store, init_connections=[ V1Init( connection=store.name, artifacts=V1ArtifactsType(files=["/foo", "/bar"]), ), V1Init( connection=store1.name, artifacts=V1ArtifactsType(files=["/foo", "/bar"]), ), ], connection_by_names={ store.name: store, store1.name: store1 }, init_containers=[], polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), ) assert containers == [ get_auth_context_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), env=self.converter.get_auth_service_env_vars(), ), get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), artifacts_store=store, run_path=self.converter.run_path, auto_resume=True, ), get_store_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), connection=store, artifacts=V1ArtifactsType(files=["/foo", "/bar"]), env=self.converter.get_init_service_env_vars(), ), get_store_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), connection=store1, artifacts=V1ArtifactsType(files=["/foo", "/bar"]), env=self.converter.get_init_service_env_vars(), ), ]