def test_get_store_container_mount_stores(self): # Managed store store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection( mount_path="/tmp", volume_claim="test", read_only=True ), ) container = get_store_container( polyaxon_init=V1PolyaxonInitContainer( image="foo/foo", image_tag="foo", image_pull_policy="IfNotPresent" ), connection=store, artifacts=None, ) mount_path = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name) assert container.name == INIT_ARTIFACTS_CONTAINER.format(store.name) assert container.image == "foo/foo:foo" assert container.image_pull_policy == "IfNotPresent" assert container.command == ["/bin/sh", "-c"] assert container.args == [ get_volume_args(store=store, mount_path=mount_path, artifacts=None) ] assert container.env == get_connection_env_var(connection=store, secret=None) assert container.env_from == [] assert container.resources is not None assert container.volume_mounts == [ get_connections_context_mount( name=constants.CONTEXT_VOLUME_ARTIFACTS, mount_path=CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name), ), get_mount_from_store(store=store), ]
def test_get_store_container_bucket_stores(self): mount_path = "/test-path" resource1 = V1K8sResourceType( name="non_mount_test1", schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]), is_requested=False, ) store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), secret=resource1.schema, ) container = get_store_container( polyaxon_init=V1PolyaxonInitContainer( image="foo/foo", image_tag="", image_pull_policy="IfNotPresent" ), connection=store, artifacts=None, mount_path=mount_path, ) assert container.name == generate_container_name( INIT_ARTIFACTS_CONTAINER_PREFIX, store.name ) assert container.image == "foo/foo" assert container.image_pull_policy == "IfNotPresent" assert container.command == ["/bin/sh", "-c"] assert container.args == [ get_volume_args(store=store, mount_path=mount_path, artifacts=None) ] assert container.env is not None assert container.env_from == [] assert container.resources == get_init_resources() assert container.volume_mounts == [ get_connections_context_mount( name=get_volume_name(mount_path), mount_path=mount_path ) ]
def handle_init_connections( self, polyaxon_init: V1PolyaxonInitContainer, artifacts_store: V1ConnectionType, init_connections: List[V1Init], connection_by_names: Dict[str, V1ConnectionType], contexts: PluginsContextsSpec, ) -> List[k8s_schemas.V1Container]: containers = [] external_host = contexts.external_host if contexts else False # Prepare connections that Polyaxon can init automatically for init_connection in init_connections: if init_connection.connection: connection_spec = connection_by_names.get( init_connection.connection) # Handling ssh with git if (V1ConnectionKind.is_ssh(connection_spec.kind) and init_connection.git): connection_spec.schema.patch_git(init_connection.git) containers.append( get_git_init_container( polyaxon_init=polyaxon_init, connection=connection_spec, container=init_connection.container, env=self.get_init_service_env_vars( external_host=external_host), mount_path=init_connection.path, contexts=contexts, track=True, )) elif V1ConnectionKind.is_git(connection_spec.kind): if init_connection.git: # Update the default schema connection_spec.schema.patch(init_connection.git) containers.append( get_git_init_container( polyaxon_init=polyaxon_init, connection=connection_spec, container=init_connection.container, env=self.get_init_service_env_vars( external_host=external_host), mount_path=init_connection.path, contexts=contexts, track=True, )) elif V1ConnectionKind.is_artifact(connection_spec.kind): containers.append( get_store_container( polyaxon_init=polyaxon_init, connection=connection_spec, artifacts=init_connection.artifacts, container=init_connection.container, env=self.get_init_service_env_vars( external_host=external_host), mount_path=init_connection.path, is_default_artifacts_store=artifacts_store and init_connection.connection == artifacts_store.name, )) else: containers.append( get_custom_init_container( connection=connection_spec, container=init_connection.container, env=self.get_init_service_env_vars( external_host=external_host), mount_path=init_connection.path, contexts=contexts, )) else: # artifacts init without connection should default to the artifactsStore if init_connection.artifacts: containers.append( get_store_container( polyaxon_init=polyaxon_init, connection=artifacts_store, artifacts=init_connection.artifacts, container=init_connection.container, env=self.get_init_service_env_vars( external_host=external_host), mount_path=init_connection.path, is_default_artifacts_store=True, )) # git init without connection if init_connection.git: git_name = init_connection.git.get_name() containers.append( get_git_init_container( polyaxon_init=polyaxon_init, connection=V1ConnectionType( name=git_name, kind=V1ConnectionKind.GIT, schema=init_connection.git, secret=None, ), container=init_connection.container, env=self.get_init_service_env_vars( external_host=external_host), mount_path=init_connection.path, contexts=contexts, track=False, )) # Dockerfile initialization if init_connection.dockerfile: containers.append( get_dockerfile_init_container( polyaxon_init=polyaxon_init, dockerfile_args=init_connection.dockerfile, env=self.get_init_service_env_vars( external_host=external_host), mount_path=init_connection.path, contexts=contexts, run_path=self.run_path, run_instance=self.run_instance, )) # File initialization if init_connection.file: containers.append( get_file_init_container( polyaxon_init=polyaxon_init, file_args=init_connection.file, env=self.get_init_service_env_vars( external_host=external_host), mount_path=init_connection.path, contexts=contexts, run_path=self.run_path, run_instance=self.run_instance, )) return containers
def test_get_init_containers_with_claim_outputs(self): store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(mount_path="/claim/path", volume_claim="claim", read_only=True), ) # No context to enable the outputs containers = self.converter.get_init_containers( contexts=None, artifacts_store=store.name, init_connections=None, connection_by_names={}, init_containers=[], polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), ) assert containers == [] # Enable outputs containers = self.converter.get_init_containers( contexts=PluginsContextsSpec.from_config( V1Plugins(collect_artifacts=True, collect_logs=False)), artifacts_store=store, connection_by_names={}, init_connections=None, init_containers=[], polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), ) assert containers == [ get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), artifacts_store=store, run_path=self.converter.run_path, auto_resume=True, ), ] # Use store for init containers = self.converter.get_init_containers( contexts=None, artifacts_store=None, connection_by_names={store.name: store}, init_connections=[V1Init(connection=store.name)], init_containers=[], polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), ) assert containers == [ get_store_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), connection=store, artifacts=None, env=self.converter.get_init_service_env_vars(), ) ] # Use store for init and outputs containers = self.converter.get_init_containers( contexts=PluginsContextsSpec.from_config( V1Plugins(collect_artifacts=True, collect_logs=False)), artifacts_store=store, init_connections=[V1Init(connection=store.name)], connection_by_names={store.name: store}, init_containers=[], polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), ) assert containers == [ get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), artifacts_store=store, run_path=self.converter.run_path, auto_resume=True, ), get_store_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), connection=store, artifacts=None, env=self.converter.get_init_service_env_vars(), ), ] # Add Store store1 = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3://foo"), secret=None, ) containers = self.converter.get_init_containers( contexts=PluginsContextsSpec.from_config( V1Plugins(collect_artifacts=True, collect_logs=False, auth=True)), artifacts_store=store, init_connections=[ V1Init( connection=store.name, artifacts=V1ArtifactsType(files=["/foo", "/bar"]), ), V1Init( connection=store1.name, artifacts=V1ArtifactsType(files=["/foo", "/bar"]), ), ], connection_by_names={ store.name: store, store1.name: store1 }, init_containers=[], polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), ) assert containers == [ get_auth_context_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), env=self.converter.get_auth_service_env_vars(), ), get_artifacts_path_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), artifacts_store=store, run_path=self.converter.run_path, auto_resume=True, ), get_store_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), connection=store, artifacts=V1ArtifactsType(files=["/foo", "/bar"]), env=self.converter.get_init_service_env_vars(), ), get_store_container( polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"), connection=store1, artifacts=V1ArtifactsType(files=["/foo", "/bar"]), env=self.converter.get_init_service_env_vars(), ), ]
def handle_init_connections( self, polyaxon_init: V1PolyaxonInitContainer, init_connections: List[V1Init], connection_by_names: Dict[str, V1ConnectionType], contexts: PluginsContextsSpec, ) -> List[k8s_schemas.V1Container]: containers = [] # Prepare connections that Polyaxon can inti automatically for init_connection in init_connections: if init_connection.connection: connection_spec = connection_by_names.get( init_connection.connection) if init_connection.git: # Update the default schema connection_spec.schema.patch(init_connection.git) if connection_spec.kind == V1ConnectionKind.GIT: containers.append( get_git_init_container( polyaxon_init=polyaxon_init, connection=connection_spec, container=init_connection.container, env=self.get_init_service_env_vars(), mount_path=init_connection.path, contexts=contexts, track=True, )) if V1ConnectionKind.is_artifact(connection_spec.kind): containers.append( get_store_container( polyaxon_init=polyaxon_init, connection=connection_spec, artifacts=init_connection.artifacts, container=init_connection.container, env=self.get_init_service_env_vars(), mount_path=init_connection.path, )) else: # git init without connection if init_connection.git: git_name = init_connection.git.get_name() containers.append( get_git_init_container( polyaxon_init=polyaxon_init, connection=V1ConnectionType( name=git_name, kind=V1ConnectionKind.GIT, schema=init_connection.git, secret=None, ), container=init_connection.container, env=self.get_init_service_env_vars(), mount_path=init_connection.path, contexts=contexts, track=False, )) # Dockerfile initialization if init_connection.dockerfile: containers.append( get_dockerfile_init_container( polyaxon_init=polyaxon_init, dockerfile_args=init_connection.dockerfile, env=self.get_init_service_env_vars(), mount_path=init_connection.path, contexts=contexts, run_path=self.run_path, )) return containers