def assert_artifacts_store_raises(self, store, run_path=None): with self.assertRaises(PolypodException): get_sidecar_container( container_id=MAIN_JOB_CONTAINER, contexts=PluginsContextsSpec.from_config( V1Plugins(collect_logs=True, collect_artifacts=False)), env=None, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=213, ), artifacts_store=store, run_path=run_path, )
def test_get_sidecar_container_with_non_managed_bucket_artifacts_logs_store( self): env_vars = [ get_env_var(name="key1", value="value1"), get_env_var(name="key2", value="value2"), ] bucket_non_managed_store = V1ConnectionType( name="test_s3", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3//:foo"), ) sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=-1, ), artifacts_store=bucket_non_managed_store, contexts=PluginsContextsSpec.from_config( V1Plugins(collect_logs=False, collect_artifacts=False, auth=True)), run_path=None, ) assert sidecar is None
def test_get_sidecar_container_with_non_managed_mount_outputs_logs_store( self): env_vars = [ get_env_var(name="key1", value="value1"), get_env_var(name="key2", value="value2"), ] mount_non_managed_store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(volume_claim="test", mount_path="/tmp", read_only=True), ) sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=-1, ), artifacts_store=mount_non_managed_store, contexts=PluginsContextsSpec.from_config( V1Plugins(collect_logs=False, collect_artifacts=False, auth=True)), run_path=None, ) assert sidecar is None
def test_get_sidecar_container_with_managed_mount_outputs_and_blob_logs_store(self): env_vars = [ get_env_var(name="key1", value="value1"), get_env_var(name="key2", value="value2"), ] resource1 = V1K8sResourceType( name="test1", schema=V1K8sResourceSchema(name="ref1"), is_requested=False ) blob_managed_store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), secret=resource1.schema, ) # logs and outputs sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=blob_managed_store, contexts=PluginsContextsSpec.from_config(None, default_auth=True), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.args == get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212, monitor_logs=False, ) assert ( sidecar.env == get_sidecar_env_vars( env_vars=env_vars, container_id=MAIN_JOB_CONTAINER, artifacts_store_name=blob_managed_store.name, ) + get_connection_env_var(connection=blob_managed_store, secret=resource1) ) assert sidecar.env_from == [get_env_from_secret(secret=resource1)] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), ]
def test_get_sidecar_container_without_an_artifacts_store(self): sidecar = get_sidecar_container( env=None, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=213, ), artifacts_store=None, contexts=PluginsContextsSpec.from_config(None), run_path=None, ) assert sidecar is None
def get_sidecar_containers( self, polyaxon_sidecar: V1PolyaxonSidecarContainer, contexts: PluginsContextsSpec, artifacts_store: V1ConnectionType, sidecar_containers: List[k8s_schemas.V1Container], ) -> List[k8s_schemas.V1Container]: polyaxon_sidecar_container = get_sidecar_container( polyaxon_sidecar=polyaxon_sidecar, env=self.get_polyaxon_sidecar_service_env_vars(), artifacts_store=artifacts_store, contexts=contexts, run_path=self.run_path, ) containers = to_list(polyaxon_sidecar_container, check_none=True) containers += to_list(sidecar_containers, check_none=True) return containers
def test_get_sidecar_container_host_paths(self): artifacts_store = V1ConnectionType( name="plx-outputs", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection(mount_path="/tmp/plx/outputs", host_path="/tmp/plx/outputs"), ) contexts = PluginsContextsSpec( auth=True, docker=False, shm=False, collect_logs=True, collect_artifacts=True, collect_resources=True, auto_resume=True, sync_statuses=True, external_host=False, sidecar=None, ) sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="foo", image_pull_policy="sdf", sleep_interval=2, sync_interval=212, ), env=[], artifacts_store=artifacts_store, contexts=contexts, run_path="test", ) assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), get_mount_from_store(store=artifacts_store), ]
def get_sidecar_containers( self, polyaxon_sidecar: V1PolyaxonSidecarContainer, contexts: PluginsContextsSpec, artifacts_store: V1ConnectionType, sidecar_containers: List[k8s_schemas.V1Container], ) -> List[k8s_schemas.V1Container]: sidecar_containers = [ ensure_container_name(container=c, prefix=SIDECAR_PREFIX) for c in to_list(sidecar_containers, check_none=True) ] polyaxon_sidecar_container = get_sidecar_container( container_id=self.MAIN_CONTAINER_ID, polyaxon_sidecar=polyaxon_sidecar, env=self.get_polyaxon_sidecar_service_env_vars(), artifacts_store=artifacts_store, contexts=contexts, run_path=self.run_path, ) containers = to_list(polyaxon_sidecar_container, check_none=True) containers += sidecar_containers return [sanitize_container_command_args(c) for c in containers]
def test_get_sidecar_container_with_managed_mount_outputs_logs_store(self): env_vars = [ get_env_var(name="key1", value="value1"), get_env_var(name="key2", value="value2"), ] mount_managed_store = V1ConnectionType( name="test_path", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection(mount_path="/tmp", host_path="/tmp"), secret=None, ) # logs and outputs sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=mount_managed_store, contexts=PluginsContextsSpec.from_config(None, default_auth=True), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.args == get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212, monitor_logs=False, ) assert sidecar.env == get_sidecar_env_vars( env_vars=env_vars, container_id=MAIN_JOB_CONTAINER, artifacts_store_name=mount_managed_store.name, ) + get_connection_env_var(connection=mount_managed_store, secret=None) assert sidecar.env_from == [] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), get_mount_from_store(store=mount_managed_store), ] # logs and no outputs sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=mount_managed_store, contexts=PluginsContextsSpec.from_config( V1Plugins(collect_artifacts=False, auth=True)), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.args == get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212, monitor_logs=False, ) assert sidecar.env == get_sidecar_env_vars( env_vars=env_vars, container_id=MAIN_JOB_CONTAINER, artifacts_store_name=mount_managed_store.name, ) + get_connection_env_var(connection=mount_managed_store, secret=None) assert sidecar.env_from == [] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_mount_from_store(store=mount_managed_store), ] # outputs and no logs sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=mount_managed_store, contexts=PluginsContextsSpec.from_config( V1Plugins(collect_logs=False, auth=True)), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.args == get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212, monitor_logs=False, ) assert sidecar.env == get_sidecar_env_vars( env_vars=env_vars, container_id=MAIN_JOB_CONTAINER, artifacts_store_name=mount_managed_store.name, ) + get_connection_env_var(connection=mount_managed_store, secret=None) assert sidecar.env_from == [] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), get_mount_from_store(store=mount_managed_store), ]
def test_get_sidecar_container_auth_context(self, ): env_vars = [ get_env_var(name="key1", value="value1"), get_env_var(name="key2", value="value2"), ] resource1 = V1K8sResourceType( name="test1", schema=V1K8sResourceSchema(name="ref1", items=["item1", "item2"]), is_requested=False, ) bucket_managed_store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), secret=resource1.schema, ) # Default auth is included sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=bucket_managed_store, contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), ] # Nno auth sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=-212, ), artifacts_store=bucket_managed_store, contexts=PluginsContextsSpec.from_config(V1Plugins(auth=False)), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.volume_mounts == [ get_artifacts_context_mount(read_only=False) ]
def test_get_sidecars(self): assert (self.converter.get_sidecar_containers( contexts=None, artifacts_store=None, sidecar_containers=[], polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", sleep_interval=12, sync_interval=-1), ) == []) # Store with single path, no secret is passed and not required store = V1ConnectionType( name="test", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3://foo"), ) contexts = PluginsContextsSpec.from_config( V1Plugins(collect_logs=True, collect_artifacts=True, auth=True)) assert self.converter.get_sidecar_containers( contexts=contexts, artifacts_store=store, sidecar_containers=[], polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", sleep_interval=12, sync_interval=12), ) == [ get_sidecar_container( contexts=contexts, env=self.converter.get_polyaxon_sidecar_service_env_vars(), polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", sleep_interval=12, sync_interval=12), artifacts_store=store, run_path=self.converter.run_path, ) ] secret1 = V1K8sResourceType( name="test1", schema=V1K8sResourceSchema(name="ref1", items=["item1", "item2"]), is_requested=True, ) store.secret = secret1.schema polyaxon_sidecar = V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_pull_policy=None, sleep_interval=12, sync_interval=-1, ) assert self.converter.get_sidecar_containers( contexts=contexts, artifacts_store=store, polyaxon_sidecar=polyaxon_sidecar, sidecar_containers=[], ) == [ get_sidecar_container( contexts=contexts, env=self.converter.get_polyaxon_sidecar_service_env_vars(), polyaxon_sidecar=polyaxon_sidecar, artifacts_store=store, run_path=self.converter.run_path, ) ]
def test_get_sidecar_container_override_sync(self): artifacts_store = V1ConnectionType( name="plx-outputs", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection(mount_path="/tmp/plx/outputs", host_path="/tmp/plx/outputs"), ) contexts = PluginsContextsSpec( auth=True, docker=False, shm=False, collect_logs=True, collect_artifacts=True, collect_resources=True, auto_resume=True, sync_statuses=True, external_host=False, sidecar=None, ) sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="foo", image_pull_policy="sdf", sleep_interval=2, sync_interval=212, ), env=[], artifacts_store=artifacts_store, contexts=contexts, run_path="test", ) assert sidecar.args == [ "--container-id=polyaxon-main", "--sleep-interval=2", "--sync-interval=212", ] contexts = PluginsContextsSpec( auth=True, docker=False, shm=False, collect_logs=True, collect_artifacts=True, collect_resources=True, auto_resume=True, sync_statuses=True, external_host=False, sidecar=V1PolyaxonSidecarContainer( sleep_interval=-1, sync_interval=-1, ), ) sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="foo", image_pull_policy="sdf", sleep_interval=2, sync_interval=212, ), env=[], artifacts_store=artifacts_store, contexts=contexts, run_path="test", ) assert sidecar.args == [ "--container-id=polyaxon-main", "--sleep-interval=-1", "--sync-interval=-1", ]
def test_get_sidecar_container_with_managed_bucket_outputs_logs_store_and_mount_secret( self, ): env_vars = [ get_env_var(name="key1", value="value1"), get_env_var(name="key2", value="value2"), ] resource1 = V1K8sResourceType( name="test1", schema=V1K8sResourceSchema( name="test1", items=["item1", "item2"], mount_path="/path" ), is_requested=False, ) bucket_managed_store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), secret=resource1.schema, ) # Both logs and outputs sidecar = get_sidecar_container( env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=bucket_managed_store, contexts=PluginsContextsSpec.from_config(None), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["/bin/bash", "-c"] assert sidecar.args == [ get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212 ) ] assert sidecar.env == get_sidecar_env_vars( env_vars=env_vars, job_container_name=MAIN_JOB_CONTAINER, artifacts_store_name=bucket_managed_store.name, ) + get_items_from_secret(secret=resource1) + get_connection_env_var( connection=bucket_managed_store, secret=resource1 ) assert sidecar.env_from == [] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), get_mount_from_resource(resource=resource1), ] # logs and no outputs sidecar = get_sidecar_container( env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=bucket_managed_store, contexts=PluginsContextsSpec.from_config( V1Plugins(collect_artifacts=False) ), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["/bin/bash", "-c"] assert sidecar.args == [ get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212 ) ] assert sidecar.env == get_sidecar_env_vars( env_vars=env_vars, job_container_name=MAIN_JOB_CONTAINER, artifacts_store_name=bucket_managed_store.name, ) + get_items_from_secret(secret=resource1) + get_connection_env_var( connection=bucket_managed_store, secret=resource1 ) assert sidecar.env_from == [] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_mount_from_resource(resource=resource1), ] # outputs and no logs sidecar = get_sidecar_container( env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=bucket_managed_store, contexts=PluginsContextsSpec.from_config(V1Plugins(collect_logs=False)), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["/bin/bash", "-c"] assert sidecar.args == [ get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212 ) ] assert sidecar.env == get_sidecar_env_vars( env_vars=env_vars, job_container_name=MAIN_JOB_CONTAINER, artifacts_store_name=bucket_managed_store.name, ) + get_items_from_secret(secret=resource1) + get_connection_env_var( connection=bucket_managed_store, secret=resource1 ) assert sidecar.env_from == [] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), get_mount_from_resource(resource=resource1), ]