def test_agent_config_from_str_envs(self): config_dict = { POLYAXON_KEYS_K8S_NAMESPACE: "foo", POLYAXON_KEYS_AGENT_ARTIFACTS_STORE: ujson.dumps( { "name": "test1", "kind": V1ConnectionKind.GCS, "schema": V1BucketConnection(bucket="gs://test").to_dict(), "secret": V1K8sResourceSchema(name="some").to_dict(), } ), POLYAXON_KEYS_AGENT_NOTIFICATION_CONNECTIONS: ujson.dumps( [ { "name": "slack", "kind": V1ConnectionKind.SLACK, "secret": V1K8sResourceSchema(name="some").to_dict(), } ], ), POLYAXON_KEYS_AGENT_CONNECTIONS: ujson.dumps( [ { "name": "test2", "kind": V1ConnectionKind.GCS, "schema": V1BucketConnection(bucket="gs://test").to_dict(), "secret": V1K8sResourceSchema(name="some").to_dict(), } ] ), } config = AgentConfig.from_dict(config_dict) assert len(config.secrets) == 1 assert len(config.to_light_dict()[POLYAXON_KEYS_AGENT_CONNECTIONS]) == 1
def test_agent_config(self): config_dict = {POLYAXON_KEYS_AGENT_ARTIFACTS_STORE: 12} with self.assertRaises(ValidationError): AgentConfig.from_dict(config_dict) config_dict = {POLYAXON_KEYS_AGENT_ARTIFACTS_STORE: "some"} with self.assertRaises(ValidationError): AgentConfig.from_dict(config_dict) config_dict = { POLYAXON_KEYS_K8S_NAMESPACE: "foo", POLYAXON_KEYS_AGENT_ARTIFACTS_STORE: { "name": "some", "kind": V1ConnectionKind.GCS, "schema": V1BucketConnection(bucket="gs://test").to_dict(), }, POLYAXON_KEYS_AGENT_CONNECTIONS: [], } config = AgentConfig.from_dict(config_dict) assert config.to_light_dict() == config_dict config_dict = { POLYAXON_KEYS_K8S_NAMESPACE: "foo", POLYAXON_KEYS_AGENT_ARTIFACTS_STORE: "some", POLYAXON_KEYS_AGENT_CONNECTIONS: [ { "name": "some", "kind": V1ConnectionKind.GCS, "schema": V1BucketConnection(bucket="gs://test").to_dict(), "secretResource": "some", } ], } with self.assertRaises(ValidationError): AgentConfig.from_dict(config_dict) config_dict = { POLYAXON_KEYS_K8S_NAMESPACE: "foo", POLYAXON_KEYS_AGENT_ARTIFACTS_STORE: { "name": "test", "kind": V1ConnectionKind.GCS, "schema": V1BucketConnection(bucket="gs://test").to_dict(), "secret": V1K8sResourceSchema(name="some").to_dict(), }, POLYAXON_KEYS_AGENT_CONNECTIONS: [ { "name": "some", "kind": V1ConnectionKind.GCS, "schema": V1BucketConnection(bucket="gs://test").to_dict(), "secret": V1K8sResourceSchema(name="some").to_dict(), }, { "name": "slack", "kind": V1ConnectionKind.SLACK, "secret": V1K8sResourceSchema(name="some").to_dict(), }, ], } config = AgentConfig.from_dict(config_dict) assert config.to_light_dict() == config_dict
def test_resolve_connections_with_invalid_config(self): fpath = tempfile.mkdtemp() AgentConfigManager.CONFIG_PATH = fpath secret1 = V1K8sResourceType( name="secret1", schema=V1K8sResourceSchema(name="secret1"), is_requested=True, ) secret2 = V1K8sResourceType( name="secret2", schema=V1K8sResourceSchema(name="secret2"), is_requested=True, ) artifacts_store = V1ConnectionType( name="test_s3", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3//:foo"), secret=secret1.schema, ) connection1 = V1ConnectionType( name="connection1", kind=V1ConnectionKind.REGISTRY, schema=V1HostConnection(url="localhost:5000"), secret=secret2.schema, ) connection2 = V1ConnectionType( name="connection2", kind=V1ConnectionKind.REGISTRY, ) settings.AGENT_CONFIG = AgentConfig( namespace="foo", artifacts_store=artifacts_store, connections=[connection1, connection2], ) resolver = BaseResolver( run=None, compiled_operation=self.compiled_operation, owner_name="user", project_name="p1", project_uuid=None, run_name="j1", run_uuid=None, run_path="test", params=None, ) resolver.resolve_connections() assert resolver.namespace == "foo" assert resolver.connection_by_names == { artifacts_store.name: artifacts_store, connection1.name: connection1, connection2.name: connection2, } assert resolver.artifacts_store == artifacts_store assert resolver.polyaxon_sidecar == get_default_sidecar_container() assert resolver.polyaxon_init == get_default_init_container() resolver.resolve_artifacts_lineage() assert len(resolver.artifacts) == 3
def setUp(self): self.spec1 = V1K8sResourceType(name="test1", is_requested=True) self.spec2 = V1K8sResourceType(name="test2", schema=V1K8sResourceSchema(name="ref2"), is_requested=False) self.spec3 = V1K8sResourceType( name="test3", schema=V1K8sResourceSchema(name="ref3", items=["item45"], mount_path="/some_path"), is_requested=False, ) super().setUp()
def test_claim_connect_config(self): config_dict = {} with self.assertRaises(ValidationError): V1K8sResourceSchema.from_dict(config_dict) with self.assertRaises(ValidationError): validate_k8s_resource(config_dict) config_dict = {"name": "sdf"} V1K8sResourceSchema.from_dict(config_dict) validate_k8s_resource(config_dict) config_dict = {"name": "sdf", "items": ["foo"], "mountPath": "/bar"} config = V1K8sResourceSchema.from_dict(config_dict) assert config.to_dict() == config_dict validate_k8s_resource(config_dict)
def test_get_env_from_secrets(self): # None assert get_env_from_secrets(secrets=None) == [] # Secret with items secret1 = V1K8sResourceType( name="test1", schema=V1K8sResourceSchema(name="test1", items=["item1", "item2"]), is_requested=True, ) # Secret secret2 = V1K8sResourceType( name="test2", schema=V1K8sResourceSchema(name="test_ref"), is_requested=True) assert get_env_from_secrets(secrets=[secret1, secret2]) == [ get_env_from_secret(secret2) ]
def test_get_sidecar_container_with_managed_mount_outputs_and_blob_logs_store(self): env_vars = [ get_env_var(name="key1", value="value1"), get_env_var(name="key2", value="value2"), ] resource1 = V1K8sResourceType( name="test1", schema=V1K8sResourceSchema(name="ref1"), is_requested=False ) blob_managed_store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), secret=resource1.schema, ) # logs and outputs sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=blob_managed_store, contexts=PluginsContextsSpec.from_config(None, default_auth=True), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.args == get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212, monitor_logs=False, ) assert ( sidecar.env == get_sidecar_env_vars( env_vars=env_vars, container_id=MAIN_JOB_CONTAINER, artifacts_store_name=blob_managed_store.name, ) + get_connection_env_var(connection=blob_managed_store, secret=resource1) ) assert sidecar.env_from == [get_env_from_secret(secret=resource1)] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), ]
def test_get_env_from_secret(self): # None assert get_env_from_secret(secret=None) is None # Secret with items secret = V1K8sResourceType( name="test", schema=V1K8sResourceSchema(name="test", items=["item1", "item2"]), is_requested=True, ) assert get_env_from_secret(secret=secret) is None # Secret secret = V1K8sResourceType(name="test", schema=V1K8sResourceSchema(name="test_ref"), is_requested=True) assert get_env_from_secret(secret=secret).secret_ref == { "name": "test_ref" }
def test_get_env_from_config_maps(self): # None assert get_env_from_config_maps(config_maps=None) == [] # ConfigMap with items config_map1 = V1K8sResourceType( name="test1", schema=V1K8sResourceSchema(name="test1", items=["item1", "item2"]), is_requested=True, ) # ConfigMap config_map2 = V1K8sResourceType( name="test2", schema=V1K8sResourceSchema(name="test_ref"), is_requested=True) assert get_env_from_config_maps( config_maps=[config_map1, config_map2]) == [ get_env_from_config_map(config_map2) ]
def test_get_env_from_k8s_resources(self): assert get_env_from_k8s_resources(secrets=[], config_maps=[]) == [] res1 = V1K8sResourceType(name="test", schema=V1K8sResourceSchema(name="test"), is_requested=True) res2 = V1K8sResourceType(name="test2", schema=V1K8sResourceSchema(name="test2"), is_requested=True) expected = get_env_from_secrets(secrets=[res1, res2]) assert (get_env_from_k8s_resources(secrets=[res1, res2], config_maps=[]) == expected) expected = get_env_from_secrets( secrets=[res1]) + get_env_from_config_maps(config_maps=[res2]) assert (get_env_from_k8s_resources(secrets=[res1], config_maps=[res2]) == expected) expected = get_env_from_config_maps(config_maps=[res1, res2]) assert (get_env_from_k8s_resources(secrets=[], config_maps=[res1, res2]) == expected)
def test_get_env_from_config_map(self): # None assert get_env_from_config_map(config_map=None) is None # ConfigMap with items config_map = V1K8sResourceType( name="test", schema=V1K8sResourceSchema(name="test", items=["item1", "item2"]), is_requested=True, ) assert get_env_from_config_map(config_map=config_map) is None # ConfigMap config_map = V1K8sResourceType( name="test", schema=V1K8sResourceSchema(name="test_ref"), is_requested=True) assert get_env_from_config_map( config_map=config_map).config_map_ref == { "name": "test_ref" }
def test_get_env_vars_from_k8s_resources(self): assert get_env_vars_from_k8s_resources(secrets=[], config_maps=[]) == [] res1 = V1K8sResourceType(name="test", schema=V1K8sResourceSchema(name="test"), is_requested=True) res2 = V1K8sResourceType(name="test2", schema=V1K8sResourceSchema(name="test2"), is_requested=True) assert (get_env_vars_from_k8s_resources(secrets=[res1, res2], config_maps=[]) == []) assert get_env_vars_from_k8s_resources(secrets=[res1], config_maps=[res2]) == [] assert (get_env_vars_from_k8s_resources(secrets=[], config_maps=[res1, res2]) == []) res1 = V1K8sResourceType( name="test", schema=V1K8sResourceSchema(name="test", items=["item1", "item2"]), is_requested=True, ) res2 = V1K8sResourceType( name="test2", schema=V1K8sResourceSchema(name="test2", items=["item1", "item2"]), is_requested=True, ) expected = get_items_from_secret(res1) + get_items_from_secret(res2) assert (get_env_vars_from_k8s_resources(secrets=[res1, res2], config_maps=[]) == expected) expected = get_items_from_secret(res1) + get_items_from_config_map( res2) assert (get_env_vars_from_k8s_resources(secrets=[res1], config_maps=[res2 ]) == expected) expected = get_items_from_config_map(res1) + get_items_from_config_map( res2) assert (get_env_vars_from_k8s_resources(secrets=[], config_maps=[res1, res2 ]) == expected)
def setUp(self): super().setUp() self.store_root = set_store() settings.AGENT_CONFIG.notification_connections = [ V1ConnectionType( name="slack", kind=V1ConnectionKind.SLACK, secret=V1K8sResourceSchema(name="some"), ) ] self.client = get_streams_client() self.base_url = STREAMS_URL + "/namespace/owner/project/runs/uuid/notify"
def test_get_mount_from_resource(self): # Non mouth resource assert get_mount_from_resource(None) is None resource = V1K8sResourceType( name="test1", schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]), is_requested=False, ) assert get_mount_from_resource(resource=resource) is None # Resource with mount resource = V1K8sResourceType( name="test1", schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"], mount_path="/tmp"), is_requested=False, ) mount = get_mount_from_resource(resource=resource) assert mount.name == resource.name assert mount.mount_path == resource.schema.mount_path assert mount.read_only is True
def test_get_volume_from_secret(self): # No store assert get_volume_from_secret(None) is None # Store with mount path resource1 = V1K8sResourceType( name="test1", schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]), is_requested=False, ) assert get_volume_from_secret(resource1) is None # Claim store resource1 = V1K8sResourceType( name="test1", schema=V1K8sResourceSchema( name="ref", items=["item1", "item2"], mount_path="/tmp" ), is_requested=False, ) volume = get_volume_from_secret(resource1) assert volume.name == resource1.name assert volume.secret.secret_name == resource1.name assert volume.secret.items == resource1.schema.items
def get_items_from_config_map(self): # None assert get_items_from_secret(None) == [] # Secret without items secret = V1K8sResourceType(name="test", schema=V1K8sResourceSchema(name="test"), is_requested=True) assert get_items_from_secret(secret) == [] secret = V1K8sResourceType( name="test", schema=V1K8sResourceSchema(name="test", items=[]), is_requested=True, ) assert get_items_from_secret(secret) == [] # Secret with items secret = V1K8sResourceType( name="test", schema=V1K8sResourceSchema(name="test", items=["item1", "item2"]), is_requested=True, ) assert get_items_from_secret(secret) == [ get_from_config_map("item1", "item1", secret.schema.name), get_from_config_map("item2", "item2", secret.schema.name), ]
def setUp(self): super().setUp() # Secrets self.resource1 = V1K8sResourceType( name="non_mount_test1", schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]), is_requested=False, ) self.resource2 = V1K8sResourceType( name="non_mount_test2", schema=V1K8sResourceSchema(name="ref"), is_requested=False, ) self.resource3 = V1K8sResourceType( name="non_mount_test1", schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]), is_requested=True, ) self.resource4 = V1K8sResourceType( name="non_mount_test2", schema=V1K8sResourceSchema(name="ref"), is_requested=True, ) self.resource5 = V1K8sResourceType( name="non_mount_test2", schema=V1K8sResourceSchema(name="ref"), is_requested=True, ) self.resource6 = V1K8sResourceType( name="mount_test", schema=V1K8sResourceSchema(name="ref", mount_path="/test"), is_requested=True, ) # Connections self.bucket_store = V1ConnectionType( name="test_s3", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3//:foo"), secret=self.resource3.schema, ) self.mount_store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(mount_path="/tmp", volume_claim="test", read_only=True), )
def test_get_store_container_bucket_stores(self): mount_path = "/test-path" resource1 = V1K8sResourceType( name="non_mount_test1", schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]), is_requested=False, ) store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), secret=resource1.schema, ) container = get_store_container( polyaxon_init=V1PolyaxonInitContainer( image="foo/foo", image_tag="", image_pull_policy="IfNotPresent" ), connection=store, artifacts=None, mount_path=mount_path, ) assert container.name == generate_container_name( INIT_ARTIFACTS_CONTAINER_PREFIX, store.name ) assert container.image == "foo/foo" assert container.image_pull_policy == "IfNotPresent" assert container.command == ["/bin/sh", "-c"] assert container.args == [ get_volume_args(store=store, mount_path=mount_path, artifacts=None) ] assert container.env is not None assert container.env_from == [] assert container.resources == get_init_resources() assert container.volume_mounts == [ get_connections_context_mount( name=get_volume_name(mount_path), mount_path=mount_path ) ]
def test_resource_config(self): config_dict = {"name": "foo"} config = V1K8sResourceSchema.from_dict(config_dict) assert_equal_dict(config.to_dict(), config_dict) config_dict = {"name": "foo", "mount_path": 213} with self.assertRaises(ValidationError): V1K8sResourceSchema.from_dict(config_dict) config_dict = {"name": "foo", "items": 213} with self.assertRaises(ValidationError): V1K8sResourceSchema.from_dict(config_dict) config_dict = { "name": "foo", "mountPath": "/foo/path", "items": ["item1", "item2"], } config = V1K8sResourceSchema.from_dict(config_dict) assert_equal_dict(config.to_dict(), config_dict) config_dict = {"name": "foo", "items": ["item1", "item2"]} config = V1K8sResourceSchema.from_dict(config_dict) assert_equal_dict(config.to_dict(), config_dict)
def setUp(self): super().setUp() # Secrets and config maps self.non_mount_resource1 = V1K8sResourceType( name="non_mount_test1", schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]), is_requested=False, ) self.non_mount_resource2 = V1K8sResourceType( name="non_mount_test2", schema=V1K8sResourceSchema(name="ref"), is_requested=False, ) self.mount_resource1 = V1K8sResourceType( name="mount_test1", schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"], mount_path="/tmp1"), is_requested=False, ) self.mount_resource2 = V1K8sResourceType( name="mount_test2", schema=V1K8sResourceSchema(name="ref", mount_path="/tmp2"), is_requested=False, ) # Volumes self.vol1 = get_volume(volume="vol1", claim_name="claim1") self.vol2 = get_volume(volume="vol2", host_path="/path2") self.vol3 = get_volume(volume="vol3") # connections self.s3_store = V1ConnectionType( name="test_s3", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3//:foo"), secret=self.mount_resource1.schema, ) self.gcs_store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), secret=self.mount_resource1.schema, ) self.az_store = V1ConnectionType( name="test_az", kind=V1ConnectionKind.WASB, schema=V1BucketConnection( bucket="wasb://[email protected]"), secret=self.mount_resource1.schema, ) self.claim_store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(mount_path="/tmp", volume_claim="test", read_only=True), ) self.host_path_store = V1ConnectionType( name="test_path", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection(mount_path="/tmp", host_path="/tmp", read_only=True), )
def setUp(self): super().setUp() # Secrets self.resource1 = V1K8sResourceType( name="non_mount_test1", schema=V1K8sResourceSchema(name="non_mount_test1", items=["item1", "item2"]), is_requested=False, ) self.resource2 = V1K8sResourceType( name="non_mount_test2", schema=V1K8sResourceSchema(name="non_mount_test2"), is_requested=False, ) self.resource3 = V1K8sResourceType( name="non_mount_test3", schema=V1K8sResourceSchema(name="non_mount_test3", items=["item1", "item2"]), is_requested=True, ) self.resource4 = V1K8sResourceType( name="non_mount_test4", schema=V1K8sResourceSchema(name="non_mount_test4"), is_requested=True, ) self.resource5 = V1K8sResourceType( name="non_mount_test1", schema=V1K8sResourceSchema(name="non_mount_test1"), is_requested=True, ) # Connections self.s3_store = V1ConnectionType( name="test_s3", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3//:foo"), secret=self.resource1.schema, ) self.gcs_store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gcs//:foo"), secret=self.resource2.schema, ) self.az_store = V1ConnectionType( name="test_az", kind=V1ConnectionKind.WASB, schema=V1BucketConnection( bucket="wasb://[email protected]"), secret=self.resource3.schema, ) self.claim_store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(mount_path="/tmp", volume_claim="test"), ) self.host_path_store = V1ConnectionType( name="test_path", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection(mount_path="/tmp", host_path="/tmp", read_only=True), )
def test_get_sidecar_container_with_managed_bucket_outputs_logs_store_and_mount_secret( self, ): env_vars = [ get_env_var(name="key1", value="value1"), get_env_var(name="key2", value="value2"), ] resource1 = V1K8sResourceType( name="test1", schema=V1K8sResourceSchema( name="test1", items=["item1", "item2"], mount_path="/path" ), is_requested=False, ) bucket_managed_store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), secret=resource1.schema, ) # Both logs and outputs sidecar = get_sidecar_container( env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=bucket_managed_store, contexts=PluginsContextsSpec.from_config(None), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["/bin/bash", "-c"] assert sidecar.args == [ get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212 ) ] assert sidecar.env == get_sidecar_env_vars( env_vars=env_vars, job_container_name=MAIN_JOB_CONTAINER, artifacts_store_name=bucket_managed_store.name, ) + get_items_from_secret(secret=resource1) + get_connection_env_var( connection=bucket_managed_store, secret=resource1 ) assert sidecar.env_from == [] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), get_mount_from_resource(resource=resource1), ] # logs and no outputs sidecar = get_sidecar_container( env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=bucket_managed_store, contexts=PluginsContextsSpec.from_config( V1Plugins(collect_artifacts=False) ), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["/bin/bash", "-c"] assert sidecar.args == [ get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212 ) ] assert sidecar.env == get_sidecar_env_vars( env_vars=env_vars, job_container_name=MAIN_JOB_CONTAINER, artifacts_store_name=bucket_managed_store.name, ) + get_items_from_secret(secret=resource1) + get_connection_env_var( connection=bucket_managed_store, secret=resource1 ) assert sidecar.env_from == [] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_mount_from_resource(resource=resource1), ] # outputs and no logs sidecar = get_sidecar_container( env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=bucket_managed_store, contexts=PluginsContextsSpec.from_config(V1Plugins(collect_logs=False)), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["/bin/bash", "-c"] assert sidecar.args == [ get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212 ) ] assert sidecar.env == get_sidecar_env_vars( env_vars=env_vars, job_container_name=MAIN_JOB_CONTAINER, artifacts_store_name=bucket_managed_store.name, ) + get_items_from_secret(secret=resource1) + get_connection_env_var( connection=bucket_managed_store, secret=resource1 ) assert sidecar.env_from == [] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), get_mount_from_resource(resource=resource1), ]
def test_get_sidecar_container_auth_context(self, ): env_vars = [ get_env_var(name="key1", value="value1"), get_env_var(name="key2", value="value2"), ] resource1 = V1K8sResourceType( name="test1", schema=V1K8sResourceSchema(name="ref1", items=["item1", "item2"]), is_requested=False, ) bucket_managed_store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), secret=resource1.schema, ) # Default auth is included sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=bucket_managed_store, contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), ] # Nno auth sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=-212, ), artifacts_store=bucket_managed_store, contexts=PluginsContextsSpec.from_config(V1Plugins(auth=False)), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.volume_mounts == [ get_artifacts_context_mount(read_only=False) ]
def test_get_base_store_container_with_store_with_secret(self): non_mount_resource1 = V1K8sResourceType( name="resource", schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]), is_requested=False, ) bucket_store_with_secret = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), secret=non_mount_resource1.schema, ) container = get_base_store_container( container=k8s_schemas.V1Container(name="init"), container_name="init", polyaxon_init=V1PolyaxonInitContainer(image_tag=""), store=bucket_store_with_secret, env=None, env_from=None, volume_mounts=None, args=None, ) assert container.name == "init" assert container.image == "polyaxon/polyaxon-init" assert container.image_pull_policy is None assert container.command == ["/bin/sh", "-c"] assert container.args is None env = get_items_from_secret( secret=non_mount_resource1) + get_connection_env_var( connection=bucket_store_with_secret, secret=non_mount_resource1) assert container.env == env assert container.env_from == [] assert container.resources is not None assert container.volume_mounts == [] mount_resource1 = V1K8sResourceType( name="resource", schema=V1K8sResourceSchema(name="resource", items=["item1", "item2"], mount_path="/tmp1"), is_requested=False, ) bucket_store_with_secret.secret = mount_resource1.schema container = get_base_store_container( container=k8s_schemas.V1Container(name="init"), container_name="init", polyaxon_init=V1PolyaxonInitContainer(image_tag=""), store=bucket_store_with_secret, env=None, env_from=None, volume_mounts=None, args=None, ) assert container.name == "init" assert container.image == "polyaxon/polyaxon-init" assert container.image_pull_policy is None assert container.command == ["/bin/sh", "-c"] assert container.args is None assert container.env == get_connection_env_var( connection=bucket_store_with_secret, secret=mount_resource1) assert container.env_from == [] assert container.resources is not None assert container.volume_mounts == [ get_mount_from_resource(resource=mount_resource1) ]
def test_resolve_connections_with_invalid_config(self): fpath = tempfile.mkdtemp() AgentManager.CONFIG_PATH = fpath secret1 = V1K8sResourceType( name="secret1", schema=V1K8sResourceSchema(name="secret1"), is_requested=True, ) secret2 = V1K8sResourceType( name="secret2", schema=V1K8sResourceSchema(name="secret2"), is_requested=True, ) connection1 = V1ConnectionType( name="test_s3", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3//:foo"), secret=secret1.schema, ) connection2 = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gcs//:foo"), secret=secret1.schema, ) connection3 = V1ConnectionType( name="test_wasb", kind=V1ConnectionKind.WASB, schema=V1BucketConnection(bucket="wasbs//:foo"), secret=secret2.schema, ) settings.AGENT_CONFIG = AgentConfig( namespace="foo", artifacts_store=connection1, connections=[connection2, connection3], ) resolver = BaseResolver( run=None, compiled_operation=self.compiled_operation, owner_name="user", project_name="p1", project_uuid=None, run_name="j1", run_uuid=None, run_path="test", params=None, ) resolver.resolve_connections() assert resolver.namespace == "foo" assert resolver.connection_by_names == {connection1.name: connection1} assert resolver.artifacts_store == connection1 assert [s.schema for s in resolver.secrets] == [secret1.schema, secret2.schema] assert resolver.polyaxon_sidecar == get_default_sidecar_container() assert resolver.polyaxon_init == get_default_init_container() # Add run spec to resolve connections compiled_operation = V1CompiledOperation.read({ "version": 1.1, "kind": kinds.COMPILED_OPERATION, "plugins": { "auth": False, "shm": False, "collectLogs": False, "collectArtifacts": False, "collectResources": False, }, "run": { "kind": V1RunKind.JOB, "container": { "image": "test" }, "connections": {connection3.name}, }, }) resolver = BaseResolver( run=None, compiled_operation=compiled_operation, owner_name="user", project_name="p1", project_uuid=None, run_name="j1", run_uuid=None, run_path="test", params=None, ) resolver.resolve_connections() assert resolver.namespace == "foo" assert resolver.connection_by_names == { connection1.name: connection1, connection3.name: connection3, } assert [s.schema for s in resolver.secrets] == [secret1.schema, secret2.schema] assert resolver.artifacts_store == connection1 assert resolver.polyaxon_sidecar == get_default_sidecar_container() assert resolver.polyaxon_init == get_default_init_container() # Add run spec to resolve connections compiled_operation = V1CompiledOperation.read({ "version": 1.1, "kind": kinds.COMPILED_OPERATION, "plugins": { "auth": False, "shm": False, "collectLogs": False, "collectArtifacts": False, "collectResources": False, }, "run": { "kind": V1RunKind.JOB, "container": { "image": "test" }, "connections": { connection1.name, connection2.name, connection3.name, }, }, }) resolver = BaseResolver( run=None, compiled_operation=compiled_operation, owner_name="user", project_name="p1", project_uuid=None, run_name="j1", run_uuid=None, run_path="test", params=None, ) resolver.resolve_connections() assert resolver.namespace == "foo" assert resolver.connection_by_names == { connection3.name: connection3, connection2.name: connection2, connection1.name: connection1, } assert [s.schema for s in resolver.secrets] == [secret1.schema, secret2.schema] assert resolver.artifacts_store == connection1 assert resolver.polyaxon_sidecar == get_default_sidecar_container() assert resolver.polyaxon_init == get_default_init_container()
def test_get_sidecars(self): assert (self.converter.get_sidecar_containers( contexts=None, artifacts_store=None, sidecar_containers=[], polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", sleep_interval=12, sync_interval=-1), ) == []) # Store with single path, no secret is passed and not required store = V1ConnectionType( name="test", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3://foo"), ) contexts = PluginsContextsSpec.from_config( V1Plugins(collect_logs=True, collect_artifacts=True, auth=True)) assert self.converter.get_sidecar_containers( contexts=contexts, artifacts_store=store, sidecar_containers=[], polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", sleep_interval=12, sync_interval=12), ) == [ get_sidecar_container( contexts=contexts, env=self.converter.get_polyaxon_sidecar_service_env_vars(), polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", sleep_interval=12, sync_interval=12), artifacts_store=store, run_path=self.converter.run_path, ) ] secret1 = V1K8sResourceType( name="test1", schema=V1K8sResourceSchema(name="ref1", items=["item1", "item2"]), is_requested=True, ) store.secret = secret1.schema polyaxon_sidecar = V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_pull_policy=None, sleep_interval=12, sync_interval=-1, ) assert self.converter.get_sidecar_containers( contexts=contexts, artifacts_store=store, polyaxon_sidecar=polyaxon_sidecar, sidecar_containers=[], ) == [ get_sidecar_container( contexts=contexts, env=self.converter.get_polyaxon_sidecar_service_env_vars(), polyaxon_sidecar=polyaxon_sidecar, artifacts_store=store, run_path=self.converter.run_path, ) ]