def test_host_path_connect_config(self): config_dict = {} with self.assertRaises(ValidationError): V1HostPathConnection.from_dict(config_dict) with self.assertRaises(ValidationError): validate_connection(V1ConnectionKind.HOST_PATH, config_dict) config_dict = {"host_path": "foo"} with self.assertRaises(ValidationError): V1HostPathConnection.from_dict(config_dict) with self.assertRaises(ValidationError): validate_connection(V1ConnectionKind.HOST_PATH, config_dict) config_dict = {"hostPath": "foo", "mountPath": "foo", "readOnly": True} config = V1HostPathConnection.from_dict(config_dict) assert config.to_dict() == config_dict with self.assertRaises(ValidationError): validate_connection(V1ConnectionKind.S3, config_dict) with self.assertRaises(ValidationError): validate_connection(V1ConnectionKind.WASB, config_dict) validate_connection(V1ConnectionKind.HOST_PATH, config_dict)
def test_get_mount_from_store(self): # Non bucket stores assert get_mount_from_store(store=None) is None store = V1ConnectionType( name="test", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3//:foo"), ) assert get_mount_from_store(store=store) is None # Claim store store = V1ConnectionType( name="test", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(mount_path="/tmp", volume_claim="test", read_only=True), ) mount = get_mount_from_store(store=store) assert mount.name == store.name assert mount.mount_path == store.schema.mount_path assert mount.read_only == store.schema.read_only # Host path store = V1ConnectionType( name="test", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection(mount_path="/tmp", host_path="/tmp", read_only=True), ) mount = get_mount_from_store(store=store) assert mount.name == store.name assert mount.mount_path == store.schema.mount_path assert mount.read_only == store.schema.read_only
def setUp(self): super().setUp() self.s3_store = V1ConnectionType( name="test", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3//:foo"), ) self.gcs_store = V1ConnectionType( name="test", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), ) self.az_store = V1ConnectionType( name="test", kind=V1ConnectionKind.WASB, schema=V1BucketConnection( bucket="Conwasb://[email protected]"), ) self.claim_store = V1ConnectionType( name="test", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(volume_claim="test", mount_path="/tmp", read_only=True), ) self.host_path_store = V1ConnectionType( name="test", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection(host_path="/tmp", mount_path="/tmp", read_only=True), )
def test_get_volume_args_host(self): host_path_store = V1ConnectionType( name="test_path", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection( mount_path="/tmp", host_path="/tmp", read_only=True ), ) path_to = "/path/to/" path_from = os.path.join(host_path_store.store_path, "") assert get_volume_args(host_path_store, path_to, None) == "; ".join( [ get_or_create_args(path=path_to), cp_copy_args(path_from=path_from, path_to=path_to, is_file=False), ] ) host_path_store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection( mount_path="/tmp", host_path="/tmp", read_only=True ), ) base_path = "/path/to/" path_to1 = "/path/to/path1" path_to2 = "/path/to/path2" path_from1 = os.path.join(host_path_store.store_path, "path1") path_from2 = os.path.join(host_path_store.store_path, "path2") assert get_volume_args( host_path_store, "/path/to", artifacts=V1ArtifactsType(dirs=["path1", "path2"]), ) == "; ".join( [ get_or_create_args(path=base_path), cp_copy_args(path_from=path_from1, path_to=path_to1, is_file=False), get_or_create_args(path=base_path), cp_copy_args(path_from=path_from2, path_to=path_to2, is_file=False), ] )
def test_get_main_container_host_paths(self): contexts = PluginsContextsSpec( auth=True, docker=False, shm=False, collect_logs=True, collect_artifacts=True, collect_resources=True, auto_resume=True, sync_statuses=True, external_host=False, sidecar=None, ) volume_mounts = get_mounts( use_auth_context=contexts.auth, use_artifacts_context=False, use_docker_context=contexts.docker, use_shm_context=contexts.shm, ) artifacts_store = V1ConnectionType( name="plx-outputs", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection(mount_path="/tmp/plx/outputs", host_path="/tmp/plx/outputs"), ) container = get_main_container( container_id="test", main_container=k8s_schemas.V1Container(name="main"), contexts=PluginsContextsSpec.from_config( V1Plugins(collect_artifacts=True, collect_logs=True)), volume_mounts=volume_mounts, log_level=None, artifacts_store=artifacts_store, init=[], connections=[], connection_by_names={artifacts_store.name: artifacts_store}, secrets=[], config_maps=[], kv_env_vars=None, env=None, ports=None, run_path="run_path", ) assert container.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), ]
def set_store(): store_root = tempfile.mkdtemp() settings.AGENT_CONFIG = AgentConfig( artifacts_store=V1ConnectionType( name="test", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection(host_path=store_root, mount_path=store_root), secret=None, ), connections=[], ) settings.CLIENT_CONFIG.archive_root = tempfile.mkdtemp() return store_root
def setUp(self): super().setUp() self.s3_store = V1ConnectionType( name="test", kind=V1ConnectionKind.S3, tags=["test", "foo"], schema=V1BucketConnection(bucket="s3//:foo"), ) self.gcs_store = V1ConnectionType( name="test", kind=V1ConnectionKind.GCS, tags=["test"], schema=V1BucketConnection(bucket="gs//:foo"), ) self.az_store = V1ConnectionType( name="test", kind=V1ConnectionKind.WASB, schema=V1BucketConnection( bucket="Conwasb://[email protected]"), ) self.claim_store = V1ConnectionType( name="test", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(volume_claim="test", mount_path="/tmp", read_only=True), ) self.host_path_store = V1ConnectionType( name="test", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection(host_path="/tmp", mount_path="/tmp", read_only=True), ) self.custom_connection1 = V1ConnectionType( name="db", kind=V1ConnectionKind.POSTGRES, ) self.custom_connection2 = V1ConnectionType( name="ssh", kind=V1ConnectionKind.SSH, schema=V1CustomConnection(key1="val1", key2="val2"), )
def test_get_sidecar_container_host_paths(self): artifacts_store = V1ConnectionType( name="plx-outputs", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection(mount_path="/tmp/plx/outputs", host_path="/tmp/plx/outputs"), ) contexts = PluginsContextsSpec( auth=True, docker=False, shm=False, collect_logs=True, collect_artifacts=True, collect_resources=True, auto_resume=True, sync_statuses=True, external_host=False, sidecar=None, ) sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="foo", image_pull_policy="sdf", sleep_interval=2, sync_interval=212, ), env=[], artifacts_store=artifacts_store, contexts=contexts, run_path="test", ) assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), get_mount_from_store(store=artifacts_store), ]
def test_get_volume_from_connection(self): # No store assert get_volume_from_connection(connection=None) is None # Bucket store store = V1ConnectionType( name="test", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3//:foo"), ) assert get_volume_from_connection(connection=store) is None # Claim store store = V1ConnectionType( name="test", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection( mount_path="/tmp", volume_claim="test", read_only=True ), ) volume = get_volume_from_connection(connection=store) assert volume.name == store.name assert volume.persistent_volume_claim.claim_name == store.schema.volume_claim assert volume.persistent_volume_claim.read_only == store.schema.read_only # Host path store = V1ConnectionType( name="test", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection( mount_path="/tmp", host_path="/tmp", read_only=True ), ) volume = get_volume_from_connection(connection=store) assert volume.name == store.name assert volume.host_path == k8s_schemas.V1HostPathVolumeSource( path=store.schema.host_path )
def setUp(self): super().setUp() # Secrets and config maps self.non_mount_resource1 = V1K8sResourceType( name="non_mount_test1", schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]), is_requested=False, ) self.non_mount_resource2 = V1K8sResourceType( name="non_mount_test2", schema=V1K8sResourceSchema(name="ref"), is_requested=False, ) self.mount_resource1 = V1K8sResourceType( name="mount_test1", schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"], mount_path="/tmp1"), is_requested=False, ) self.mount_resource2 = V1K8sResourceType( name="mount_test2", schema=V1K8sResourceSchema(name="ref", mount_path="/tmp2"), is_requested=False, ) # Volumes self.vol1 = get_volume(volume="vol1", claim_name="claim1") self.vol2 = get_volume(volume="vol2", host_path="/path2") self.vol3 = get_volume(volume="vol3") # connections self.s3_store = V1ConnectionType( name="test_s3", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3//:foo"), secret=self.mount_resource1.schema, ) self.gcs_store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), secret=self.mount_resource1.schema, ) self.az_store = V1ConnectionType( name="test_az", kind=V1ConnectionKind.WASB, schema=V1BucketConnection( bucket="wasb://[email protected]"), secret=self.mount_resource1.schema, ) self.claim_store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(mount_path="/tmp", volume_claim="test", read_only=True), ) self.host_path_store = V1ConnectionType( name="test_path", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection(mount_path="/tmp", host_path="/tmp", read_only=True), )
def test_get_sidecar_container_with_managed_mount_outputs_logs_store(self): env_vars = [ get_env_var(name="key1", value="value1"), get_env_var(name="key2", value="value2"), ] mount_managed_store = V1ConnectionType( name="test_path", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection(mount_path="/tmp", host_path="/tmp"), secret=None, ) # logs and outputs sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=mount_managed_store, contexts=PluginsContextsSpec.from_config(None, default_auth=True), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.args == get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212, monitor_logs=False, ) assert sidecar.env == get_sidecar_env_vars( env_vars=env_vars, container_id=MAIN_JOB_CONTAINER, artifacts_store_name=mount_managed_store.name, ) + get_connection_env_var(connection=mount_managed_store, secret=None) assert sidecar.env_from == [] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), get_mount_from_store(store=mount_managed_store), ] # logs and no outputs sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=mount_managed_store, contexts=PluginsContextsSpec.from_config( V1Plugins(collect_artifacts=False, auth=True)), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.args == get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212, monitor_logs=False, ) assert sidecar.env == get_sidecar_env_vars( env_vars=env_vars, container_id=MAIN_JOB_CONTAINER, artifacts_store_name=mount_managed_store.name, ) + get_connection_env_var(connection=mount_managed_store, secret=None) assert sidecar.env_from == [] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_mount_from_store(store=mount_managed_store), ] # outputs and no logs sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=mount_managed_store, contexts=PluginsContextsSpec.from_config( V1Plugins(collect_logs=False, auth=True)), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.args == get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212, monitor_logs=False, ) assert sidecar.env == get_sidecar_env_vars( env_vars=env_vars, container_id=MAIN_JOB_CONTAINER, artifacts_store_name=mount_managed_store.name, ) + get_connection_env_var(connection=mount_managed_store, secret=None) assert sidecar.env_from == [] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), get_mount_from_store(store=mount_managed_store), ]
def setUp(self): super().setUp() # Secrets self.resource1 = V1K8sResourceType( name="non_mount_test1", schema=V1K8sResourceSchema(name="non_mount_test1", items=["item1", "item2"]), is_requested=False, ) self.resource2 = V1K8sResourceType( name="non_mount_test2", schema=V1K8sResourceSchema(name="non_mount_test2"), is_requested=False, ) self.resource3 = V1K8sResourceType( name="non_mount_test3", schema=V1K8sResourceSchema(name="non_mount_test3", items=["item1", "item2"]), is_requested=True, ) self.resource4 = V1K8sResourceType( name="non_mount_test4", schema=V1K8sResourceSchema(name="non_mount_test4"), is_requested=True, ) self.resource5 = V1K8sResourceType( name="non_mount_test1", schema=V1K8sResourceSchema(name="non_mount_test1"), is_requested=True, ) # Connections self.s3_store = V1ConnectionType( name="test_s3", kind=V1ConnectionKind.S3, schema=V1BucketConnection(bucket="s3//:foo"), secret=self.resource1.schema, ) self.gcs_store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gcs//:foo"), secret=self.resource2.schema, ) self.az_store = V1ConnectionType( name="test_az", kind=V1ConnectionKind.WASB, schema=V1BucketConnection( bucket="wasb://[email protected]"), secret=self.resource3.schema, ) self.claim_store = V1ConnectionType( name="test_claim", kind=V1ConnectionKind.VOLUME_CLAIM, schema=V1ClaimConnection(mount_path="/tmp", volume_claim="test"), ) self.host_path_store = V1ConnectionType( name="test_path", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection(mount_path="/tmp", host_path="/tmp", read_only=True), )
def test_get_sidecar_container_override_sync(self): artifacts_store = V1ConnectionType( name="plx-outputs", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection(mount_path="/tmp/plx/outputs", host_path="/tmp/plx/outputs"), ) contexts = PluginsContextsSpec( auth=True, docker=False, shm=False, collect_logs=True, collect_artifacts=True, collect_resources=True, auto_resume=True, sync_statuses=True, external_host=False, sidecar=None, ) sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="foo", image_pull_policy="sdf", sleep_interval=2, sync_interval=212, ), env=[], artifacts_store=artifacts_store, contexts=contexts, run_path="test", ) assert sidecar.args == [ "--container-id=polyaxon-main", "--sleep-interval=2", "--sync-interval=212", ] contexts = PluginsContextsSpec( auth=True, docker=False, shm=False, collect_logs=True, collect_artifacts=True, collect_resources=True, auto_resume=True, sync_statuses=True, external_host=False, sidecar=V1PolyaxonSidecarContainer( sleep_interval=-1, sync_interval=-1, ), ) sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="foo", image_pull_policy="sdf", sleep_interval=2, sync_interval=212, ), env=[], artifacts_store=artifacts_store, contexts=contexts, run_path="test", ) assert sidecar.args == [ "--container-id=polyaxon-main", "--sleep-interval=-1", "--sync-interval=-1", ]