def test_get_sidecar_container_with_managed_mount_outputs_logs_store(self): env_vars = [ get_env_var(name="key1", value="value1"), get_env_var(name="key2", value="value2"), ] mount_managed_store = V1ConnectionType( name="test_path", kind=V1ConnectionKind.HOST_PATH, schema=V1HostPathConnection(mount_path="/tmp", host_path="/tmp"), secret=None, ) # logs and outputs sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=mount_managed_store, contexts=PluginsContextsSpec.from_config(None, default_auth=True), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.args == get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212, monitor_logs=False, ) assert (sidecar.env == get_sidecar_env_vars( env_vars=env_vars, container_id=MAIN_JOB_CONTAINER, artifacts_store_name=mount_managed_store.name, ) + get_connection_env_var(connection=mount_managed_store, secret=None)) assert sidecar.env_from == [] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), get_mount_from_store(store=mount_managed_store), ] # logs and no outputs sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=mount_managed_store, contexts=PluginsContextsSpec.from_config( V1Plugins(collect_artifacts=False, auth=True)), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.args == get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212, monitor_logs=False, ) assert (sidecar.env == get_sidecar_env_vars( env_vars=env_vars, container_id=MAIN_JOB_CONTAINER, artifacts_store_name=mount_managed_store.name, ) + get_connection_env_var(connection=mount_managed_store, secret=None)) assert sidecar.env_from == [] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_mount_from_store(store=mount_managed_store), ] # outputs and no logs sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=mount_managed_store, contexts=PluginsContextsSpec.from_config( V1Plugins(collect_logs=False, auth=True)), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.args == get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212, monitor_logs=False, ) assert (sidecar.env == get_sidecar_env_vars( env_vars=env_vars, container_id=MAIN_JOB_CONTAINER, artifacts_store_name=mount_managed_store.name, ) + get_connection_env_var(connection=mount_managed_store, secret=None)) assert sidecar.env_from == [] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), get_mount_from_store(store=mount_managed_store), ]
def test_get_sidecar_container_with_managed_bucket_outputs_logs_store_and_env_from( self, ): env_vars = [ get_env_var(name="key1", value="value1"), get_env_var(name="key2", value="value2"), ] resource1 = V1K8sResourceType(name="test1", schema=V1K8sResourceSchema(name="ref"), is_requested=False) bucket_managed_store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), secret=resource1.schema, ) # both logs and outputs sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=bucket_managed_store, contexts=PluginsContextsSpec.from_config(None, default_auth=True), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.args == get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212, monitor_logs=False, ) assert (sidecar.env == get_sidecar_env_vars( env_vars=env_vars, container_id=MAIN_JOB_CONTAINER, artifacts_store_name=bucket_managed_store.name, ) + get_items_from_secret(secret=resource1) + get_connection_env_var( connection=bucket_managed_store, secret=resource1)) assert sidecar.env_from == [get_env_from_secret(secret=resource1)] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), ] # logs and no outputs sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=bucket_managed_store, contexts=PluginsContextsSpec.from_config( V1Plugins(collect_artifacts=False, auth=True)), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.args == get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212, monitor_logs=False, ) assert (sidecar.env == get_sidecar_env_vars( env_vars=env_vars, container_id=MAIN_JOB_CONTAINER, artifacts_store_name=bucket_managed_store.name, ) + get_items_from_secret(secret=resource1) + get_connection_env_var( connection=bucket_managed_store, secret=resource1)) assert sidecar.env_from == [get_env_from_secret(secret=resource1)] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), ] # outputs and no logs sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=bucket_managed_store, contexts=PluginsContextsSpec.from_config( V1Plugins(collect_logs=False, auth=True)), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.args == get_sidecar_args( container_id=MAIN_JOB_CONTAINER, sleep_interval=213, sync_interval=212, monitor_logs=False, ) assert (sidecar.env == get_sidecar_env_vars( env_vars=env_vars, container_id=MAIN_JOB_CONTAINER, artifacts_store_name=bucket_managed_store.name, ) + get_items_from_secret(secret=resource1) + get_connection_env_var( connection=bucket_managed_store, secret=resource1)) assert sidecar.env_from == [get_env_from_secret(secret=resource1)] assert sidecar.resources == get_sidecar_resources() assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), ]
def test_get_sidecar_container_auth_context(self, ): env_vars = [ get_env_var(name="key1", value="value1"), get_env_var(name="key2", value="value2"), ] resource1 = V1K8sResourceType( name="test1", schema=V1K8sResourceSchema(name="ref1", items=["item1", "item2"]), is_requested=False, ) bucket_managed_store = V1ConnectionType( name="test_gcs", kind=V1ConnectionKind.GCS, schema=V1BucketConnection(bucket="gs//:foo"), secret=resource1.schema, ) # Default auth is included sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=212, ), artifacts_store=bucket_managed_store, contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.volume_mounts == [ get_auth_context_mount(read_only=True), get_artifacts_context_mount(read_only=False), ] # Nno auth sidecar = get_sidecar_container( container_id=MAIN_JOB_CONTAINER, env=env_vars, polyaxon_sidecar=V1PolyaxonSidecarContainer( image="sidecar/sidecar", image_tag="", image_pull_policy="IfNotPresent", sleep_interval=213, sync_interval=-212, ), artifacts_store=bucket_managed_store, contexts=PluginsContextsSpec.from_config(V1Plugins(auth=False)), run_path="test", ) assert sidecar.name == SIDECAR_CONTAINER assert sidecar.image == "sidecar/sidecar" assert sidecar.image_pull_policy == "IfNotPresent" assert sidecar.command == ["polyaxon", "sidecar"] assert sidecar.volume_mounts == [ get_artifacts_context_mount(read_only=False) ]
def test_get_git_init_container(self): connection = V1ConnectionType( name="user/foo", kind=V1ConnectionKind.GIT, schema=V1GitConnection(url="foo.com"), ) container = get_git_init_container( polyaxon_init=V1PolyaxonInitContainer(image="foo", image_tag=""), connection=connection, contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)), ) assert (generate_container_name(INIT_GIT_CONTAINER_PREFIX, connection.name, False) in container.name) assert container.image == "foo" assert container.image_pull_policy is None assert container.command == ["polyaxon", "initializer", "git"] assert container.env == get_connection_env_var(connection=connection, secret=None) assert container.resources == get_init_resources() assert container.volume_mounts == [ get_connections_context_mount( name=constants.CONTEXT_VOLUME_ARTIFACTS, mount_path=CONTEXT_MOUNT_ARTIFACTS, ), get_auth_context_mount(read_only=True), ] container = get_git_init_container( polyaxon_init=V1PolyaxonInitContainer( image="init/init", image_tag="", image_pull_policy="IfNotPresent"), connection=connection, contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)), ) assert (generate_container_name(INIT_GIT_CONTAINER_PREFIX, connection.name, False) in container.name) assert container.image == "init/init" assert container.image_pull_policy == "IfNotPresent" assert container.command == ["polyaxon", "initializer", "git"] assert container.args == [ "--repo-path={}/{}".format(CONTEXT_MOUNT_ARTIFACTS, connection.name), "--url={}".format(connection.schema.url), ] assert container.resources == get_init_resources() assert container.volume_mounts == [ get_connections_context_mount( name=constants.CONTEXT_VOLUME_ARTIFACTS, mount_path=CONTEXT_MOUNT_ARTIFACTS, ), get_auth_context_mount(read_only=True), ] connection = V1ConnectionType( name="user/foo", kind=V1ConnectionKind.GIT, schema=V1GitConnection( url="foo.com", revision="00b9d2ea01c40f58d6b4051319f9375675a43c02"), ) container = get_git_init_container( polyaxon_init=V1PolyaxonInitContainer( image="init/init", image_tag="", image_pull_policy="IfNotPresent"), connection=connection, mount_path="/somepath", contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)), ) assert (generate_container_name(INIT_GIT_CONTAINER_PREFIX, connection.name, False) in container.name) assert container.image == "init/init" assert container.image_pull_policy == "IfNotPresent" assert container.command == ["polyaxon", "initializer", "git"] assert container.args == [ "--repo-path=/somepath/{}".format(connection.name), "--url={}".format(connection.schema.url), "--revision=00b9d2ea01c40f58d6b4051319f9375675a43c02", ] assert container.resources == get_init_resources() assert container.volume_mounts == [ get_connections_context_mount(name=get_volume_name("/somepath"), mount_path="/somepath"), get_auth_context_mount(read_only=True), ] connection = V1ConnectionType( name="user/foo", kind=V1ConnectionKind.GIT, schema=V1GitConnection( url="foo.com", revision="00b9d2ea01c40f58d6b4051319f9375675a43c02", flags=["--falg1", "--flag2=test", "k=v"], ), ) container = get_git_init_container( polyaxon_init=V1PolyaxonInitContainer( image="init/init", image_tag="", image_pull_policy="IfNotPresent"), connection=connection, mount_path="/somepath", contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)), ) assert (generate_container_name(INIT_GIT_CONTAINER_PREFIX, connection.name, False) in container.name) assert container.image == "init/init" assert container.image_pull_policy == "IfNotPresent" assert container.command == ["polyaxon", "initializer", "git"] assert container.args == [ "--repo-path=/somepath/{}".format(connection.name), "--url={}".format(connection.schema.url), "--revision=00b9d2ea01c40f58d6b4051319f9375675a43c02", '--flags=["--falg1","--flag2=test","k=v"]', ] assert container.resources == get_init_resources() assert container.volume_mounts == [ get_connections_context_mount(name=get_volume_name("/somepath"), mount_path="/somepath"), get_auth_context_mount(read_only=True), ]
def test_get_dockerfile_init_container(self): dockerfile_args = V1DockerfileType(image="test/test") container = get_dockerfile_init_container( polyaxon_init=V1PolyaxonInitContainer(image="foo", image_tag=""), dockerfile_args=dockerfile_args, env=None, contexts=PluginsContextsSpec.from_config(V1Plugins()), run_path="test", ) assert INIT_DOCKERFILE_CONTAINER.format("") in container.name assert container.image == "foo" assert container.image_pull_policy is None assert container.command == ["polyaxon", "docker", "generate"] assert container.args == [ "--build-context={}".format(dockerfile_args.to_dict(dump=True)), "--destination={}".format(CONTEXT_MOUNT_ARTIFACTS), "--copy-path={}".format( CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("test")), ] assert container.env == [get_run_instance_env_var()] assert container.resources == get_init_resources() assert container.volume_mounts == [ get_connections_context_mount( name=constants.CONTEXT_VOLUME_ARTIFACTS, mount_path=CONTEXT_MOUNT_ARTIFACTS, ), get_auth_context_mount(read_only=True), ] dockerfile_args = V1DockerfileType( image="test/test", lang_env="LANG", run=["step1", "step2"], env=[["key1", "val1"], ["key2", "val2"]], uid=2222, gid=2222, ) container = get_dockerfile_init_container( polyaxon_init=V1PolyaxonInitContainer( image="init/init", image_tag="", image_pull_policy="IfNotPresent"), env=[], dockerfile_args=dockerfile_args, mount_path="/somepath", contexts=PluginsContextsSpec.from_config(V1Plugins()), run_path="test", ) assert INIT_DOCKERFILE_CONTAINER.format("") in container.name assert container.image == "init/init" assert container.image_pull_policy == "IfNotPresent" assert container.command == ["polyaxon", "docker", "generate"] assert container.args == [ "--build-context={}".format(dockerfile_args.to_dict(dump=True)), "--destination=/somepath", "--copy-path={}".format( CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("test")), ] assert container.env == [get_run_instance_env_var()] assert container.resources == get_init_resources() assert container.volume_mounts == [ get_connections_context_mount(name=get_volume_name("/somepath"), mount_path="/somepath"), get_auth_context_mount(read_only=True), ]
def get_git_init_container( polyaxon_init: V1PolyaxonInitContainer, connection: V1ConnectionType, contexts: PluginsContextsSpec, container: Optional[k8s_schemas.V1Container] = None, env: List[k8s_schemas.V1EnvVar] = None, mount_path: str = None, track: bool = False, ) -> k8s_schemas.V1Container: if not connection: raise PolypodException("A connection is required to create a repo context.") container_name = generate_container_name(INIT_GIT_CONTAINER_PREFIX, connection.name) if not container: container = k8s_schemas.V1Container(name=container_name) volume_name = ( get_volume_name(mount_path) if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS ) mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS volume_mounts = [ get_connections_context_mount(name=volume_name, mount_path=mount_path) ] if contexts and contexts.auth: volume_mounts.append(get_auth_context_mount(read_only=True)) env = to_list(env, check_none=True) env_from = [] secret = connection.get_secret() if secret: volume_mounts += to_list( get_mount_from_resource(resource=secret), check_none=True ) env += to_list(get_items_from_secret(secret=secret), check_none=True) env_from = to_list(get_env_from_secret(secret=secret), check_none=True) env += to_list( get_connection_env_var(connection=connection, secret=secret), check_none=True ) # Add special handling to auto-inject ssh mount path if connection.kind == V1ConnectionKind.SSH and secret.schema.mount_path: env += [get_env_var(POLYAXON_KEYS_SSH_PATH, secret.schema.mount_path)] config_map = connection.get_config_map() if config_map: volume_mounts += to_list( get_mount_from_resource(resource=config_map), check_none=True ) env += to_list( get_items_from_config_map(config_map=config_map), check_none=True ) env_from = to_list( get_env_from_config_map(config_map=config_map), check_none=True ) args = get_repo_context_args( name=connection.name, # Handle the case of custom connection url=getattr(connection.schema, "url", None), revision=getattr(connection.schema, "revision", None), flags=getattr(connection.schema, "flags", None), mount_path=mount_path, connection=connection.name if track else None, ) return patch_container( container=container, name=container_name, image=polyaxon_init.get_image(), image_pull_policy=polyaxon_init.image_pull_policy, command=["polyaxon", "initializer", "git"], args=args, env=env, env_from=env_from, volume_mounts=volume_mounts, resources=polyaxon_init.get_resources(), )