コード例 #1
0
    def test_get_file_init_container(self):
        file_args = V1FileType(content="test")
        container = get_file_init_container(
            polyaxon_init=V1PolyaxonInitContainer(image="foo", image_tag=""),
            contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
            file_args=V1FileType(content="test"),
            run_path="test",
            run_instance="foo.bar.runs.uuid",
        )
        assert INIT_FILE_CONTAINER_PREFIX in container.name
        assert container.image == "foo"
        assert container.image_pull_policy is None
        assert container.command == ["polyaxon", "initializer", "file"]
        assert container.resources == get_init_resources()
        assert container.volume_mounts == [
            get_connections_context_mount(
                name=constants.CONTEXT_VOLUME_ARTIFACTS,
                mount_path=CONTEXT_MOUNT_ARTIFACTS,
            ),
            get_auth_context_mount(read_only=True),
        ]
        assert file_args.to_dict(dump=True) == '{"content":"test"}'
        assert container.args == [
            "--file-context={}".format('{"content":"test","filename":"file"}'),
            "--filepath={}".format(CONTEXT_MOUNT_ARTIFACTS),
            "--copy-path={}".format(
                CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("test")),
            "--track",
        ]

        file_args = V1FileType(filename="test", content="test")
        container = get_file_init_container(
            polyaxon_init=V1PolyaxonInitContainer(
                image="init/init",
                image_tag="",
                image_pull_policy="IfNotPresent"),
            contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
            file_args=file_args,
            run_path="test",
            run_instance="foo.bar.runs.uuid",
        )
        assert INIT_FILE_CONTAINER_PREFIX in container.name
        assert container.image == "init/init"
        assert container.image_pull_policy == "IfNotPresent"
        assert container.command == ["polyaxon", "initializer", "file"]
        assert container.args == [
            "--file-context={}".format(file_args.to_dict(dump=True)),
            "--filepath={}".format(CONTEXT_MOUNT_ARTIFACTS),
            "--copy-path={}".format(
                CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("test")),
            "--track",
        ]
        assert container.resources == get_init_resources()
        assert container.volume_mounts == [
            get_connections_context_mount(
                name=constants.CONTEXT_VOLUME_ARTIFACTS,
                mount_path=CONTEXT_MOUNT_ARTIFACTS,
            ),
            get_auth_context_mount(read_only=True),
        ]
コード例 #2
0
    def test_single_connections(self):
        self.assert_single_store(store=self.s3_store, results=[])
        self.assert_single_store(store=self.gcs_store, results=[])
        self.assert_single_store(store=self.az_store, results=[])
        self.assert_single_store(
            store=self.claim_store,
            results=[get_mount_from_store(store=self.claim_store)],
        )
        self.assert_single_store(
            store=self.host_path_store,
            results=[get_mount_from_store(store=self.host_path_store)],
        )

        # Managed versions
        volume_name = get_volume_name("/test")
        self.assert_single_init_store(
            store=self.s3_store,
            results=[
                get_connections_context_mount(name=volume_name,
                                              mount_path="/test")
            ],
        )
        self.assert_single_init_store(
            store=self.gcs_store,
            results=[
                get_connections_context_mount(name=volume_name,
                                              mount_path="/test")
            ],
        )
        self.assert_single_init_store(
            store=self.az_store,
            results=[
                get_connections_context_mount(name=volume_name,
                                              mount_path="/test")
            ],
        )
        self.assert_single_init_store(
            store=self.claim_store,
            results=[
                get_connections_context_mount(name=volume_name,
                                              mount_path="/test")
            ],
        )
        self.assert_single_init_store(
            store=self.host_path_store,
            results=[
                get_connections_context_mount(name=volume_name,
                                              mount_path="/test")
            ],
        )
コード例 #3
0
ファイル: store.py プロジェクト: vamsikavuru/polyaxon
def get_store_container(
    polyaxon_init: V1PolyaxonInitContainer,
    connection: V1ConnectionType,
    artifacts: V1ArtifactsType,
    container: Optional[k8s_schemas.V1Container] = None,
    env: List[k8s_schemas.V1EnvVar] = None,
    mount_path: str = None,
) -> Optional[k8s_schemas.V1Container]:
    container_name = INIT_ARTIFACTS_CONTAINER.format(connection.name)
    if not container:
        container = k8s_schemas.V1Container(name=container_name)

    volume_name = (get_volume_name(mount_path)
                   if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
    mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(
        connection.name)
    volume_mounts = [
        get_connections_context_mount(name=volume_name, mount_path=mount_path)
    ]

    return get_base_store_container(
        container=container,
        container_name=container_name,
        polyaxon_init=polyaxon_init,
        store=connection,
        env=env,
        env_from=[],
        volume_mounts=volume_mounts,
        args=[
            get_volume_args(store=connection,
                            mount_path=mount_path,
                            artifacts=artifacts)
        ],
    )
コード例 #4
0
def get_dockerfile_init_container(
    polyaxon_init: V1PolyaxonInitContainer,
    dockerfile_args: V1DockerfileType,
    contexts: PluginsContextsSpec,
    run_path: str,
    env: List[k8s_schemas.V1EnvVar] = None,
    mount_path: Optional[str] = None,
) -> k8s_schemas.V1Container:
    env = to_list(env, check_none=True)
    env = env + [get_run_instance_env_var()]

    volume_name = (get_volume_name(mount_path)
                   if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
    mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS
    volume_mounts = [
        get_connections_context_mount(name=volume_name, mount_path=mount_path)
    ]
    if contexts and contexts.auth:
        volume_mounts.append(get_auth_context_mount(read_only=True))

    return k8s_schemas.V1Container(
        name=generate_container_name(INIT_DOCKERFILE_CONTAINER_PREFIX),
        image=polyaxon_init.get_image(),
        image_pull_policy=polyaxon_init.image_pull_policy,
        command=["polyaxon", "docker", "generate"],
        args=[
            "--build-context={}".format(dockerfile_args.to_dict(dump=True)),
            "--destination={}".format(mount_path),
            "--copy-path={}".format(
                CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_path) + "/outputs"),
        ],
        env=env,
        resources=polyaxon_init.get_resources(),
        volume_mounts=volume_mounts,
    )
コード例 #5
0
 def test_get_store_container_mount_stores(self):
     # Managed store
     store = V1ConnectionType(
         name="test_claim",
         kind=V1ConnectionKind.VOLUME_CLAIM,
         schema=V1ClaimConnection(
             mount_path="/tmp", volume_claim="test", read_only=True
         ),
     )
     container = get_store_container(
         polyaxon_init=V1PolyaxonInitContainer(
             image="foo/foo", image_tag="foo", image_pull_policy="IfNotPresent"
         ),
         connection=store,
         artifacts=None,
     )
     mount_path = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name)
     assert container.name == INIT_ARTIFACTS_CONTAINER.format(store.name)
     assert container.image == "foo/foo:foo"
     assert container.image_pull_policy == "IfNotPresent"
     assert container.command == ["/bin/sh", "-c"]
     assert container.args == [
         get_volume_args(store=store, mount_path=mount_path, artifacts=None)
     ]
     assert container.env == get_connection_env_var(connection=store, secret=None)
     assert container.env_from == []
     assert container.resources is not None
     assert container.volume_mounts == [
         get_connections_context_mount(
             name=constants.CONTEXT_VOLUME_ARTIFACTS,
             mount_path=CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name),
         ),
         get_mount_from_store(store=store),
     ]
コード例 #6
0
ファイル: custom.py プロジェクト: vishalbelsare/polyaxon
def get_custom_init_container(
    connection: V1ConnectionType,
    contexts: PluginsContextsSpec,
    container: Optional[k8s_schemas.V1Container],
    env: List[k8s_schemas.V1EnvVar] = None,
    mount_path: str = None,
) -> k8s_schemas.V1Container:
    if not connection:
        raise PolypodException(
            "A connection is required to create a repo context.")

    volume_name = (get_volume_name(mount_path)
                   if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
    mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS
    volume_mounts = [
        get_connections_context_mount(name=volume_name, mount_path=mount_path)
    ]

    if contexts and contexts.auth:
        volume_mounts.append(get_auth_context_mount(read_only=True))

    env = to_list(env, check_none=True)
    env_from = []
    secret = connection.get_secret()
    if secret:
        volume_mounts += to_list(get_mount_from_resource(resource=secret),
                                 check_none=True)
        env += to_list(get_items_from_secret(secret=secret), check_none=True)
        env_from = to_list(get_env_from_secret(secret=secret), check_none=True)
    env += to_list(get_connection_env_var(connection=connection,
                                          secret=secret),
                   check_none=True)
    config_map = connection.get_config_map()
    if config_map:
        volume_mounts += to_list(get_mount_from_resource(resource=config_map),
                                 check_none=True)
        env += to_list(get_items_from_config_map(config_map=config_map),
                       check_none=True)
        env_from = to_list(get_env_from_config_map(config_map=config_map),
                           check_none=True)
    container_name = container.name or generate_container_name(
        INIT_CUSTOM_CONTAINER_PREFIX, connection.name)
    return patch_container(
        container=container,
        name=container_name,
        env=env,
        env_from=env_from,
        volume_mounts=volume_mounts,
    )
コード例 #7
0
def get_volume_mounts(
    contexts: PluginsContextsSpec,
    init: Optional[List[V1Init]],
    connections: Iterable[V1ConnectionType],
    secrets: Iterable[V1K8sResourceType],
    config_maps: Iterable[V1K8sResourceType] = None,
) -> List[k8s_schemas.V1VolumeMount]:
    init = init or []
    connections = connections or []
    secrets = secrets or []
    config_maps = config_maps or []

    volume_mounts = []
    volume_names = set()
    if contexts and contexts.collect_artifacts:
        volume_mounts += to_list(
            get_artifacts_context_mount(read_only=False), check_none=True
        )
        volume_names.add(constants.CONTEXT_VOLUME_ARTIFACTS)
    for init_connection in init:
        volume_name = (
            get_volume_name(init_connection.path)
            if init_connection.path
            else constants.CONTEXT_VOLUME_ARTIFACTS
        )
        mount_path = init_connection.path or CONTEXT_MOUNT_ARTIFACTS
        if volume_name in volume_names:
            continue
        volume_names.add(volume_name)
        volume_mounts += to_list(
            get_connections_context_mount(name=volume_name, mount_path=mount_path),
            check_none=True,
        )
    for store in connections:
        volume_mounts += to_list(get_mount_from_store(store=store), check_none=True)

    for secret in secrets:
        volume_mounts += to_list(
            get_mount_from_resource(resource=secret), check_none=True
        )

    for config_map in config_maps:
        volume_mounts += to_list(
            get_mount_from_resource(resource=config_map), check_none=True
        )

    return volume_mounts
コード例 #8
0
ファイル: test_init_store.py プロジェクト: zeyaddeeb/polyaxon
 def test_get_store_container_bucket_stores(self):
     mount_path = "/test-path"
     resource1 = V1K8sResourceType(
         name="non_mount_test1",
         schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]),
         is_requested=False,
     )
     store = V1ConnectionType(
         name="test_gcs",
         kind=V1ConnectionKind.GCS,
         schema=V1BucketConnection(bucket="gs//:foo"),
         secret=resource1.schema,
     )
     container = get_store_container(
         polyaxon_init=V1PolyaxonInitContainer(
             image="foo/foo", image_tag="", image_pull_policy="IfNotPresent"
         ),
         connection=store,
         artifacts=None,
         mount_path=mount_path,
     )
     assert container.name == generate_container_name(
         INIT_ARTIFACTS_CONTAINER_PREFIX, store.name
     )
     assert container.image == "foo/foo"
     assert container.image_pull_policy == "IfNotPresent"
     assert container.command == ["/bin/sh", "-c"]
     assert container.args == [
         get_volume_args(store=store, mount_path=mount_path, artifacts=None)
     ]
     assert container.env is not None
     assert container.env_from == []
     assert container.resources == get_init_resources()
     assert container.volume_mounts == [
         get_connections_context_mount(
             name=get_volume_name(mount_path), mount_path=mount_path
         )
     ]
コード例 #9
0
ファイル: file.py プロジェクト: vishalbelsare/polyaxon
def get_file_init_container(
    polyaxon_init: V1PolyaxonInitContainer,
    file_args: V1FileType,
    contexts: PluginsContextsSpec,
    run_path: str,
    run_instance: str,
    env: List[k8s_schemas.V1EnvVar] = None,
    mount_path: Optional[str] = None,
) -> k8s_schemas.V1Container:
    env = to_list(env, check_none=True)
    env = env + [get_run_instance_env_var(run_instance)]

    volume_name = (get_volume_name(mount_path)
                   if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
    mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS
    volume_mounts = [
        get_connections_context_mount(name=volume_name, mount_path=mount_path)
    ]
    if contexts and contexts.auth:
        volume_mounts.append(get_auth_context_mount(read_only=True))

    file_args.filename = file_args.filename or "file"
    return k8s_schemas.V1Container(
        name=generate_container_name(INIT_FILE_CONTAINER_PREFIX),
        image=polyaxon_init.get_image(),
        image_pull_policy=polyaxon_init.image_pull_policy,
        command=["polyaxon", "initializer", "file"],
        args=[
            "--file-context={}".format(file_args.to_dict(dump=True)),
            "--filepath={}".format(mount_path),
            "--copy-path={}".format(
                CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format(run_path)),
            "--track",
        ],
        env=env,
        resources=polyaxon_init.get_resources(),
        volume_mounts=volume_mounts,
    )
コード例 #10
0
 def test_get_connections_context_mount(self):
     mount = get_connections_context_mount(name="test", mount_path="/test")
     assert mount.name == "test"
     assert mount.mount_path == "/test"
     assert mount.read_only is None
コード例 #11
0
def get_git_init_container(
    polyaxon_init: V1PolyaxonInitContainer,
    connection: V1ConnectionType,
    contexts: PluginsContextsSpec,
    container: Optional[k8s_schemas.V1Container] = None,
    env: List[k8s_schemas.V1EnvVar] = None,
    mount_path: str = None,
    track: bool = False,
) -> k8s_schemas.V1Container:
    if not connection:
        raise PolypodException(
            "A connection is required to create a repo context.")
    if not container:
        container = k8s_schemas.V1Container(name=generate_container_name(
            INIT_GIT_CONTAINER_PREFIX, connection.name), )

    volume_name = (get_volume_name(mount_path)
                   if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
    mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS
    volume_mounts = [
        get_connections_context_mount(name=volume_name, mount_path=mount_path)
    ]

    if contexts and contexts.auth:
        volume_mounts.append(get_auth_context_mount(read_only=True))

    env = to_list(env, check_none=True)
    env_from = []
    secret = connection.get_secret()
    if secret:
        volume_mounts += to_list(get_mount_from_resource(resource=secret),
                                 check_none=True)
        env += to_list(get_items_from_secret(secret=secret), check_none=True)
        env_from = to_list(get_env_from_secret(secret=secret), check_none=True)
    env += to_list(get_connection_env_var(connection=connection,
                                          secret=secret),
                   check_none=True)
    config_map = connection.get_config_map()
    if config_map:
        volume_mounts += to_list(get_mount_from_resource(resource=config_map),
                                 check_none=True)
        env += to_list(get_items_from_config_map(config_map=config_map),
                       check_none=True)
        env_from = to_list(get_env_from_config_map(config_map=config_map),
                           check_none=True)
    args = get_repo_context_args(
        name=connection.name,
        url=connection.schema.url,
        revision=connection.schema.revision,
        mount_path=mount_path,
        connection=connection.name if track else None,
    )
    return patch_container(
        container=container,
        name=generate_container_name(INIT_GIT_CONTAINER_PREFIX,
                                     connection.name),
        image=polyaxon_init.get_image(),
        image_pull_policy=polyaxon_init.image_pull_policy,
        command=["polyaxon", "initializer", "git"],
        args=args,
        env=env,
        env_from=env_from,
        volume_mounts=volume_mounts,
        resources=polyaxon_init.get_resources(),
    )
コード例 #12
0
ファイル: test_init_git.py プロジェクト: yangjun1994/polyaxon
    def test_get_git_init_container(self):
        connection = V1ConnectionType(
            name="user/foo",
            kind=V1ConnectionKind.GIT,
            schema=V1GitConnection(url="foo.com"),
        )
        container = get_git_init_container(
            polyaxon_init=V1PolyaxonInitContainer(image="foo", image_tag=""),
            connection=connection,
            contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
        )
        assert container.name == INIT_GIT_CONTAINER.format(connection.name)
        assert container.image == "foo"
        assert container.image_pull_policy is None
        assert container.command == ["polyaxon", "initializer", "git"]
        assert container.env == get_connection_env_var(connection=connection,
                                                       secret=None)
        assert container.resources == get_init_resources()
        assert container.volume_mounts == [
            get_connections_context_mount(
                name=constants.CONTEXT_VOLUME_ARTIFACTS,
                mount_path=CONTEXT_MOUNT_ARTIFACTS,
            ),
            get_auth_context_mount(read_only=True),
        ]

        container = get_git_init_container(
            polyaxon_init=V1PolyaxonInitContainer(
                image="init/init",
                image_tag="",
                image_pull_policy="IfNotPresent"),
            connection=connection,
            contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
        )
        assert container.name == INIT_GIT_CONTAINER.format(connection.name)
        assert container.image == "init/init"
        assert container.image_pull_policy == "IfNotPresent"
        assert container.command == ["polyaxon", "initializer", "git"]
        assert container.args == [
            "--repo_path={}/{}".format(CONTEXT_MOUNT_ARTIFACTS,
                                       connection.name),
            "--url={}".format(connection.schema.url),
        ]
        assert container.resources == get_init_resources()
        assert container.volume_mounts == [
            get_connections_context_mount(
                name=constants.CONTEXT_VOLUME_ARTIFACTS,
                mount_path=CONTEXT_MOUNT_ARTIFACTS,
            ),
            get_auth_context_mount(read_only=True),
        ]

        connection = V1ConnectionType(
            name="user/foo",
            kind=V1ConnectionKind.GIT,
            schema=V1GitConnection(
                url="foo.com",
                revision="00b9d2ea01c40f58d6b4051319f9375675a43c02"),
        )
        container = get_git_init_container(
            polyaxon_init=V1PolyaxonInitContainer(
                image="init/init",
                image_tag="",
                image_pull_policy="IfNotPresent"),
            connection=connection,
            mount_path="/somepath",
            contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
        )
        assert container.name == INIT_GIT_CONTAINER.format(connection.name)
        assert container.image == "init/init"
        assert container.image_pull_policy == "IfNotPresent"
        assert container.command == ["polyaxon", "initializer", "git"]
        assert container.args == [
            "--repo_path=/somepath/{}".format(connection.name),
            "--url={}".format(connection.schema.url),
            "--revision=00b9d2ea01c40f58d6b4051319f9375675a43c02",
        ]
        assert container.resources == get_init_resources()
        assert container.volume_mounts == [
            get_connections_context_mount(name=get_volume_name("/somepath"),
                                          mount_path="/somepath"),
            get_auth_context_mount(read_only=True),
        ]
コード例 #13
0
    def test_get_dockerfile_init_container(self):
        dockerfile_args = V1DockerfileType(image="test/test")
        container = get_dockerfile_init_container(
            polyaxon_init=V1PolyaxonInitContainer(image="foo", image_tag=""),
            dockerfile_args=dockerfile_args,
            env=None,
            contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
            run_path="test",
            run_instance="foo.bar.runs.uuid",
        )
        assert INIT_DOCKERFILE_CONTAINER_PREFIX in container.name
        assert container.image == "foo"
        assert container.image_pull_policy is None
        assert container.command == ["polyaxon", "docker", "generate"]
        assert container.args == [
            "--build-context={}".format(dockerfile_args.to_dict(dump=True)),
            "--destination={}".format(CONTEXT_MOUNT_ARTIFACTS),
            "--copy-path={}".format(
                CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("test")),
            "--track",
        ]
        assert container.env == [
            get_run_instance_env_var(run_instance="foo.bar.runs.uuid")
        ]
        assert container.resources == get_init_resources()
        assert container.volume_mounts == [
            get_connections_context_mount(
                name=constants.CONTEXT_VOLUME_ARTIFACTS,
                mount_path=CONTEXT_MOUNT_ARTIFACTS,
            ),
            get_auth_context_mount(read_only=True),
        ]

        dockerfile_args = V1DockerfileType(
            image="test/test",
            lang_env="LANG",
            run=["step1", "step2"],
            env=[["key1", "val1"], ["key2", "val2"]],
            uid=2222,
            gid=2222,
        )
        container = get_dockerfile_init_container(
            polyaxon_init=V1PolyaxonInitContainer(
                image="init/init",
                image_tag="",
                image_pull_policy="IfNotPresent"),
            env=[],
            dockerfile_args=dockerfile_args,
            mount_path="/somepath",
            contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
            run_path="test",
            run_instance="foo.bar.runs.uuid",
        )
        assert INIT_DOCKERFILE_CONTAINER_PREFIX in container.name
        assert container.image == "init/init"
        assert container.image_pull_policy == "IfNotPresent"
        assert container.command == ["polyaxon", "docker", "generate"]
        assert container.args == [
            "--build-context={}".format(dockerfile_args.to_dict(dump=True)),
            "--destination=/somepath",
            "--copy-path={}".format(
                CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("test")),
            "--track",
        ]
        assert container.env == [
            get_run_instance_env_var(run_instance="foo.bar.runs.uuid")
        ]
        assert container.resources == get_init_resources()
        assert container.volume_mounts == [
            get_connections_context_mount(name=get_volume_name("/somepath"),
                                          mount_path="/somepath"),
            get_auth_context_mount(read_only=True),
        ]