def test_get_artifacts_path_container_with_non_managed_mount_store(self):
store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/claim/path",
volume_claim="claim"),
)
container = get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"),
artifacts_store=store,
run_path="run_uid",
auto_resume=True,
)
init_args = init_artifact_context_args("run_uid")
init_args.append(
get_volume_args(
store=store,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
artifacts=V1ArtifactsType(dirs=["run_uid"]),
))
assert container == get_base_store_container(
container=k8s_schemas.V1Container(name="init"),
container_name=generate_container_name(
INIT_ARTIFACTS_CONTAINER_PREFIX, "default", False),
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"),
store=store,
env=[],
env_from=[],
volume_mounts=[get_artifacts_context_mount()],
args=[" ".join(init_args)],
)
container = get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"),
artifacts_store=store,
run_path="run_uid",
auto_resume=False,
)
init_args = init_artifact_context_args("run_uid")
assert container == get_base_store_container(
container=k8s_schemas.V1Container(name="init"),
container_name=generate_container_name(
INIT_ARTIFACTS_CONTAINER_PREFIX, "default", False),
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"),
store=store,
env=[],
env_from=[],
volume_mounts=[get_artifacts_context_mount()],
args=[" ".join(init_args)],
)
def get_artifacts_path_container(
polyaxon_init: V1PolyaxonInitContainer,
artifacts_store: V1ConnectionType,
run_path: str,
auto_resume: bool,
) -> Optional[k8s_schemas.V1Container]:
if not artifacts_store:
raise PolypodException("Init artifacts container requires a store.")
init_args = init_artifact_context_args(run_path=run_path)
if auto_resume:
init_args.append(
get_volume_args(
store=artifacts_store,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
artifacts=V1ArtifactsType(dirs=[run_path]),
))
container_name = generate_container_name(INIT_ARTIFACTS_CONTAINER_PREFIX,
DEFAULT)
container = k8s_schemas.V1Container(name=container_name)
return get_base_store_container(
container_name=container_name,
container=container,
polyaxon_init=polyaxon_init,
store=artifacts_store,
env=[],
env_from=[],
volume_mounts=[get_artifacts_context_mount()],
# If we are dealing with a volume we need to make sure the path exists for the user
# We also clean the path if this is not a resume run
args=[" ".join(init_args)],
)
def get_store_container(
polyaxon_init: V1PolyaxonInitContainer,
connection: V1ConnectionType,
artifacts: V1ArtifactsType,
container: Optional[k8s_schemas.V1Container] = None,
env: List[k8s_schemas.V1EnvVar] = None,
mount_path: str = None,
) -> Optional[k8s_schemas.V1Container]:
container_name = generate_container_name(INIT_ARTIFACTS_CONTAINER_PREFIX,
connection.name)
if not container:
container = k8s_schemas.V1Container(name=container_name)
volume_name = (get_volume_name(mount_path)
if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(
connection.name)
volume_mounts = [
get_connections_context_mount(name=volume_name, mount_path=mount_path)
]
return get_base_store_container(
container=container,
container_name=container_name,
polyaxon_init=polyaxon_init,
store=connection,
env=env,
env_from=[],
volume_mounts=volume_mounts,
args=[
get_volume_args(store=connection,
mount_path=mount_path,
artifacts=artifacts)
],
)
def get_dockerfile_init_container(
polyaxon_init: V1PolyaxonInitContainer,
dockerfile_args: V1DockerfileType,
contexts: PluginsContextsSpec,
run_path: str,
env: List[k8s_schemas.V1EnvVar] = None,
mount_path: Optional[str] = None,
) -> k8s_schemas.V1Container:
env = to_list(env, check_none=True)
env = env + [get_run_instance_env_var()]
volume_name = (get_volume_name(mount_path)
if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS
volume_mounts = [
get_connections_context_mount(name=volume_name, mount_path=mount_path)
]
if contexts and contexts.auth:
volume_mounts.append(get_auth_context_mount(read_only=True))
return k8s_schemas.V1Container(
name=generate_container_name(INIT_DOCKERFILE_CONTAINER_PREFIX),
image=polyaxon_init.get_image(),
image_pull_policy=polyaxon_init.image_pull_policy,
command=["polyaxon", "docker", "generate"],
args=[
"--build-context={}".format(dockerfile_args.to_dict(dump=True)),
"--destination={}".format(mount_path),
"--copy-path={}".format(
CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_path) + "/outputs"),
],
env=env,
resources=polyaxon_init.get_resources(),
volume_mounts=volume_mounts,
)
def ensure_container_name(container: k8s_schemas.V1Container,
prefix: str = None) -> k8s_schemas.V1Container:
if not container:
return container
name = container.name
if not name:
container.name = generate_container_name(prefix=prefix)
return container
def get_custom_init_container(
connection: V1ConnectionType,
contexts: PluginsContextsSpec,
container: Optional[k8s_schemas.V1Container],
env: List[k8s_schemas.V1EnvVar] = None,
mount_path: str = None,
) -> k8s_schemas.V1Container:
if not connection:
raise PolypodException(
"A connection is required to create a repo context.")
volume_name = (get_volume_name(mount_path)
if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS
volume_mounts = [
get_connections_context_mount(name=volume_name, mount_path=mount_path)
]
if contexts and contexts.auth:
volume_mounts.append(get_auth_context_mount(read_only=True))
env = to_list(env, check_none=True)
env_from = []
secret = connection.get_secret()
if secret:
volume_mounts += to_list(get_mount_from_resource(resource=secret),
check_none=True)
env += to_list(get_items_from_secret(secret=secret), check_none=True)
env_from = to_list(get_env_from_secret(secret=secret), check_none=True)
env += to_list(get_connection_env_var(connection=connection,
secret=secret),
check_none=True)
config_map = connection.get_config_map()
if config_map:
volume_mounts += to_list(get_mount_from_resource(resource=config_map),
check_none=True)
env += to_list(get_items_from_config_map(config_map=config_map),
check_none=True)
env_from = to_list(get_env_from_config_map(config_map=config_map),
check_none=True)
container_name = container.name or generate_container_name(
INIT_CUSTOM_CONTAINER_PREFIX, connection.name)
return patch_container(
container=container,
name=container_name,
env=env,
env_from=env_from,
volume_mounts=volume_mounts,
)
class InitSchema(BaseCamelSchema):
artifacts = fields.Nested(ArtifactsTypeSchema, allow_none=True)
git = fields.Nested(GitTypeSchema, allow_none=True)
dockerfile = fields.Nested(DockerfileTypeSchema, allow_none=True)
file = fields.Nested(FileTypeSchema, allow_none=True)
connection = fields.Str(allow_none=True)
path = fields.Str(allow_none=True)
container = SwaggerField(
cls=k8s_schemas.V1Container,
defaults={
"name": generate_container_name(prefix=POLYAXON_INIT_PREFIX)
},
allow_none=True,
)
@staticmethod
def schema_config():
return V1Init
@validates_schema
@check_partial
def validate_init(self, data, **kwargs):
artifacts = data.get("artifacts")
git = data.get("git")
dockerfile = data.get("dockerfile")
file = data.get("file")
connection = data.get("connection")
schemas = 0
if artifacts:
schemas += 1
if git:
schemas += 1
if dockerfile:
schemas += 1
if file:
schemas += 1
if schemas > 1:
raise ValidationError(
"One of artifacts, git, file, or dockerfile can be set")
if not connection and git and not git.url:
raise ValidationError(
"git field without a valid url requires a connection is required to be passed."
)
def test_get_store_container_bucket_stores(self):
mount_path = "/test-path"
resource1 = V1K8sResourceType(
name="non_mount_test1",
schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]),
is_requested=False,
)
store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
secret=resource1.schema,
)
container = get_store_container(
polyaxon_init=V1PolyaxonInitContainer(
image="foo/foo", image_tag="", image_pull_policy="IfNotPresent"
),
connection=store,
artifacts=None,
mount_path=mount_path,
)
assert container.name == generate_container_name(
INIT_ARTIFACTS_CONTAINER_PREFIX, store.name
)
assert container.image == "foo/foo"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["/bin/sh", "-c"]
assert container.args == [
get_volume_args(store=store, mount_path=mount_path, artifacts=None)
]
assert container.env is not None
assert container.env_from == []
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=get_volume_name(mount_path), mount_path=mount_path
)
]
def get_file_init_container(
polyaxon_init: V1PolyaxonInitContainer,
file_args: V1FileType,
contexts: PluginsContextsSpec,
run_path: str,
run_instance: str,
env: List[k8s_schemas.V1EnvVar] = None,
mount_path: Optional[str] = None,
) -> k8s_schemas.V1Container:
env = to_list(env, check_none=True)
env = env + [get_run_instance_env_var(run_instance)]
volume_name = (get_volume_name(mount_path)
if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS
volume_mounts = [
get_connections_context_mount(name=volume_name, mount_path=mount_path)
]
if contexts and contexts.auth:
volume_mounts.append(get_auth_context_mount(read_only=True))
file_args.filename = file_args.filename or "file"
return k8s_schemas.V1Container(
name=generate_container_name(INIT_FILE_CONTAINER_PREFIX),
image=polyaxon_init.get_image(),
image_pull_policy=polyaxon_init.image_pull_policy,
command=["polyaxon", "initializer", "file"],
args=[
"--file-context={}".format(file_args.to_dict(dump=True)),
"--filepath={}".format(mount_path),
"--copy-path={}".format(
CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format(run_path)),
"--track",
],
env=env,
resources=polyaxon_init.get_resources(),
volume_mounts=volume_mounts,
)
def test_get_store_container_mount_stores(self):
# Managed store
store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/tmp",
volume_claim="test",
read_only=True),
)
container = get_store_container(
polyaxon_init=V1PolyaxonInitContainer(
image="foo/foo",
image_tag="foo",
image_pull_policy="IfNotPresent"),
connection=store,
artifacts=None,
)
mount_path = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name)
assert container.name == generate_container_name(
INIT_ARTIFACTS_CONTAINER_PREFIX, store.name)
assert container.image == "foo/foo:foo"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["/bin/sh", "-c"]
assert container.args == [
get_volume_args(store=store, mount_path=mount_path, artifacts=None)
]
assert container.env == get_connection_env_var(connection=store,
secret=None)
assert container.env_from == []
assert container.resources is not None
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name),
),
get_mount_from_store(store=store),
]
def get_git_init_container(
polyaxon_init: V1PolyaxonInitContainer,
connection: V1ConnectionType,
contexts: PluginsContextsSpec,
container: Optional[k8s_schemas.V1Container] = None,
env: List[k8s_schemas.V1EnvVar] = None,
mount_path: str = None,
track: bool = False,
) -> k8s_schemas.V1Container:
if not connection:
raise PolypodException(
"A connection is required to create a repo context.")
if not container:
container = k8s_schemas.V1Container(name=generate_container_name(
INIT_GIT_CONTAINER_PREFIX, connection.name), )
volume_name = (get_volume_name(mount_path)
if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS
volume_mounts = [
get_connections_context_mount(name=volume_name, mount_path=mount_path)
]
if contexts and contexts.auth:
volume_mounts.append(get_auth_context_mount(read_only=True))
env = to_list(env, check_none=True)
env_from = []
secret = connection.get_secret()
if secret:
volume_mounts += to_list(get_mount_from_resource(resource=secret),
check_none=True)
env += to_list(get_items_from_secret(secret=secret), check_none=True)
env_from = to_list(get_env_from_secret(secret=secret), check_none=True)
env += to_list(get_connection_env_var(connection=connection,
secret=secret),
check_none=True)
config_map = connection.get_config_map()
if config_map:
volume_mounts += to_list(get_mount_from_resource(resource=config_map),
check_none=True)
env += to_list(get_items_from_config_map(config_map=config_map),
check_none=True)
env_from = to_list(get_env_from_config_map(config_map=config_map),
check_none=True)
args = get_repo_context_args(
name=connection.name,
url=connection.schema.url,
revision=connection.schema.revision,
mount_path=mount_path,
connection=connection.name if track else None,
)
return patch_container(
container=container,
name=generate_container_name(INIT_GIT_CONTAINER_PREFIX,
connection.name),
image=polyaxon_init.get_image(),
image_pull_policy=polyaxon_init.image_pull_policy,
command=["polyaxon", "initializer", "git"],
args=args,
env=env,
env_from=env_from,
volume_mounts=volume_mounts,
resources=polyaxon_init.get_resources(),
)
def test_get_git_init_container(self):
connection = V1ConnectionType(
name="user/foo",
kind=V1ConnectionKind.GIT,
schema=V1GitConnection(url="foo.com"),
)
container = get_git_init_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo", image_tag=""),
connection=connection,
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
)
assert container.name == generate_container_name(
INIT_GIT_CONTAINER_PREFIX, connection.name)
assert container.image == "foo"
assert container.image_pull_policy is None
assert container.command == ["polyaxon", "initializer", "git"]
assert container.env == get_connection_env_var(connection=connection,
secret=None)
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
),
get_auth_context_mount(read_only=True),
]
container = get_git_init_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init/init",
image_tag="",
image_pull_policy="IfNotPresent"),
connection=connection,
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
)
assert container.name == generate_container_name(
INIT_GIT_CONTAINER_PREFIX, connection.name)
assert container.image == "init/init"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["polyaxon", "initializer", "git"]
assert container.args == [
"--repo-path={}/{}".format(CONTEXT_MOUNT_ARTIFACTS,
connection.name),
"--url={}".format(connection.schema.url),
]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
),
get_auth_context_mount(read_only=True),
]
connection = V1ConnectionType(
name="user/foo",
kind=V1ConnectionKind.GIT,
schema=V1GitConnection(
url="foo.com",
revision="00b9d2ea01c40f58d6b4051319f9375675a43c02"),
)
container = get_git_init_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init/init",
image_tag="",
image_pull_policy="IfNotPresent"),
connection=connection,
mount_path="/somepath",
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
)
assert container.name == generate_container_name(
INIT_GIT_CONTAINER_PREFIX, connection.name)
assert container.image == "init/init"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["polyaxon", "initializer", "git"]
assert container.args == [
"--repo-path=/somepath/{}".format(connection.name),
"--url={}".format(connection.schema.url),
"--revision=00b9d2ea01c40f58d6b4051319f9375675a43c02",
]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(name=get_volume_name("/somepath"),
mount_path="/somepath"),
get_auth_context_mount(read_only=True),
]