def test_get_file_init_container(self):
file_args = V1FileType(content="test")
container = get_file_init_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo", image_tag=""),
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
file_args=V1FileType(content="test"),
run_path="test",
run_instance="foo.bar.runs.uuid",
)
assert INIT_FILE_CONTAINER_PREFIX in container.name
assert container.image == "foo"
assert container.image_pull_policy is None
assert container.command == ["polyaxon", "initializer", "file"]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
),
get_auth_context_mount(read_only=True),
]
assert file_args.to_dict(dump=True) == '{"content":"test"}'
assert container.args == [
"--file-context={}".format('{"content":"test","filename":"file"}'),
"--filepath={}".format(CONTEXT_MOUNT_ARTIFACTS),
"--copy-path={}".format(
CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("test")),
"--track",
]
file_args = V1FileType(filename="test", content="test")
container = get_file_init_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init/init",
image_tag="",
image_pull_policy="IfNotPresent"),
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
file_args=file_args,
run_path="test",
run_instance="foo.bar.runs.uuid",
)
assert INIT_FILE_CONTAINER_PREFIX in container.name
assert container.image == "init/init"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["polyaxon", "initializer", "file"]
assert container.args == [
"--file-context={}".format(file_args.to_dict(dump=True)),
"--filepath={}".format(CONTEXT_MOUNT_ARTIFACTS),
"--copy-path={}".format(
CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("test")),
"--track",
]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
),
get_auth_context_mount(read_only=True),
]
def test_single_connections(self):
self.assert_single_store(store=self.s3_store, results=[])
self.assert_single_store(store=self.gcs_store, results=[])
self.assert_single_store(store=self.az_store, results=[])
self.assert_single_store(
store=self.claim_store,
results=[get_mount_from_store(store=self.claim_store)],
)
self.assert_single_store(
store=self.host_path_store,
results=[get_mount_from_store(store=self.host_path_store)],
)
# Managed versions
volume_name = get_volume_name("/test")
self.assert_single_init_store(
store=self.s3_store,
results=[
get_connections_context_mount(name=volume_name,
mount_path="/test")
],
)
self.assert_single_init_store(
store=self.gcs_store,
results=[
get_connections_context_mount(name=volume_name,
mount_path="/test")
],
)
self.assert_single_init_store(
store=self.az_store,
results=[
get_connections_context_mount(name=volume_name,
mount_path="/test")
],
)
self.assert_single_init_store(
store=self.claim_store,
results=[
get_connections_context_mount(name=volume_name,
mount_path="/test")
],
)
self.assert_single_init_store(
store=self.host_path_store,
results=[
get_connections_context_mount(name=volume_name,
mount_path="/test")
],
)
def get_store_container(
polyaxon_init: V1PolyaxonInitContainer,
connection: V1ConnectionType,
artifacts: V1ArtifactsType,
container: Optional[k8s_schemas.V1Container] = None,
env: List[k8s_schemas.V1EnvVar] = None,
mount_path: str = None,
) -> Optional[k8s_schemas.V1Container]:
container_name = INIT_ARTIFACTS_CONTAINER.format(connection.name)
if not container:
container = k8s_schemas.V1Container(name=container_name)
volume_name = (get_volume_name(mount_path)
if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(
connection.name)
volume_mounts = [
get_connections_context_mount(name=volume_name, mount_path=mount_path)
]
return get_base_store_container(
container=container,
container_name=container_name,
polyaxon_init=polyaxon_init,
store=connection,
env=env,
env_from=[],
volume_mounts=volume_mounts,
args=[
get_volume_args(store=connection,
mount_path=mount_path,
artifacts=artifacts)
],
)
def get_dockerfile_init_container(
polyaxon_init: V1PolyaxonInitContainer,
dockerfile_args: V1DockerfileType,
contexts: PluginsContextsSpec,
run_path: str,
env: List[k8s_schemas.V1EnvVar] = None,
mount_path: Optional[str] = None,
) -> k8s_schemas.V1Container:
env = to_list(env, check_none=True)
env = env + [get_run_instance_env_var()]
volume_name = (get_volume_name(mount_path)
if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS
volume_mounts = [
get_connections_context_mount(name=volume_name, mount_path=mount_path)
]
if contexts and contexts.auth:
volume_mounts.append(get_auth_context_mount(read_only=True))
return k8s_schemas.V1Container(
name=generate_container_name(INIT_DOCKERFILE_CONTAINER_PREFIX),
image=polyaxon_init.get_image(),
image_pull_policy=polyaxon_init.image_pull_policy,
command=["polyaxon", "docker", "generate"],
args=[
"--build-context={}".format(dockerfile_args.to_dict(dump=True)),
"--destination={}".format(mount_path),
"--copy-path={}".format(
CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_path) + "/outputs"),
],
env=env,
resources=polyaxon_init.get_resources(),
volume_mounts=volume_mounts,
)
def test_get_store_container_mount_stores(self):
# Managed store
store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(
mount_path="/tmp", volume_claim="test", read_only=True
),
)
container = get_store_container(
polyaxon_init=V1PolyaxonInitContainer(
image="foo/foo", image_tag="foo", image_pull_policy="IfNotPresent"
),
connection=store,
artifacts=None,
)
mount_path = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name)
assert container.name == INIT_ARTIFACTS_CONTAINER.format(store.name)
assert container.image == "foo/foo:foo"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["/bin/sh", "-c"]
assert container.args == [
get_volume_args(store=store, mount_path=mount_path, artifacts=None)
]
assert container.env == get_connection_env_var(connection=store, secret=None)
assert container.env_from == []
assert container.resources is not None
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name),
),
get_mount_from_store(store=store),
]
def get_custom_init_container(
connection: V1ConnectionType,
contexts: PluginsContextsSpec,
container: Optional[k8s_schemas.V1Container],
env: List[k8s_schemas.V1EnvVar] = None,
mount_path: str = None,
) -> k8s_schemas.V1Container:
if not connection:
raise PolypodException(
"A connection is required to create a repo context.")
volume_name = (get_volume_name(mount_path)
if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS
volume_mounts = [
get_connections_context_mount(name=volume_name, mount_path=mount_path)
]
if contexts and contexts.auth:
volume_mounts.append(get_auth_context_mount(read_only=True))
env = to_list(env, check_none=True)
env_from = []
secret = connection.get_secret()
if secret:
volume_mounts += to_list(get_mount_from_resource(resource=secret),
check_none=True)
env += to_list(get_items_from_secret(secret=secret), check_none=True)
env_from = to_list(get_env_from_secret(secret=secret), check_none=True)
env += to_list(get_connection_env_var(connection=connection,
secret=secret),
check_none=True)
config_map = connection.get_config_map()
if config_map:
volume_mounts += to_list(get_mount_from_resource(resource=config_map),
check_none=True)
env += to_list(get_items_from_config_map(config_map=config_map),
check_none=True)
env_from = to_list(get_env_from_config_map(config_map=config_map),
check_none=True)
container_name = container.name or generate_container_name(
INIT_CUSTOM_CONTAINER_PREFIX, connection.name)
return patch_container(
container=container,
name=container_name,
env=env,
env_from=env_from,
volume_mounts=volume_mounts,
)
def get_volume_mounts(
contexts: PluginsContextsSpec,
init: Optional[List[V1Init]],
connections: Iterable[V1ConnectionType],
secrets: Iterable[V1K8sResourceType],
config_maps: Iterable[V1K8sResourceType] = None,
) -> List[k8s_schemas.V1VolumeMount]:
init = init or []
connections = connections or []
secrets = secrets or []
config_maps = config_maps or []
volume_mounts = []
volume_names = set()
if contexts and contexts.collect_artifacts:
volume_mounts += to_list(
get_artifacts_context_mount(read_only=False), check_none=True
)
volume_names.add(constants.CONTEXT_VOLUME_ARTIFACTS)
for init_connection in init:
volume_name = (
get_volume_name(init_connection.path)
if init_connection.path
else constants.CONTEXT_VOLUME_ARTIFACTS
)
mount_path = init_connection.path or CONTEXT_MOUNT_ARTIFACTS
if volume_name in volume_names:
continue
volume_names.add(volume_name)
volume_mounts += to_list(
get_connections_context_mount(name=volume_name, mount_path=mount_path),
check_none=True,
)
for store in connections:
volume_mounts += to_list(get_mount_from_store(store=store), check_none=True)
for secret in secrets:
volume_mounts += to_list(
get_mount_from_resource(resource=secret), check_none=True
)
for config_map in config_maps:
volume_mounts += to_list(
get_mount_from_resource(resource=config_map), check_none=True
)
return volume_mounts
def test_get_store_container_bucket_stores(self):
mount_path = "/test-path"
resource1 = V1K8sResourceType(
name="non_mount_test1",
schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]),
is_requested=False,
)
store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
secret=resource1.schema,
)
container = get_store_container(
polyaxon_init=V1PolyaxonInitContainer(
image="foo/foo", image_tag="", image_pull_policy="IfNotPresent"
),
connection=store,
artifacts=None,
mount_path=mount_path,
)
assert container.name == generate_container_name(
INIT_ARTIFACTS_CONTAINER_PREFIX, store.name
)
assert container.image == "foo/foo"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["/bin/sh", "-c"]
assert container.args == [
get_volume_args(store=store, mount_path=mount_path, artifacts=None)
]
assert container.env is not None
assert container.env_from == []
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=get_volume_name(mount_path), mount_path=mount_path
)
]
def get_file_init_container(
polyaxon_init: V1PolyaxonInitContainer,
file_args: V1FileType,
contexts: PluginsContextsSpec,
run_path: str,
run_instance: str,
env: List[k8s_schemas.V1EnvVar] = None,
mount_path: Optional[str] = None,
) -> k8s_schemas.V1Container:
env = to_list(env, check_none=True)
env = env + [get_run_instance_env_var(run_instance)]
volume_name = (get_volume_name(mount_path)
if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS
volume_mounts = [
get_connections_context_mount(name=volume_name, mount_path=mount_path)
]
if contexts and contexts.auth:
volume_mounts.append(get_auth_context_mount(read_only=True))
file_args.filename = file_args.filename or "file"
return k8s_schemas.V1Container(
name=generate_container_name(INIT_FILE_CONTAINER_PREFIX),
image=polyaxon_init.get_image(),
image_pull_policy=polyaxon_init.image_pull_policy,
command=["polyaxon", "initializer", "file"],
args=[
"--file-context={}".format(file_args.to_dict(dump=True)),
"--filepath={}".format(mount_path),
"--copy-path={}".format(
CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format(run_path)),
"--track",
],
env=env,
resources=polyaxon_init.get_resources(),
volume_mounts=volume_mounts,
)
def test_get_connections_context_mount(self):
mount = get_connections_context_mount(name="test", mount_path="/test")
assert mount.name == "test"
assert mount.mount_path == "/test"
assert mount.read_only is None
def get_git_init_container(
polyaxon_init: V1PolyaxonInitContainer,
connection: V1ConnectionType,
contexts: PluginsContextsSpec,
container: Optional[k8s_schemas.V1Container] = None,
env: List[k8s_schemas.V1EnvVar] = None,
mount_path: str = None,
track: bool = False,
) -> k8s_schemas.V1Container:
if not connection:
raise PolypodException(
"A connection is required to create a repo context.")
if not container:
container = k8s_schemas.V1Container(name=generate_container_name(
INIT_GIT_CONTAINER_PREFIX, connection.name), )
volume_name = (get_volume_name(mount_path)
if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS
volume_mounts = [
get_connections_context_mount(name=volume_name, mount_path=mount_path)
]
if contexts and contexts.auth:
volume_mounts.append(get_auth_context_mount(read_only=True))
env = to_list(env, check_none=True)
env_from = []
secret = connection.get_secret()
if secret:
volume_mounts += to_list(get_mount_from_resource(resource=secret),
check_none=True)
env += to_list(get_items_from_secret(secret=secret), check_none=True)
env_from = to_list(get_env_from_secret(secret=secret), check_none=True)
env += to_list(get_connection_env_var(connection=connection,
secret=secret),
check_none=True)
config_map = connection.get_config_map()
if config_map:
volume_mounts += to_list(get_mount_from_resource(resource=config_map),
check_none=True)
env += to_list(get_items_from_config_map(config_map=config_map),
check_none=True)
env_from = to_list(get_env_from_config_map(config_map=config_map),
check_none=True)
args = get_repo_context_args(
name=connection.name,
url=connection.schema.url,
revision=connection.schema.revision,
mount_path=mount_path,
connection=connection.name if track else None,
)
return patch_container(
container=container,
name=generate_container_name(INIT_GIT_CONTAINER_PREFIX,
connection.name),
image=polyaxon_init.get_image(),
image_pull_policy=polyaxon_init.image_pull_policy,
command=["polyaxon", "initializer", "git"],
args=args,
env=env,
env_from=env_from,
volume_mounts=volume_mounts,
resources=polyaxon_init.get_resources(),
)
def test_get_git_init_container(self):
connection = V1ConnectionType(
name="user/foo",
kind=V1ConnectionKind.GIT,
schema=V1GitConnection(url="foo.com"),
)
container = get_git_init_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo", image_tag=""),
connection=connection,
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
)
assert container.name == INIT_GIT_CONTAINER.format(connection.name)
assert container.image == "foo"
assert container.image_pull_policy is None
assert container.command == ["polyaxon", "initializer", "git"]
assert container.env == get_connection_env_var(connection=connection,
secret=None)
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
),
get_auth_context_mount(read_only=True),
]
container = get_git_init_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init/init",
image_tag="",
image_pull_policy="IfNotPresent"),
connection=connection,
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
)
assert container.name == INIT_GIT_CONTAINER.format(connection.name)
assert container.image == "init/init"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["polyaxon", "initializer", "git"]
assert container.args == [
"--repo_path={}/{}".format(CONTEXT_MOUNT_ARTIFACTS,
connection.name),
"--url={}".format(connection.schema.url),
]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
),
get_auth_context_mount(read_only=True),
]
connection = V1ConnectionType(
name="user/foo",
kind=V1ConnectionKind.GIT,
schema=V1GitConnection(
url="foo.com",
revision="00b9d2ea01c40f58d6b4051319f9375675a43c02"),
)
container = get_git_init_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init/init",
image_tag="",
image_pull_policy="IfNotPresent"),
connection=connection,
mount_path="/somepath",
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
)
assert container.name == INIT_GIT_CONTAINER.format(connection.name)
assert container.image == "init/init"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["polyaxon", "initializer", "git"]
assert container.args == [
"--repo_path=/somepath/{}".format(connection.name),
"--url={}".format(connection.schema.url),
"--revision=00b9d2ea01c40f58d6b4051319f9375675a43c02",
]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(name=get_volume_name("/somepath"),
mount_path="/somepath"),
get_auth_context_mount(read_only=True),
]
def test_get_dockerfile_init_container(self):
dockerfile_args = V1DockerfileType(image="test/test")
container = get_dockerfile_init_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo", image_tag=""),
dockerfile_args=dockerfile_args,
env=None,
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
run_path="test",
run_instance="foo.bar.runs.uuid",
)
assert INIT_DOCKERFILE_CONTAINER_PREFIX in container.name
assert container.image == "foo"
assert container.image_pull_policy is None
assert container.command == ["polyaxon", "docker", "generate"]
assert container.args == [
"--build-context={}".format(dockerfile_args.to_dict(dump=True)),
"--destination={}".format(CONTEXT_MOUNT_ARTIFACTS),
"--copy-path={}".format(
CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("test")),
"--track",
]
assert container.env == [
get_run_instance_env_var(run_instance="foo.bar.runs.uuid")
]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
),
get_auth_context_mount(read_only=True),
]
dockerfile_args = V1DockerfileType(
image="test/test",
lang_env="LANG",
run=["step1", "step2"],
env=[["key1", "val1"], ["key2", "val2"]],
uid=2222,
gid=2222,
)
container = get_dockerfile_init_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init/init",
image_tag="",
image_pull_policy="IfNotPresent"),
env=[],
dockerfile_args=dockerfile_args,
mount_path="/somepath",
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
run_path="test",
run_instance="foo.bar.runs.uuid",
)
assert INIT_DOCKERFILE_CONTAINER_PREFIX in container.name
assert container.image == "init/init"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["polyaxon", "docker", "generate"]
assert container.args == [
"--build-context={}".format(dockerfile_args.to_dict(dump=True)),
"--destination=/somepath",
"--copy-path={}".format(
CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("test")),
"--track",
]
assert container.env == [
get_run_instance_env_var(run_instance="foo.bar.runs.uuid")
]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(name=get_volume_name("/somepath"),
mount_path="/somepath"),
get_auth_context_mount(read_only=True),
]