def test_get_init_containers_with_git_without_connection(self):
git1 = V1GitConnection(revision="test", url="https://test.com")
git2 = V1GitConnection(revision="test", url="https://test.com")
containers = self.converter.get_init_containers(
contexts=None,
artifacts_store=None,
init_connections=[
V1Init(git=git1,
container=k8s_schemas.V1Container(name="test")),
V1Init(git=git2, path="/test"),
],
init_containers=[],
connection_by_names={},
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_git_init_container(
connection=V1ConnectionType(name=git1.get_name(),
kind=V1ConnectionKind.GIT,
schema=git1),
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
env=self.converter.get_init_service_env_vars(),
contexts=None,
),
get_git_init_container(
container=k8s_schemas.V1Container(name="test"),
connection=V1ConnectionType(name=git2.get_name(),
kind=V1ConnectionKind.GIT,
schema=git1),
mount_path="/test",
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
env=self.converter.get_init_service_env_vars(),
contexts=None,
),
]
def test_get_file_init_container(self):
file_args = V1FileType(content="test")
container = get_file_init_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo", image_tag=""),
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
file_args=V1FileType(content="test"),
run_path="test",
run_instance="foo.bar.runs.uuid",
)
assert INIT_FILE_CONTAINER_PREFIX in container.name
assert container.image == "foo"
assert container.image_pull_policy is None
assert container.command == ["polyaxon", "initializer", "file"]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
),
get_auth_context_mount(read_only=True),
]
assert file_args.to_dict(dump=True) == '{"content":"test"}'
assert container.args == [
"--file-context={}".format('{"content":"test","filename":"file"}'),
"--filepath={}".format(CONTEXT_MOUNT_ARTIFACTS),
"--copy-path={}".format(
CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("test")),
"--track",
]
file_args = V1FileType(filename="test", content="test")
container = get_file_init_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init/init",
image_tag="",
image_pull_policy="IfNotPresent"),
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
file_args=file_args,
run_path="test",
run_instance="foo.bar.runs.uuid",
)
assert INIT_FILE_CONTAINER_PREFIX in container.name
assert container.image == "init/init"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["polyaxon", "initializer", "file"]
assert container.args == [
"--file-context={}".format(file_args.to_dict(dump=True)),
"--filepath={}".format(CONTEXT_MOUNT_ARTIFACTS),
"--copy-path={}".format(
CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("test")),
"--track",
]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
),
get_auth_context_mount(read_only=True),
]
def test_get_artifacts_path_container_with_managed_mount_store(self):
store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/claim/path",
volume_claim="claim"),
)
container = get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"),
artifacts_store=store,
run_path="run_uid",
auto_resume=True,
)
init_args = init_artifact_context_args("run_uid")
init_args.append(
get_volume_args(
store=store,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
artifacts=V1ArtifactsType(dirs=["run_uid"]),
))
assert container == get_base_store_container(
container=k8s_schemas.V1Container(name="default"),
container_name=generate_container_name(
INIT_ARTIFACTS_CONTAINER_PREFIX, "default", False),
polyaxon_init=V1PolyaxonInitContainer(
image="init", image_pull_policy="IfNotPresent"),
store=store,
env=[],
env_from=[],
volume_mounts=[get_artifacts_context_mount()],
args=[" ".join(init_args)],
)
def get_base_store_container(
container: Optional[k8s_schemas.V1Container],
container_name: str,
polyaxon_init: V1PolyaxonInitContainer,
store: V1ConnectionType,
env: List[k8s_schemas.V1EnvVar],
env_from: List[k8s_schemas.V1EnvFromSource],
volume_mounts: List[k8s_schemas.V1VolumeMount],
args: List[str],
) -> Optional[k8s_schemas.V1Container]:
env = env or []
env_from = env_from or []
volume_mounts = volume_mounts or []
# Artifact store needs to allow init the contexts as well, so the store is not required
if not store:
raise PolypodException("Init store container requires a store")
secret = None
if store.is_bucket:
secret = store.get_secret()
volume_mounts = volume_mounts + to_list(
get_mount_from_resource(resource=secret), check_none=True)
env = env + to_list(get_items_from_secret(secret=secret),
check_none=True)
env_from = env_from + to_list(get_env_from_secret(secret=secret),
check_none=True)
env += to_list(get_connection_env_var(connection=store, secret=secret),
check_none=True)
config_map = store.get_config_map()
volume_mounts = volume_mounts + to_list(
get_mount_from_resource(resource=config_map), check_none=True)
env = env + to_list(get_items_from_config_map(config_map=config_map),
check_none=True)
env_from = env_from + to_list(
get_env_from_config_map(config_map=config_map), check_none=True)
else:
volume_mounts = volume_mounts + to_list(
get_mount_from_store(store=store), check_none=True)
env += to_list(get_connection_env_var(connection=store, secret=secret),
check_none=True)
return patch_container(
container=container,
name=container_name,
image=polyaxon_init.get_image(),
image_pull_policy=polyaxon_init.image_pull_policy,
command=["/bin/sh", "-c"],
args=args,
env=env,
env_from=env_from,
resources=polyaxon_init.get_resources(),
volume_mounts=volume_mounts,
)
def test_get_git_init_container_raises_for_missing_info(self):
with self.assertRaises(PolypodException):
get_git_init_container(polyaxon_init=V1PolyaxonInitContainer(),
connection=None,
contexts=None)
with self.assertRaises(PolypodException):
get_git_init_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/test"),
connection=None,
mount_path=None,
contexts=None,
)
def get_auth_context_container(
polyaxon_init: V1PolyaxonInitContainer, env: List[k8s_schemas.V1EnvVar] = None
) -> k8s_schemas.V1Container:
env = to_list(env, check_none=True)
return k8s_schemas.V1Container(
name=INIT_AUTH_CONTAINER,
image=polyaxon_init.get_image(),
image_pull_policy=polyaxon_init.image_pull_policy,
command=["polyaxon", "initializer", "auth"],
env=env,
resources=polyaxon_init.get_resources(),
volume_mounts=[get_auth_context_mount(read_only=False)],
)
def test_get_init_containers_with_dockerfiles(self):
dockerfile_args1 = V1DockerfileType(image="foo/test",
lang_env="LANG",
env=[],
run=["step1", "step2"])
dockerfile_args2 = V1DockerfileType(
image="foo/test",
lang_env="LANG",
env=[],
run=["step1", "step2"],
filename="dockerfile2",
path="/test",
)
containers = self.converter.get_init_containers(
contexts=None,
artifacts_store=None,
init_connections=[
V1Init(dockerfile=dockerfile_args1),
V1Init(dockerfile=dockerfile_args2, path="/test"),
],
init_containers=[],
connection_by_names={},
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
for container in containers:
container.name = ""
expected_containers = [
get_dockerfile_init_container(
dockerfile_args=dockerfile_args1,
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
env=self.converter.get_init_service_env_vars(),
contexts=None,
run_path=self.converter.run_path,
run_instance=self.converter.run_instance,
),
get_dockerfile_init_container(
dockerfile_args=dockerfile_args2,
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
env=self.converter.get_init_service_env_vars(),
mount_path="/test",
contexts=None,
run_path=self.converter.run_path,
run_instance=self.converter.run_instance,
),
]
for container in expected_containers:
container.name = ""
assert expected_containers == containers
def test_get_base_container(self):
store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/tmp",
volume_claim="test",
read_only=True),
)
env = [get_env_var(name="key", value="value")]
env_from = [k8s_schemas.V1EnvFromSource(secret_ref={"name": "ref"})]
mounts = [k8s_schemas.V1VolumeMount(name="test", mount_path="/test")]
container = get_base_store_container(
container=k8s_schemas.V1Container(name="init"),
container_name="init",
polyaxon_init=V1PolyaxonInitContainer(
image="foo/foo",
image_tag="",
image_pull_policy="IfNotPresent"),
store=store,
env=env,
env_from=env_from,
volume_mounts=mounts,
args=["test"],
)
assert container.name == "init"
assert container.image == "foo/foo"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["/bin/sh", "-c"]
assert container.args == ["test"]
assert container.env == env
assert container.env_from == env_from
assert container.resources is not None
assert container.volume_mounts == mounts + [
get_mount_from_store(store=store)
]
def test_get_base_store_container_with_mount_store(self):
claim_store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/tmp",
volume_claim="test",
read_only=True),
)
container = get_base_store_container(
container=k8s_schemas.V1Container(name="init"),
container_name="init",
polyaxon_init=V1PolyaxonInitContainer(image_tag=""),
store=claim_store,
env=None,
env_from=None,
volume_mounts=None,
args=None,
)
assert container.name == "init"
assert container.image == "polyaxon/polyaxon-init"
assert container.image_pull_policy is None
assert container.command == ["/bin/sh", "-c"]
assert container.args is None
assert container.env == get_connection_env_var(connection=claim_store,
secret=None)
assert container.env_from == []
assert container.resources is not None
assert container.volume_mounts == [
get_mount_from_store(store=claim_store)
]
def test_get_base_store_container_with_store_without_secret(self):
bucket_store_without_secret = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
)
container = get_base_store_container(
container=k8s_schemas.V1Container(name="test"),
container_name="init",
polyaxon_init=V1PolyaxonInitContainer(image_tag=""),
store=bucket_store_without_secret,
env=None,
env_from=None,
volume_mounts=None,
args=None,
)
assert container.name == "init"
assert container.image == "polyaxon/polyaxon-init"
assert container.image_pull_policy is None
assert container.command == ["/bin/sh", "-c"]
assert container.args is None
assert container.env == get_connection_env_var(
connection=bucket_store_without_secret, secret=None)
assert container.env_from == []
assert container.resources is not None
assert container.volume_mounts == []
def test_get_init_containers_with_auth(self):
containers = self.converter.get_init_containers(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_logs=False, collect_artifacts=False, auth=True)
),
artifacts_store=None,
init_connections=None,
connection_by_names={},
init_containers=[],
)
assert containers == [
get_auth_context_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
env=self.converter.get_auth_service_env_vars(),
)
]
def test_get_artifacts_path_container_with_none_values(self):
with self.assertRaises(PolypodException):
get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(),
artifacts_store=None,
run_path="",
auto_resume=True,
)
def test_get_init_containers_none(self):
containers = self.converter.get_init_containers(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
contexts=None,
artifacts_store=None,
init_connections=None,
connection_by_names={},
init_containers=[],
)
assert containers == []
def test_get_init_containers_with_store_outputs(self):
store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3://foo"),
)
# No context
containers = self.converter.get_init_containers(
contexts=None,
artifacts_store=store,
init_connections=[],
init_containers=[],
connection_by_names={},
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == []
# With context
containers = self.converter.get_init_containers(
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=True,
collect_logs=False,
auth=True)),
artifacts_store=store,
init_connections=[],
init_containers=[],
connection_by_names={},
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_auth_context_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
env=self.converter.get_auth_service_env_vars(),
),
get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
artifacts_store=store,
run_path=self.converter.run_path,
auto_resume=True,
),
]
def test_get_base_store_container_with_none_values(self):
with self.assertRaises(PolypodException):
get_base_store_container(
container=k8s_schemas.V1Container(name="init"),
container_name=None,
polyaxon_init=V1PolyaxonInitContainer(),
store=None,
env=None,
env_from=None,
volume_mounts=None,
args=None,
)
def get_dockerfile_init_container(
polyaxon_init: V1PolyaxonInitContainer,
dockerfile_args: V1DockerfileType,
contexts: PluginsContextsSpec,
run_path: str,
run_instance: str,
env: List[k8s_schemas.V1EnvVar] = None,
mount_path: Optional[str] = None,
) -> k8s_schemas.V1Container:
env = to_list(env, check_none=True)
env = env + [get_run_instance_env_var(run_instance)]
volume_name = (
get_volume_name(mount_path)
if mount_path
else constants.CONTEXT_VOLUME_ARTIFACTS
)
mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS
volume_mounts = [
get_connections_context_mount(name=volume_name, mount_path=mount_path)
]
if contexts and contexts.auth:
volume_mounts.append(get_auth_context_mount(read_only=True))
return k8s_schemas.V1Container(
name=generate_container_name(INIT_DOCKERFILE_CONTAINER_PREFIX),
image=polyaxon_init.get_image(),
image_pull_policy=polyaxon_init.image_pull_policy,
command=["polyaxon", "docker", "generate"],
args=[
"--build-context={}".format(dockerfile_args.to_dict(dump=True)),
"--destination={}".format(mount_path),
"--copy-path={}".format(CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format(run_path)),
"--track",
],
env=env,
resources=polyaxon_init.get_resources(),
volume_mounts=volume_mounts,
)
def test_get_init_containers_with_files(self):
file_args1 = V1FileType(filename="test.sh", content="test", chmod="+x")
file_args2 = V1FileType(
filename="test.csv",
content="csv",
kind=V1ArtifactKind.CSV,
)
containers = self.converter.get_init_containers(
contexts=None,
artifacts_store=None,
init_connections=[
V1Init(dockerfile=file_args1),
V1Init(dockerfile=file_args2, path="/test"),
],
init_containers=[],
connection_by_names={},
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
expected_containers = [
get_dockerfile_init_container(
dockerfile_args=file_args1,
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
env=self.converter.get_init_service_env_vars(),
contexts=None,
run_path=self.converter.run_path,
run_instance=self.converter.run_instance,
),
get_dockerfile_init_container(
dockerfile_args=file_args2,
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
env=self.converter.get_init_service_env_vars(),
mount_path="/test",
contexts=None,
run_path=self.converter.run_path,
run_instance=self.converter.run_instance,
),
]
self.assert_containers(expected_containers, containers)
def test_get_auth_context_container(self):
container = get_auth_context_container(
polyaxon_init=V1PolyaxonInitContainer(
image="foo/foo", image_tag="", image_pull_policy="IfNotPresent"
),
env=[],
)
assert container.name == INIT_AUTH_CONTAINER
assert container.image == "foo/foo"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["polyaxon", "initializer", "auth"]
assert container.args is None
assert container.env == []
assert container.resources == get_init_resources()
assert container.volume_mounts == [get_auth_context_mount(read_only=False)]
def sync_compatible_updates(self, compatible_updates: Dict):
if compatible_updates and settings.AGENT_CONFIG:
init = compatible_updates.get("init")
if init:
init = V1PolyaxonInitContainer.from_dict(init)
settings.AGENT_CONFIG.init = settings.AGENT_CONFIG.init.patch(
init)
sidecar = compatible_updates.get("sidecar")
if sidecar:
sidecar = V1PolyaxonSidecarContainer.from_dict(sidecar)
settings.AGENT_CONFIG.sidecar = settings.AGENT_CONFIG.sidecar.patch(
sidecar)
connections = compatible_updates.get("connections")
if connections:
settings.AGENT_CONFIG.connections = [
V1ConnectionType.from_dict(c) for c in connections
]
self.sync()
def test_get_store_container_bucket_stores(self):
mount_path = "/test-path"
resource1 = V1K8sResourceType(
name="non_mount_test1",
schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]),
is_requested=False,
)
store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
secret=resource1.schema,
)
container = get_store_container(
polyaxon_init=V1PolyaxonInitContainer(
image="foo/foo", image_tag="", image_pull_policy="IfNotPresent"
),
connection=store,
artifacts=None,
mount_path=mount_path,
)
assert container.name == generate_container_name(
INIT_ARTIFACTS_CONTAINER_PREFIX, store.name
)
assert container.image == "foo/foo"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["/bin/sh", "-c"]
assert container.args == [
get_volume_args(store=store, mount_path=mount_path, artifacts=None)
]
assert container.env is not None
assert container.env_from == []
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=get_volume_name(mount_path), mount_path=mount_path
)
]
def test_get_store_container_mount_stores(self):
# Managed store
store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/tmp",
volume_claim="test",
read_only=True),
)
container = get_store_container(
polyaxon_init=V1PolyaxonInitContainer(
image="foo/foo",
image_tag="foo",
image_pull_policy="IfNotPresent"),
connection=store,
artifacts=None,
)
mount_path = CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(store.name)
assert container.name == generate_container_name(
INIT_ARTIFACTS_CONTAINER_PREFIX, store.name)
assert container.image == "foo/foo:foo"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["/bin/sh", "-c"]
assert container.args == [
get_volume_args(store=store, mount_path=mount_path, artifacts=None)
]
assert container.env == get_connection_env_var(connection=store,
secret=None)
assert container.env_from == []
assert container.resources is not None
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
),
get_mount_from_store(store=store),
]
def test_get_base_store_container_with_store_with_secret(self):
non_mount_resource1 = V1K8sResourceType(
name="resource",
schema=V1K8sResourceSchema(name="ref", items=["item1", "item2"]),
is_requested=False,
)
bucket_store_with_secret = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
secret=non_mount_resource1.schema,
)
container = get_base_store_container(
container=k8s_schemas.V1Container(name="init"),
container_name="init",
polyaxon_init=V1PolyaxonInitContainer(image_tag=""),
store=bucket_store_with_secret,
env=None,
env_from=None,
volume_mounts=None,
args=None,
)
assert container.name == "init"
assert container.image == "polyaxon/polyaxon-init"
assert container.image_pull_policy is None
assert container.command == ["/bin/sh", "-c"]
assert container.args is None
env = get_items_from_secret(
secret=non_mount_resource1) + get_connection_env_var(
connection=bucket_store_with_secret,
secret=non_mount_resource1)
assert container.env == env
assert container.env_from == []
assert container.resources is not None
assert container.volume_mounts == []
mount_resource1 = V1K8sResourceType(
name="resource",
schema=V1K8sResourceSchema(name="resource",
items=["item1", "item2"],
mount_path="/tmp1"),
is_requested=False,
)
bucket_store_with_secret.secret = mount_resource1.schema
container = get_base_store_container(
container=k8s_schemas.V1Container(name="init"),
container_name="init",
polyaxon_init=V1PolyaxonInitContainer(image_tag=""),
store=bucket_store_with_secret,
env=None,
env_from=None,
volume_mounts=None,
args=None,
)
assert container.name == "init"
assert container.image == "polyaxon/polyaxon-init"
assert container.image_pull_policy is None
assert container.command == ["/bin/sh", "-c"]
assert container.args is None
assert container.env == get_connection_env_var(
connection=bucket_store_with_secret, secret=mount_resource1)
assert container.env_from == []
assert container.resources is not None
assert container.volume_mounts == [
get_mount_from_resource(resource=mount_resource1)
]
def test_get_dockerfile_init_container(self):
dockerfile_args = V1DockerfileType(image="test/test")
container = get_dockerfile_init_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo", image_tag=""),
dockerfile_args=dockerfile_args,
env=None,
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
run_path="test",
run_instance="foo.bar.runs.uuid",
)
assert INIT_DOCKERFILE_CONTAINER_PREFIX in container.name
assert container.image == "foo"
assert container.image_pull_policy is None
assert container.command == ["polyaxon", "docker", "generate"]
assert container.args == [
"--build-context={}".format(dockerfile_args.to_dict(dump=True)),
"--destination={}".format(CONTEXT_MOUNT_ARTIFACTS),
"--copy-path={}".format(
CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("test")),
"--track",
]
assert container.env == [
get_run_instance_env_var(run_instance="foo.bar.runs.uuid")
]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
),
get_auth_context_mount(read_only=True),
]
dockerfile_args = V1DockerfileType(
image="test/test",
lang_env="LANG",
run=["step1", "step2"],
env=[["key1", "val1"], ["key2", "val2"]],
uid=2222,
gid=2222,
)
container = get_dockerfile_init_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init/init",
image_tag="",
image_pull_policy="IfNotPresent"),
env=[],
dockerfile_args=dockerfile_args,
mount_path="/somepath",
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
run_path="test",
run_instance="foo.bar.runs.uuid",
)
assert INIT_DOCKERFILE_CONTAINER_PREFIX in container.name
assert container.image == "init/init"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["polyaxon", "docker", "generate"]
assert container.args == [
"--build-context={}".format(dockerfile_args.to_dict(dump=True)),
"--destination=/somepath",
"--copy-path={}".format(
CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("test")),
"--track",
]
assert container.env == [
get_run_instance_env_var(run_instance="foo.bar.runs.uuid")
]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(name=get_volume_name("/somepath"),
mount_path="/somepath"),
get_auth_context_mount(read_only=True),
]
def get_git_init_container(
polyaxon_init: V1PolyaxonInitContainer,
connection: V1ConnectionType,
contexts: PluginsContextsSpec,
container: Optional[k8s_schemas.V1Container] = None,
env: List[k8s_schemas.V1EnvVar] = None,
mount_path: str = None,
track: bool = False,
) -> k8s_schemas.V1Container:
if not connection:
raise PolypodException(
"A connection is required to create a repo context.")
if not container:
container = k8s_schemas.V1Container(name=generate_container_name(
INIT_GIT_CONTAINER_PREFIX, connection.name), )
volume_name = (get_volume_name(mount_path)
if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS
volume_mounts = [
get_connections_context_mount(name=volume_name, mount_path=mount_path)
]
if contexts and contexts.auth:
volume_mounts.append(get_auth_context_mount(read_only=True))
env = to_list(env, check_none=True)
env_from = []
secret = connection.get_secret()
if secret:
volume_mounts += to_list(get_mount_from_resource(resource=secret),
check_none=True)
env += to_list(get_items_from_secret(secret=secret), check_none=True)
env_from = to_list(get_env_from_secret(secret=secret), check_none=True)
env += to_list(get_connection_env_var(connection=connection,
secret=secret),
check_none=True)
config_map = connection.get_config_map()
if config_map:
volume_mounts += to_list(get_mount_from_resource(resource=config_map),
check_none=True)
env += to_list(get_items_from_config_map(config_map=config_map),
check_none=True)
env_from = to_list(get_env_from_config_map(config_map=config_map),
check_none=True)
args = get_repo_context_args(
name=connection.name,
url=connection.schema.url,
revision=connection.schema.revision,
mount_path=mount_path,
connection=connection.name if track else None,
)
return patch_container(
container=container,
name=generate_container_name(INIT_GIT_CONTAINER_PREFIX,
connection.name),
image=polyaxon_init.get_image(),
image_pull_policy=polyaxon_init.image_pull_policy,
command=["polyaxon", "initializer", "git"],
args=args,
env=env,
env_from=env_from,
volume_mounts=volume_mounts,
resources=polyaxon_init.get_resources(),
)
def test_get_git_init_container(self):
connection = V1ConnectionType(
name="user/foo",
kind=V1ConnectionKind.GIT,
schema=V1GitConnection(url="foo.com"),
)
container = get_git_init_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo", image_tag=""),
connection=connection,
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
)
assert container.name == generate_container_name(
INIT_GIT_CONTAINER_PREFIX, connection.name)
assert container.image == "foo"
assert container.image_pull_policy is None
assert container.command == ["polyaxon", "initializer", "git"]
assert container.env == get_connection_env_var(connection=connection,
secret=None)
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
),
get_auth_context_mount(read_only=True),
]
container = get_git_init_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init/init",
image_tag="",
image_pull_policy="IfNotPresent"),
connection=connection,
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
)
assert container.name == generate_container_name(
INIT_GIT_CONTAINER_PREFIX, connection.name)
assert container.image == "init/init"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["polyaxon", "initializer", "git"]
assert container.args == [
"--repo-path={}/{}".format(CONTEXT_MOUNT_ARTIFACTS,
connection.name),
"--url={}".format(connection.schema.url),
]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
),
get_auth_context_mount(read_only=True),
]
connection = V1ConnectionType(
name="user/foo",
kind=V1ConnectionKind.GIT,
schema=V1GitConnection(
url="foo.com",
revision="00b9d2ea01c40f58d6b4051319f9375675a43c02"),
)
container = get_git_init_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init/init",
image_tag="",
image_pull_policy="IfNotPresent"),
connection=connection,
mount_path="/somepath",
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
)
assert container.name == generate_container_name(
INIT_GIT_CONTAINER_PREFIX, connection.name)
assert container.image == "init/init"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["polyaxon", "initializer", "git"]
assert container.args == [
"--repo-path=/somepath/{}".format(connection.name),
"--url={}".format(connection.schema.url),
"--revision=00b9d2ea01c40f58d6b4051319f9375675a43c02",
]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(name=get_volume_name("/somepath"),
mount_path="/somepath"),
get_auth_context_mount(read_only=True),
]
def _extra_processing(cls, value):
if not value:
return value
V1PolyaxonInitContainer.from_dict(value)
return value
def test_get_init_containers_with_claim_outputs(self):
store = V1ConnectionType(
name="test_claim",
kind=V1ConnectionKind.VOLUME_CLAIM,
schema=V1ClaimConnection(mount_path="/claim/path",
volume_claim="claim",
read_only=True),
)
# No context to enable the outputs
containers = self.converter.get_init_containers(
contexts=None,
artifacts_store=store.name,
init_connections=None,
connection_by_names={},
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == []
# Enable outputs
containers = self.converter.get_init_containers(
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=True, collect_logs=False)),
artifacts_store=store,
connection_by_names={},
init_connections=None,
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
artifacts_store=store,
run_path=self.converter.run_path,
auto_resume=True,
),
]
# Use store for init
containers = self.converter.get_init_containers(
contexts=None,
artifacts_store=None,
connection_by_names={store.name: store},
init_connections=[V1Init(connection=store.name)],
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_store_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
connection=store,
artifacts=None,
env=self.converter.get_init_service_env_vars(),
)
]
# Use store for init and outputs
containers = self.converter.get_init_containers(
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=True, collect_logs=False)),
artifacts_store=store,
init_connections=[V1Init(connection=store.name)],
connection_by_names={store.name: store},
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
artifacts_store=store,
run_path=self.converter.run_path,
auto_resume=True,
),
get_store_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
connection=store,
artifacts=None,
env=self.converter.get_init_service_env_vars(),
is_default_artifacts_store=True,
),
]
# Add Store
store1 = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.S3,
schema=V1BucketConnection(bucket="s3://foo"),
secret=None,
)
containers = self.converter.get_init_containers(
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=True,
collect_logs=False,
auth=True)),
artifacts_store=store,
init_connections=[
V1Init(
connection=store.name,
artifacts=V1ArtifactsType(files=["/foo", "/bar"]),
),
V1Init(
connection=store1.name,
artifacts=V1ArtifactsType(files=["/foo", "/bar"]),
),
],
connection_by_names={
store.name: store,
store1.name: store1
},
init_containers=[],
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
)
assert containers == [
get_auth_context_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
env=self.converter.get_auth_service_env_vars(),
),
get_artifacts_path_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
artifacts_store=store,
run_path=self.converter.run_path,
auto_resume=True,
),
get_store_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
connection=store,
artifacts=V1ArtifactsType(files=["/foo", "/bar"]),
env=self.converter.get_init_service_env_vars(),
is_default_artifacts_store=True,
),
get_store_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo/foo"),
connection=store1,
artifacts=V1ArtifactsType(files=["/foo", "/bar"]),
env=self.converter.get_init_service_env_vars(),
),
]
