def test_get_sidecar_container_with_managed_mount_outputs_logs_store(self):
env_vars = [
get_env_var(name="key1", value="value1"),
get_env_var(name="key2", value="value2"),
]
mount_managed_store = V1ConnectionType(
name="test_path",
kind=V1ConnectionKind.HOST_PATH,
schema=V1HostPathConnection(mount_path="/tmp", host_path="/tmp"),
secret=None,
)
# logs and outputs
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
env=env_vars,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar",
image_tag="",
image_pull_policy="IfNotPresent",
sleep_interval=213,
sync_interval=212,
),
artifacts_store=mount_managed_store,
contexts=PluginsContextsSpec.from_config(None, default_auth=True),
run_path="test",
)
assert sidecar.name == SIDECAR_CONTAINER
assert sidecar.image == "sidecar/sidecar"
assert sidecar.image_pull_policy == "IfNotPresent"
assert sidecar.command == ["polyaxon", "sidecar"]
assert sidecar.args == get_sidecar_args(
container_id=MAIN_JOB_CONTAINER,
sleep_interval=213,
sync_interval=212,
monitor_logs=False,
)
assert (sidecar.env == get_sidecar_env_vars(
env_vars=env_vars,
container_id=MAIN_JOB_CONTAINER,
artifacts_store_name=mount_managed_store.name,
) + get_connection_env_var(connection=mount_managed_store,
secret=None))
assert sidecar.env_from == []
assert sidecar.resources == get_sidecar_resources()
assert sidecar.volume_mounts == [
get_auth_context_mount(read_only=True),
get_artifacts_context_mount(read_only=False),
get_mount_from_store(store=mount_managed_store),
]
# logs and no outputs
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
env=env_vars,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar",
image_tag="",
image_pull_policy="IfNotPresent",
sleep_interval=213,
sync_interval=212,
),
artifacts_store=mount_managed_store,
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=False, auth=True)),
run_path="test",
)
assert sidecar.name == SIDECAR_CONTAINER
assert sidecar.image == "sidecar/sidecar"
assert sidecar.image_pull_policy == "IfNotPresent"
assert sidecar.command == ["polyaxon", "sidecar"]
assert sidecar.args == get_sidecar_args(
container_id=MAIN_JOB_CONTAINER,
sleep_interval=213,
sync_interval=212,
monitor_logs=False,
)
assert (sidecar.env == get_sidecar_env_vars(
env_vars=env_vars,
container_id=MAIN_JOB_CONTAINER,
artifacts_store_name=mount_managed_store.name,
) + get_connection_env_var(connection=mount_managed_store,
secret=None))
assert sidecar.env_from == []
assert sidecar.resources == get_sidecar_resources()
assert sidecar.volume_mounts == [
get_auth_context_mount(read_only=True),
get_mount_from_store(store=mount_managed_store),
]
# outputs and no logs
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
env=env_vars,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar",
image_tag="",
image_pull_policy="IfNotPresent",
sleep_interval=213,
sync_interval=212,
),
artifacts_store=mount_managed_store,
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_logs=False, auth=True)),
run_path="test",
)
assert sidecar.name == SIDECAR_CONTAINER
assert sidecar.image == "sidecar/sidecar"
assert sidecar.image_pull_policy == "IfNotPresent"
assert sidecar.command == ["polyaxon", "sidecar"]
assert sidecar.args == get_sidecar_args(
container_id=MAIN_JOB_CONTAINER,
sleep_interval=213,
sync_interval=212,
monitor_logs=False,
)
assert (sidecar.env == get_sidecar_env_vars(
env_vars=env_vars,
container_id=MAIN_JOB_CONTAINER,
artifacts_store_name=mount_managed_store.name,
) + get_connection_env_var(connection=mount_managed_store,
secret=None))
assert sidecar.env_from == []
assert sidecar.resources == get_sidecar_resources()
assert sidecar.volume_mounts == [
get_auth_context_mount(read_only=True),
get_artifacts_context_mount(read_only=False),
get_mount_from_store(store=mount_managed_store),
]
def test_get_sidecar_container_with_managed_bucket_outputs_logs_store_and_env_from(
self, ):
env_vars = [
get_env_var(name="key1", value="value1"),
get_env_var(name="key2", value="value2"),
]
resource1 = V1K8sResourceType(name="test1",
schema=V1K8sResourceSchema(name="ref"),
is_requested=False)
bucket_managed_store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
secret=resource1.schema,
)
# both logs and outputs
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
env=env_vars,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar",
image_tag="",
image_pull_policy="IfNotPresent",
sleep_interval=213,
sync_interval=212,
),
artifacts_store=bucket_managed_store,
contexts=PluginsContextsSpec.from_config(None, default_auth=True),
run_path="test",
)
assert sidecar.name == SIDECAR_CONTAINER
assert sidecar.image == "sidecar/sidecar"
assert sidecar.image_pull_policy == "IfNotPresent"
assert sidecar.command == ["polyaxon", "sidecar"]
assert sidecar.args == get_sidecar_args(
container_id=MAIN_JOB_CONTAINER,
sleep_interval=213,
sync_interval=212,
monitor_logs=False,
)
assert (sidecar.env == get_sidecar_env_vars(
env_vars=env_vars,
container_id=MAIN_JOB_CONTAINER,
artifacts_store_name=bucket_managed_store.name,
) + get_items_from_secret(secret=resource1) + get_connection_env_var(
connection=bucket_managed_store, secret=resource1))
assert sidecar.env_from == [get_env_from_secret(secret=resource1)]
assert sidecar.resources == get_sidecar_resources()
assert sidecar.volume_mounts == [
get_auth_context_mount(read_only=True),
get_artifacts_context_mount(read_only=False),
]
# logs and no outputs
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
env=env_vars,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar",
image_tag="",
image_pull_policy="IfNotPresent",
sleep_interval=213,
sync_interval=212,
),
artifacts_store=bucket_managed_store,
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_artifacts=False, auth=True)),
run_path="test",
)
assert sidecar.name == SIDECAR_CONTAINER
assert sidecar.image == "sidecar/sidecar"
assert sidecar.image_pull_policy == "IfNotPresent"
assert sidecar.command == ["polyaxon", "sidecar"]
assert sidecar.args == get_sidecar_args(
container_id=MAIN_JOB_CONTAINER,
sleep_interval=213,
sync_interval=212,
monitor_logs=False,
)
assert (sidecar.env == get_sidecar_env_vars(
env_vars=env_vars,
container_id=MAIN_JOB_CONTAINER,
artifacts_store_name=bucket_managed_store.name,
) + get_items_from_secret(secret=resource1) + get_connection_env_var(
connection=bucket_managed_store, secret=resource1))
assert sidecar.env_from == [get_env_from_secret(secret=resource1)]
assert sidecar.resources == get_sidecar_resources()
assert sidecar.volume_mounts == [
get_auth_context_mount(read_only=True),
]
# outputs and no logs
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
env=env_vars,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar",
image_tag="",
image_pull_policy="IfNotPresent",
sleep_interval=213,
sync_interval=212,
),
artifacts_store=bucket_managed_store,
contexts=PluginsContextsSpec.from_config(
V1Plugins(collect_logs=False, auth=True)),
run_path="test",
)
assert sidecar.name == SIDECAR_CONTAINER
assert sidecar.image == "sidecar/sidecar"
assert sidecar.image_pull_policy == "IfNotPresent"
assert sidecar.command == ["polyaxon", "sidecar"]
assert sidecar.args == get_sidecar_args(
container_id=MAIN_JOB_CONTAINER,
sleep_interval=213,
sync_interval=212,
monitor_logs=False,
)
assert (sidecar.env == get_sidecar_env_vars(
env_vars=env_vars,
container_id=MAIN_JOB_CONTAINER,
artifacts_store_name=bucket_managed_store.name,
) + get_items_from_secret(secret=resource1) + get_connection_env_var(
connection=bucket_managed_store, secret=resource1))
assert sidecar.env_from == [get_env_from_secret(secret=resource1)]
assert sidecar.resources == get_sidecar_resources()
assert sidecar.volume_mounts == [
get_auth_context_mount(read_only=True),
get_artifacts_context_mount(read_only=False),
]
def test_get_sidecar_container_auth_context(self, ):
env_vars = [
get_env_var(name="key1", value="value1"),
get_env_var(name="key2", value="value2"),
]
resource1 = V1K8sResourceType(
name="test1",
schema=V1K8sResourceSchema(name="ref1", items=["item1", "item2"]),
is_requested=False,
)
bucket_managed_store = V1ConnectionType(
name="test_gcs",
kind=V1ConnectionKind.GCS,
schema=V1BucketConnection(bucket="gs//:foo"),
secret=resource1.schema,
)
# Default auth is included
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
env=env_vars,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar",
image_tag="",
image_pull_policy="IfNotPresent",
sleep_interval=213,
sync_interval=212,
),
artifacts_store=bucket_managed_store,
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
run_path="test",
)
assert sidecar.name == SIDECAR_CONTAINER
assert sidecar.image == "sidecar/sidecar"
assert sidecar.image_pull_policy == "IfNotPresent"
assert sidecar.command == ["polyaxon", "sidecar"]
assert sidecar.volume_mounts == [
get_auth_context_mount(read_only=True),
get_artifacts_context_mount(read_only=False),
]
# Nno auth
sidecar = get_sidecar_container(
container_id=MAIN_JOB_CONTAINER,
env=env_vars,
polyaxon_sidecar=V1PolyaxonSidecarContainer(
image="sidecar/sidecar",
image_tag="",
image_pull_policy="IfNotPresent",
sleep_interval=213,
sync_interval=-212,
),
artifacts_store=bucket_managed_store,
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=False)),
run_path="test",
)
assert sidecar.name == SIDECAR_CONTAINER
assert sidecar.image == "sidecar/sidecar"
assert sidecar.image_pull_policy == "IfNotPresent"
assert sidecar.command == ["polyaxon", "sidecar"]
assert sidecar.volume_mounts == [
get_artifacts_context_mount(read_only=False)
]
def test_get_git_init_container(self):
connection = V1ConnectionType(
name="user/foo",
kind=V1ConnectionKind.GIT,
schema=V1GitConnection(url="foo.com"),
)
container = get_git_init_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo", image_tag=""),
connection=connection,
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
)
assert (generate_container_name(INIT_GIT_CONTAINER_PREFIX,
connection.name, False)
in container.name)
assert container.image == "foo"
assert container.image_pull_policy is None
assert container.command == ["polyaxon", "initializer", "git"]
assert container.env == get_connection_env_var(connection=connection,
secret=None)
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
),
get_auth_context_mount(read_only=True),
]
container = get_git_init_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init/init",
image_tag="",
image_pull_policy="IfNotPresent"),
connection=connection,
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
)
assert (generate_container_name(INIT_GIT_CONTAINER_PREFIX,
connection.name, False)
in container.name)
assert container.image == "init/init"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["polyaxon", "initializer", "git"]
assert container.args == [
"--repo-path={}/{}".format(CONTEXT_MOUNT_ARTIFACTS,
connection.name),
"--url={}".format(connection.schema.url),
]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
),
get_auth_context_mount(read_only=True),
]
connection = V1ConnectionType(
name="user/foo",
kind=V1ConnectionKind.GIT,
schema=V1GitConnection(
url="foo.com",
revision="00b9d2ea01c40f58d6b4051319f9375675a43c02"),
)
container = get_git_init_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init/init",
image_tag="",
image_pull_policy="IfNotPresent"),
connection=connection,
mount_path="/somepath",
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
)
assert (generate_container_name(INIT_GIT_CONTAINER_PREFIX,
connection.name, False)
in container.name)
assert container.image == "init/init"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["polyaxon", "initializer", "git"]
assert container.args == [
"--repo-path=/somepath/{}".format(connection.name),
"--url={}".format(connection.schema.url),
"--revision=00b9d2ea01c40f58d6b4051319f9375675a43c02",
]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(name=get_volume_name("/somepath"),
mount_path="/somepath"),
get_auth_context_mount(read_only=True),
]
connection = V1ConnectionType(
name="user/foo",
kind=V1ConnectionKind.GIT,
schema=V1GitConnection(
url="foo.com",
revision="00b9d2ea01c40f58d6b4051319f9375675a43c02",
flags=["--falg1", "--flag2=test", "k=v"],
),
)
container = get_git_init_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init/init",
image_tag="",
image_pull_policy="IfNotPresent"),
connection=connection,
mount_path="/somepath",
contexts=PluginsContextsSpec.from_config(V1Plugins(auth=True)),
)
assert (generate_container_name(INIT_GIT_CONTAINER_PREFIX,
connection.name, False)
in container.name)
assert container.image == "init/init"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["polyaxon", "initializer", "git"]
assert container.args == [
"--repo-path=/somepath/{}".format(connection.name),
"--url={}".format(connection.schema.url),
"--revision=00b9d2ea01c40f58d6b4051319f9375675a43c02",
'--flags=["--falg1","--flag2=test","k=v"]',
]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(name=get_volume_name("/somepath"),
mount_path="/somepath"),
get_auth_context_mount(read_only=True),
]
def test_get_dockerfile_init_container(self):
dockerfile_args = V1DockerfileType(image="test/test")
container = get_dockerfile_init_container(
polyaxon_init=V1PolyaxonInitContainer(image="foo", image_tag=""),
dockerfile_args=dockerfile_args,
env=None,
contexts=PluginsContextsSpec.from_config(V1Plugins()),
run_path="test",
)
assert INIT_DOCKERFILE_CONTAINER.format("") in container.name
assert container.image == "foo"
assert container.image_pull_policy is None
assert container.command == ["polyaxon", "docker", "generate"]
assert container.args == [
"--build-context={}".format(dockerfile_args.to_dict(dump=True)),
"--destination={}".format(CONTEXT_MOUNT_ARTIFACTS),
"--copy-path={}".format(
CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("test")),
]
assert container.env == [get_run_instance_env_var()]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(
name=constants.CONTEXT_VOLUME_ARTIFACTS,
mount_path=CONTEXT_MOUNT_ARTIFACTS,
),
get_auth_context_mount(read_only=True),
]
dockerfile_args = V1DockerfileType(
image="test/test",
lang_env="LANG",
run=["step1", "step2"],
env=[["key1", "val1"], ["key2", "val2"]],
uid=2222,
gid=2222,
)
container = get_dockerfile_init_container(
polyaxon_init=V1PolyaxonInitContainer(
image="init/init",
image_tag="",
image_pull_policy="IfNotPresent"),
env=[],
dockerfile_args=dockerfile_args,
mount_path="/somepath",
contexts=PluginsContextsSpec.from_config(V1Plugins()),
run_path="test",
)
assert INIT_DOCKERFILE_CONTAINER.format("") in container.name
assert container.image == "init/init"
assert container.image_pull_policy == "IfNotPresent"
assert container.command == ["polyaxon", "docker", "generate"]
assert container.args == [
"--build-context={}".format(dockerfile_args.to_dict(dump=True)),
"--destination=/somepath",
"--copy-path={}".format(
CONTEXT_MOUNT_RUN_OUTPUTS_FORMAT.format("test")),
]
assert container.env == [get_run_instance_env_var()]
assert container.resources == get_init_resources()
assert container.volume_mounts == [
get_connections_context_mount(name=get_volume_name("/somepath"),
mount_path="/somepath"),
get_auth_context_mount(read_only=True),
]
def get_git_init_container(
polyaxon_init: V1PolyaxonInitContainer,
connection: V1ConnectionType,
contexts: PluginsContextsSpec,
container: Optional[k8s_schemas.V1Container] = None,
env: List[k8s_schemas.V1EnvVar] = None,
mount_path: str = None,
track: bool = False,
) -> k8s_schemas.V1Container:
if not connection:
raise PolypodException("A connection is required to create a repo context.")
container_name = generate_container_name(INIT_GIT_CONTAINER_PREFIX, connection.name)
if not container:
container = k8s_schemas.V1Container(name=container_name)
volume_name = (
get_volume_name(mount_path)
if mount_path
else constants.CONTEXT_VOLUME_ARTIFACTS
)
mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS
volume_mounts = [
get_connections_context_mount(name=volume_name, mount_path=mount_path)
]
if contexts and contexts.auth:
volume_mounts.append(get_auth_context_mount(read_only=True))
env = to_list(env, check_none=True)
env_from = []
secret = connection.get_secret()
if secret:
volume_mounts += to_list(
get_mount_from_resource(resource=secret), check_none=True
)
env += to_list(get_items_from_secret(secret=secret), check_none=True)
env_from = to_list(get_env_from_secret(secret=secret), check_none=True)
env += to_list(
get_connection_env_var(connection=connection, secret=secret), check_none=True
)
# Add special handling to auto-inject ssh mount path
if connection.kind == V1ConnectionKind.SSH and secret.schema.mount_path:
env += [get_env_var(POLYAXON_KEYS_SSH_PATH, secret.schema.mount_path)]
config_map = connection.get_config_map()
if config_map:
volume_mounts += to_list(
get_mount_from_resource(resource=config_map), check_none=True
)
env += to_list(
get_items_from_config_map(config_map=config_map), check_none=True
)
env_from = to_list(
get_env_from_config_map(config_map=config_map), check_none=True
)
args = get_repo_context_args(
name=connection.name,
# Handle the case of custom connection
url=getattr(connection.schema, "url", None),
revision=getattr(connection.schema, "revision", None),
flags=getattr(connection.schema, "flags", None),
mount_path=mount_path,
connection=connection.name if track else None,
)
return patch_container(
container=container,
name=container_name,
image=polyaxon_init.get_image(),
image_pull_policy=polyaxon_init.image_pull_policy,
command=["polyaxon", "initializer", "git"],
args=args,
env=env,
env_from=env_from,
volume_mounts=volume_mounts,
resources=polyaxon_init.get_resources(),
)
