def test_get_requested_secrets_and_secrets(self):
expected = get_requested_secrets(secrets=[],
connections=[self.s3_store])
assert [e.schema for e in expected] == [self.resource1.schema]
expected = get_requested_secrets(secrets=[self.resource2],
connections=[self.s3_store])
assert [e.schema for e in expected] == [self.resource1.schema]
expected = get_requested_secrets(
secrets=[self.resource2],
connections=[self.s3_store, self.gcs_store])
assert [e.schema for e in expected] == [
self.resource1.schema,
self.resource2.schema,
]
expected = get_requested_secrets(
secrets=[self.resource1, self.resource2],
connections=[self.s3_store, self.gcs_store, self.az_store],
)
assert [e.schema for e in expected] == [
self.resource1.schema,
self.resource2.schema,
self.resource3.schema,
]
def test_get_requested_secrets_non_values(self):
assert get_requested_secrets(secrets=None, connections=None) == []
assert get_requested_secrets(secrets=[], connections=[]) == []
assert (get_requested_secrets(secrets=[self.resource1, self.resource2],
connections=[]) == [])
assert (get_requested_secrets(
secrets=[], connections=[self.host_path_store,
self.claim_store]) == [])
def get_main_container(
container_id: str,
main_container: k8s_schemas.V1Container,
volume_mounts: List[k8s_schemas.V1VolumeMount],
contexts: PluginsContextsSpec,
artifacts_store: Optional[V1ConnectionType],
init: Optional[List[V1Init]],
connections: Optional[List[str]],
connection_by_names: Dict[str, V1ConnectionType],
secrets: Optional[Iterable[V1K8sResourceType]],
config_maps: Optional[Iterable[V1K8sResourceType]],
log_level: str,
run_path: Optional[str],
kv_env_vars=None,
env: List[k8s_schemas.V1EnvVar] = None,
ports: List[int] = None,
) -> k8s_schemas.V1Container:
"""Pod job container for task."""
connections = connections or []
connection_by_names = connection_by_names or {}
secrets = secrets or []
config_maps = config_maps or []
if artifacts_store and not run_path:
raise PolypodException("Run path is required for main container.")
if artifacts_store and not contexts.collect_artifacts:
if artifacts_store.name not in connection_by_names:
connection_by_names[artifacts_store.name] = artifacts_store
connections.append(artifacts_store.name)
requested_connections = [connection_by_names[c] for c in connections]
requested_config_maps = get_requested_config_maps(
config_maps=config_maps, connections=requested_connections)
requested_secrets = get_requested_secrets(
secrets=secrets, connections=requested_connections)
# Mounts
volume_mounts = to_list(volume_mounts, check_none=True)
volume_mounts = volume_mounts + get_volume_mounts(
contexts=contexts,
init=init,
connections=requested_connections,
secrets=requested_secrets,
config_maps=requested_config_maps,
)
# Env vars
env = to_list(env, check_none=True)
env = env + get_env_vars(
contexts=contexts,
log_level=log_level,
kv_env_vars=kv_env_vars,
connections=requested_connections,
secrets=requested_secrets,
config_maps=requested_config_maps,
)
# Env from
env_from = get_env_from_k8s_resources(secrets=requested_secrets,
config_maps=requested_config_maps)
ports = [
k8s_schemas.V1ContainerPort(container_port=port)
for port in to_list(ports, check_none=True)
]
return patch_container(
container=main_container,
name=container_id,
env=env,
env_from=env_from,
volume_mounts=volume_mounts,
ports=ports or None,
)
def get_pod_volumes(
contexts: PluginsContextsSpec,
artifacts_store: Optional[V1ConnectionType],
init_connections: Optional[List[V1Init]],
connections: List[str],
connection_by_names: Optional[Dict[str, V1ConnectionType]],
secrets: Optional[Iterable[V1K8sResourceType]],
config_maps: Optional[Iterable[V1K8sResourceType]],
volumes: List[k8s_schemas.V1Volume] = None,
):
"""Resolve all volumes that need to be mounted"""
connections = to_list(connections, check_none=True)
init_connections = to_list(init_connections, check_none=True)
secrets = to_list(secrets, check_none=True)
config_maps = to_list(config_maps, check_none=True)
volumes = to_list(volumes, check_none=True)[:]
connection_by_names = connection_by_names or {}
requested_connection_names = connections[:]
for init_connection in init_connections:
if (init_connection.connection and init_connection.connection
not in requested_connection_names):
requested_connection_names.append(init_connection.connection)
if artifacts_store and artifacts_store.name not in requested_connection_names:
requested_connection_names.append(artifacts_store.name)
requested_connections = [
connection_by_names[c] for c in requested_connection_names
]
requested_config_maps = get_requested_config_maps(
config_maps=config_maps, connections=requested_connections)
requested_secrets = get_requested_secrets(
secrets=secrets, connections=requested_connections)
def add_volume_from_connection(connection: V1ConnectionType):
volume = get_volume_from_connection(connection=connection)
if volume:
volumes.append(volume)
def add_volume_from_resource(resource: V1K8sResourceType, is_secret: bool):
if is_secret:
volume = get_volume_from_secret(secret=resource)
else:
volume = get_volume_from_config_map(config_map=resource)
if volume:
volumes.append(volume)
volume_names = set()
connection_ids = set()
# Handle context volumes from init section
for init_connection in init_connections:
volume_name = (get_volume_name(init_connection.path)
if init_connection.path else
constants.CONTEXT_VOLUME_ARTIFACTS)
if volume_name in volume_names:
continue
volume_names.add(volume_name)
volumes.append(get_connections_context_volume(name=volume_name))
# Add volumes from artifact stores
for c_name in connection_by_names:
connection = connection_by_names[c_name]
if connection.name not in connection_ids:
connection_ids.add(connection.name)
add_volume_from_connection(connection=connection)
# Add volumes from k8s config mount resources
for secret in requested_secrets:
add_volume_from_resource(resource=secret, is_secret=True)
for config_map in requested_config_maps:
add_volume_from_resource(resource=config_map, is_secret=False)
# Add logs/outputs stores
if contexts and (contexts.collect_artifacts or contexts.collect_logs):
if constants.CONTEXT_VOLUME_ARTIFACTS not in volume_names:
volumes.append(get_artifacts_context_volume())
volume_names.add(constants.CONTEXT_VOLUME_ARTIFACTS)
if artifacts_store and artifacts_store.name not in connection_ids:
connection_ids.add(artifacts_store.name)
add_volume_from_connection(connection=artifacts_store)
# Add utils contexts
if contexts and contexts.shm:
volumes.append(get_shm_context_volume())
if contexts and contexts.auth:
volumes.append(get_configs_context_volume())
if contexts and contexts.docker:
volumes.append(get_docker_context_volume())
return volumes
def test_get_requested_secrets(self):
expected = get_requested_secrets(secrets=[self.resource1],
connections=[self.s3_store])
assert [e.schema for e in expected] == [self.resource1.schema]
expected = get_requested_secrets(
secrets=[self.resource1, self.resource3],
connections=[self.s3_store])
assert [e.schema for e in expected] == [
self.resource3.schema,
self.resource1.schema,
]
expected = get_requested_secrets(
secrets=[self.resource2, self.resource3, self.resource4],
connections=[self.gcs_store],
)
assert [e.schema for e in expected] == [
self.resource3.schema,
self.resource4.schema,
self.resource2.schema,
]
expected = get_requested_secrets(
secrets=[self.resource1, self.resource2],
connections=[self.gcs_store])
assert [e.schema for e in expected] == [self.resource2.schema]
expected = get_requested_secrets(
secrets=[self.resource1, self.resource2],
connections=[self.s3_store, self.gcs_store],
)
assert [e.schema for e in expected] == [
self.resource1.schema,
self.resource2.schema,
]
expected = get_requested_secrets(
secrets=[self.resource1, self.resource2],
connections=[
self.s3_store,
self.gcs_store,
self.host_path_store,
self.claim_store,
],
)
assert [e.schema for e in expected] == [
self.resource1.schema,
self.resource2.schema,
]
new_az_store = V1ConnectionType(
name="test_az",
kind=V1ConnectionKind.WASB,
schema=V1BucketConnection(
bucket="Conwasb://[email protected]"),
secret=self.resource1,
)
expected = get_requested_secrets(
secrets=[self.resource1, self.resource2],
connections=[
self.s3_store,
self.gcs_store,
new_az_store,
self.host_path_store,
self.claim_store,
],
)
assert [e.schema for e in expected] == [
self.resource1.schema,
self.resource2.schema,
]
# Using a requested secret with same id
expected = get_requested_secrets(
secrets=[self.resource5, self.resource2],
connections=[
self.s3_store,
self.gcs_store,
new_az_store,
self.host_path_store,
self.claim_store,
],
)
assert [e.schema for e in expected] == [
self.resource5.schema,
self.resource2.schema,
]