def _resolve_replica_connections(
self, connections: List[str], init: List[V1Init], agent_config: AgentConfig
):
connections = to_list(connections, check_none=True)
self._resolve_connections(connections=connections, agent_config=agent_config)
init = to_list(init, check_none=True)
init = [i.connection for i in init if i.connection]
self._resolve_connections(connections=init, agent_config=agent_config)
def get_base_store_container(
container: Optional[k8s_schemas.V1Container],
container_name: str,
polyaxon_init: V1PolyaxonInitContainer,
store: V1ConnectionType,
env: List[k8s_schemas.V1EnvVar],
env_from: List[k8s_schemas.V1EnvFromSource],
volume_mounts: List[k8s_schemas.V1VolumeMount],
args: List[str],
is_artifact_store: Optional[bool] = False,
) -> Optional[k8s_schemas.V1Container]:
env = env or []
env_from = env_from or []
volume_mounts = volume_mounts or []
# Artifact store needs to allow init the contexts as well, so the store is not required
if not is_artifact_store and not store:
raise PolypodException("Init store container requires a store")
secret = None
if store.is_bucket:
if not is_artifact_store:
secret = store.get_secret()
volume_mounts = volume_mounts + to_list(
get_mount_from_resource(resource=secret), check_none=True)
env = env + to_list(get_items_from_secret(secret=secret),
check_none=True)
env_from = env_from + to_list(get_env_from_secret(secret=secret),
check_none=True)
env += to_list(get_connection_env_var(connection=store,
secret=secret),
check_none=True)
config_map = store.get_config_map()
volume_mounts = volume_mounts + to_list(
get_mount_from_resource(resource=config_map), check_none=True)
env = env + to_list(
get_items_from_config_map(config_map=config_map),
check_none=True)
env_from = env_from + to_list(
get_env_from_config_map(config_map=config_map),
check_none=True)
else:
volume_mounts = volume_mounts + to_list(
get_mount_from_store(store=store), check_none=True)
env += to_list(get_connection_env_var(connection=store, secret=secret),
check_none=True)
return patch_container(
container=container,
name=container_name,
image=polyaxon_init.get_image(),
image_pull_policy=polyaxon_init.image_pull_policy,
command=["/bin/sh", "-c"],
args=args,
env=env,
env_from=env_from,
resources=polyaxon_init.get_resources(),
volume_mounts=volume_mounts,
)
def sanitize_container_command_args(
container: k8s_schemas.V1Container,
) -> k8s_schemas.V1Container:
# Sanitize container command/args
if container.command:
container.command = [
str(c) for c in to_list(container.command, check_none=True) if c
]
if container.args:
container.args = [str(c) for c in to_list(container.args, check_none=True) if c]
return container
def get_pod_spec(
resource_name: str,
namespace: str,
main_container: k8s_schemas.V1Container,
sidecar_containers: Optional[List[k8s_schemas.V1Container]],
init_containers: Optional[List[k8s_schemas.V1Container]],
environment: Optional[V1Environment],
labels: Dict[str, str],
volumes: Optional[List[k8s_schemas.V1Volume]],
) -> Tuple[k8s_schemas.V1ObjectMeta, k8s_schemas.V1PodSpec]:
if not main_container:
raise PolypodException("A main container is required")
environment = environment or V1Environment()
metadata = k8s_schemas.V1ObjectMeta(
name=resource_name,
namespace=namespace,
labels=labels,
annotations=environment.annotations,
)
init_containers = to_list(init_containers, check_none=True)
containers = [main_container] + to_list(sidecar_containers,
check_none=True)
image_pull_secrets = None
if environment.image_pull_secrets:
image_pull_secrets = [
k8s_schemas.V1LocalObjectReference(name=i)
for i in environment.image_pull_secrets
]
pod_spec = k8s_schemas.V1PodSpec(
init_containers=init_containers,
containers=containers,
volumes=volumes,
restart_policy=environment.restart_policy,
image_pull_secrets=image_pull_secrets,
security_context=environment.security_context,
service_account_name=environment.service_account_name,
node_selector=environment.node_selector,
tolerations=environment.tolerations,
affinity=environment.affinity,
dns_config=environment.dns_config,
dns_policy=environment.dns_policy,
host_aliases=environment.host_aliases,
host_network=environment.host_network,
host_pid=environment.host_pid,
node_name=environment.node_name,
priority=environment.priority,
priority_class_name=environment.priority_class_name,
scheduler_name=environment.scheduler_name,
)
return metadata, pod_spec
def patch_container(
container: k8s_schemas.V1Container,
name: str = None,
command: List[str] = None,
args: List[str] = None,
image: str = None,
image_pull_policy: str = None,
env: List[k8s_schemas.V1EnvVar] = None,
env_from: List[k8s_schemas.V1EnvFromSource] = None,
volume_mounts: List[k8s_schemas.V1VolumeMount] = None,
ports: List[k8s_schemas.V1ContainerPort] = None,
resources: k8s_schemas.V1ResourceRequirements = None,
) -> k8s_schemas.V1Container:
container.name = name or container.name
container.env = to_list(container.env, check_none=True) + to_list(
env, check_none=True)
container.env_from = to_list(container.env_from,
check_none=True) + to_list(env_from,
check_none=True)
container.volume_mounts = to_list(
container.volume_mounts, check_none=True) + to_list(volume_mounts,
check_none=True)
container.ports = to_list(container.ports, check_none=True) + to_list(
ports, check_none=True)
container.resources = container.resources or resources
container.image_pull_policy = container.image_pull_policy or image_pull_policy
container.image = container.image or image
if not any([container.command, container.args]):
container.command = command
container.args = args
return sanitize_container(container)
def read_from(cls, config_values, config_type=None):
"""
Reads an ordered list of configuration values and
deep merge the values in reverse order.
"""
if not config_values:
raise PolyaxonSchemaError(
"Cannot read config_value: `{}`".format(config_values)
)
config_values = to_list(config_values, check_none=True)
config = {}
for config_value in config_values:
config_value = ConfigSpec.get_from(
value=config_value, config_type=config_type
)
config_value.check_type()
config_results = config_value.read()
if config_results and isinstance(config_results, Mapping):
config = deep_update(config, config_results)
elif config_value.check_if_exists:
raise PolyaxonSchemaError(
"Cannot read config_value: `{}`".format(config_value.value)
)
return config
def get_dockerfile_init_container(
polyaxon_init: V1PolyaxonInitContainer,
dockerfile_args: V1DockerfileType,
contexts: PluginsContextsSpec,
run_path: str,
env: List[k8s_schemas.V1EnvVar] = None,
mount_path: Optional[str] = None,
) -> k8s_schemas.V1Container:
env = to_list(env, check_none=True)
env = env + [get_run_instance_env_var()]
volume_name = (get_volume_name(mount_path)
if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS
volume_mounts = [
get_connections_context_mount(name=volume_name, mount_path=mount_path)
]
if contexts and contexts.auth:
volume_mounts.append(get_auth_context_mount(read_only=True))
return k8s_schemas.V1Container(
name=generate_container_name(INIT_DOCKERFILE_CONTAINER_PREFIX),
image=polyaxon_init.get_image(),
image_pull_policy=polyaxon_init.image_pull_policy,
command=["polyaxon", "docker", "generate"],
args=[
"--build-context={}".format(dockerfile_args.to_dict(dump=True)),
"--destination={}".format(mount_path),
"--copy-path={}".format(
CONTEXT_MOUNT_ARTIFACTS_FORMAT.format(run_path) + "/outputs"),
],
env=env,
resources=polyaxon_init.get_resources(),
volume_mounts=volume_mounts,
)
def get_repo_context_args(
name: str,
url: str,
revision: str,
mount_path: str,
connection: str = None,
flags: List[str] = None,
) -> List[str]:
if not name:
raise PolypodException("A repo name is required to create a repo context.")
if not url:
raise PolypodException("A repo url is required to create a repo context.")
args = ["--repo-path={}/{}".format(mount_path, name), "--url={}".format(url)]
if revision:
args.append("--revision={}".format(revision))
if connection:
args.append("--connection={}".format(connection))
flags = to_list(flags, check_none=True)
if flags:
args.append("--flags={}".format(ujson.dumps(flags)))
return args
def _get_valid_config(cls, config, *fields) -> ConfigType:
config = to_list(config)
web_hooks = []
for web_hook in config:
if not web_hook.get("url"):
logger.warning(
"Settings contains a non compatible web hook: `%s`", web_hook
)
continue
url = web_hook["url"]
if not validate_url(url):
raise PolyaxonNotificationException(
"{} received invalid URL `{}`.".format(cls.name, url)
)
method = web_hook.get("method", "POST")
if not isinstance(method, str):
raise PolyaxonNotificationException(
"{} received invalid method `{}`.".format(cls.name, method)
)
_method = method.upper()
if _method not in ["GET", "POST"]:
raise PolyaxonNotificationException(
"{} received non compatible method `{}`.".format(cls.name, method)
)
result_web_hook = {"url": url, "method": _method}
for field in fields:
if field in web_hook:
result_web_hook[field] = web_hook[field]
web_hooks.append(result_web_hook)
return web_hooks
def test_to_list(self):
assert to_list(None) == [None]
assert to_list(None, check_none=True) == []
assert to_list([]) == []
assert to_list(()) == []
assert to_list([1, 3]) == [1, 3]
assert to_list((1, 3)) == [1, 3]
assert to_list(1) == [1]
assert to_list("foo") == ["foo"]
def get_volume_mounts(
contexts: PluginsContextsSpec,
init: Optional[List[V1Init]],
connections: Iterable[V1ConnectionType],
secrets: Iterable[V1K8sResourceType],
config_maps: Iterable[V1K8sResourceType] = None,
) -> List[k8s_schemas.V1VolumeMount]:
init = init or []
connections = connections or []
secrets = secrets or []
config_maps = config_maps or []
volume_mounts = []
volume_names = set()
if contexts and contexts.collect_artifacts:
volume_mounts += to_list(
get_artifacts_context_mount(read_only=False), check_none=True
)
volume_names.add(constants.CONTEXT_VOLUME_ARTIFACTS)
for init_connection in init:
volume_name = (
get_volume_name(init_connection.path)
if init_connection.path
else constants.CONTEXT_VOLUME_ARTIFACTS
)
mount_path = init_connection.path or CONTEXT_MOUNT_ARTIFACTS
if volume_name in volume_names:
continue
volume_names.add(volume_name)
volume_mounts += to_list(
get_connections_context_mount(name=volume_name, mount_path=mount_path),
check_none=True,
)
for store in connections:
volume_mounts += to_list(get_mount_from_store(store=store), check_none=True)
for secret in secrets:
volume_mounts += to_list(
get_mount_from_resource(resource=secret), check_none=True
)
for config_map in config_maps:
volume_mounts += to_list(
get_mount_from_resource(resource=config_map), check_none=True
)
return volume_mounts
def get_init_containers(
self,
polyaxon_init: V1PolyaxonInitContainer,
contexts: PluginsContextsSpec,
artifacts_store: V1ConnectionType,
init_connections: List[V1Init],
init_containers: List[k8s_schemas.V1Container],
connection_by_names: Dict[str, V1ConnectionType],
) -> List[k8s_schemas.V1Container]:
init_containers = [
ensure_container_name(container=c, prefix=INIT_PREFIX)
for c in to_list(init_containers, check_none=True)
]
init_connections = to_list(init_connections, check_none=True)
containers = []
# Add auth context
if contexts and contexts.auth:
containers.append(
get_auth_context_container(
polyaxon_init=polyaxon_init,
env=self.get_auth_service_env_vars(
external_host=contexts.external_host),
))
# Add outputs
if contexts and contexts.collect_artifacts:
containers += to_list(
get_artifacts_path_container(
polyaxon_init=polyaxon_init,
artifacts_store=artifacts_store,
run_path=self.run_path,
auto_resume=contexts.auto_resume,
),
check_none=True,
)
containers += self.handle_init_connections(
polyaxon_init=polyaxon_init,
artifacts_store=artifacts_store,
init_connections=init_connections,
connection_by_names=connection_by_names,
contexts=contexts,
)
init_containers = containers + init_containers
return [sanitize_container(c) for c in init_containers]
def render(self):
docker_template = jinja2.Template(POLYAXON_DOCKER_TEMPLATE)
return docker_template.render(
image=self.build_context.image,
copy=to_list(self.build_context.copy, check_none=True),
run=to_list(self.build_context.run, check_none=True),
env=to_list(self.build_context.env,
check_none=True,
check_dict=True),
workdir=self.build_context.workdir,
path=to_list(self.build_context.path, check_none=True),
workdir_path=self.build_context.workdir_path,
lang_env=self.build_context.lang_env,
uid=self.build_context.uid,
gid=self.build_context.gid,
shell=self.build_context.shell,
)
def __init__(self, filepaths):
filepaths = to_list(filepaths)
for filepath in filepaths:
if not os.path.isfile(filepath):
raise PolyaxonfileError("`{}` must be a valid file".format(filepath))
self._filenames = [os.path.basename(filepath) for filepath in filepaths]
self.config = get_specification(data=reader.read(filepaths))
def get_sidecar_containers(
self,
polyaxon_sidecar: V1PolyaxonSidecarContainer,
contexts: PluginsContextsSpec,
artifacts_store: V1ConnectionType,
sidecar_containers: List[k8s_schemas.V1Container],
) -> List[k8s_schemas.V1Container]:
polyaxon_sidecar_container = get_sidecar_container(
polyaxon_sidecar=polyaxon_sidecar,
env=self.get_polyaxon_sidecar_service_env_vars(),
artifacts_store=artifacts_store,
contexts=contexts,
run_path=self.run_path,
)
containers = to_list(polyaxon_sidecar_container, check_none=True)
containers += to_list(sidecar_containers, check_none=True)
return containers
def validate_tags(tags):
if not tags:
return None
if isinstance(tags, str):
tags = [tag.strip() for tag in tags.split(",")]
tags = to_list(tags)
tags = [tag for tag in tags if (tag and isinstance(tag, str))]
return tags
def get_files_in_path(path: str, exclude: List[str] = None) -> List[str]:
result_files = []
for root, dirs, files in os.walk(path, topdown=True):
exclude = to_list(exclude, check_none=True)
if exclude:
dirs[:] = [d for d in dirs if d not in exclude]
logger.debug("Root:%s, Dirs:%s", root, dirs)
for file_name in files:
result_files.append(os.path.join(root, file_name))
return result_files
def metric_condition(
queryset: Any,
params: Union[str, Iterable],
negation: bool,
query_backend: Any,
timezone: str = None,
) -> Any:
params = to_list(params)
if len(params) == 1 and to_bool(params[0]) is True:
return queryset.filter(metric_annotations__name=True)
return queryset
def independent_condition(
params: Union[str, Iterable],
negation: bool,
query_backend: Any,
timezone: str = None,
queryset: Any = None,
) -> Any:
params = to_list(params)
if len(params) == 1 and to_bool(params[0]) is True:
return queryset.filter(experiment_group__isnull=True)
return queryset
def create(view, request, *args, **kwargs):
if not request.data:
raise ValidationError("Received no artifacts.")
data = to_list(request.data)
try:
[V1RunArtifact(r) for r in data]
except MarshmallowValidationError as e:
raise ValidationError(e)
view.audit(request, *args, **kwargs, artifacts=data)
return Response(status=status.HTTP_201_CREATED)
def get_sidecar_env_vars(
env_vars: List[k8s_schemas.V1EnvVar],
container_id: str,
artifacts_store_name: str,
) -> List[k8s_schemas.V1EnvVar]:
env_vars = to_list(env_vars, check_none=True)[:]
env_vars.append(get_env_var(name=POLYAXON_KEYS_CONTAINER_ID, value=container_id))
env_vars.append(
get_env_var(name=POLYAXON_KEYS_ARTIFACTS_STORE_NAME, value=artifacts_store_name)
)
return env_vars
def _resolve_connections_contexts(
contexts: Dict,
connections: List[str],
connection_by_names: Dict[str, V1ConnectionType],
key: str = "connections",
) -> Dict:
connections = to_list(connections, check_none=True)
for connection in connections:
if connection_by_names[connection].schema:
contexts[key][connection] = connection_by_names[
connection].schema.to_dict()
return contexts
def read(cls, values, is_preset: bool = False):
if is_preset:
if isinstance(values, cls.CONFIG):
values.is_preset = True
return values
elif isinstance(values, Mapping):
values[cls.IS_PRESET] = True
else:
values = to_list(values)
values = [{cls.IS_PRESET: True}] + values
return super().read(values)
def collect_artifacts_from_io_section(
io_section: List[V1IO],
connection_by_names: Dict[str, V1ConnectionType],
is_input: bool,
) -> List[V1RunArtifact]:
io_section = to_list(io_section, check_none=True)
artifacts = [
collect_artifacts_from_io(io,
connection_by_names=connection_by_names,
is_input=is_input) for io in io_section
]
return [a for a in artifacts if a]
def get_auth_context_container(
polyaxon_init: V1PolyaxonInitContainer, env: List[k8s_schemas.V1EnvVar] = None
) -> k8s_schemas.V1Container:
env = to_list(env, check_none=True)
return k8s_schemas.V1Container(
name=INIT_AUTH_CONTAINER,
image=polyaxon_init.get_image(),
image_pull_policy=polyaxon_init.image_pull_policy,
command=["polyaxon", "initializer", "auth"],
env=env,
resources=polyaxon_init.get_resources(),
volume_mounts=[get_auth_context_mount(read_only=False)],
)
def get_sidecar_containers(
self,
polyaxon_sidecar: V1PolyaxonSidecarContainer,
contexts: PluginsContextsSpec,
artifacts_store: V1ConnectionType,
sidecar_containers: List[k8s_schemas.V1Container],
) -> List[k8s_schemas.V1Container]:
sidecar_containers = [
ensure_container_name(container=c, prefix=SIDECAR_PREFIX)
for c in to_list(sidecar_containers, check_none=True)
]
polyaxon_sidecar_container = get_sidecar_container(
container_id=self.MAIN_CONTAINER_ID,
polyaxon_sidecar=polyaxon_sidecar,
env=self.get_polyaxon_sidecar_service_env_vars(),
artifacts_store=artifacts_store,
contexts=contexts,
run_path=self.run_path,
)
containers = to_list(polyaxon_sidecar_container, check_none=True)
containers += sidecar_containers
return [sanitize_container_command_args(c) for c in containers]
def get_custom_init_container(
connection: V1ConnectionType,
contexts: PluginsContextsSpec,
container: Optional[k8s_schemas.V1Container],
env: List[k8s_schemas.V1EnvVar] = None,
mount_path: str = None,
) -> k8s_schemas.V1Container:
if not connection:
raise PolypodException(
"A connection is required to create a repo context.")
volume_name = (get_volume_name(mount_path)
if mount_path else constants.CONTEXT_VOLUME_ARTIFACTS)
mount_path = mount_path or CONTEXT_MOUNT_ARTIFACTS
volume_mounts = [
get_connections_context_mount(name=volume_name, mount_path=mount_path)
]
if contexts and contexts.auth:
volume_mounts.append(get_auth_context_mount(read_only=True))
env = to_list(env, check_none=True)
env_from = []
secret = connection.get_secret()
if secret:
volume_mounts += to_list(get_mount_from_resource(resource=secret),
check_none=True)
env += to_list(get_items_from_secret(secret=secret), check_none=True)
env_from = to_list(get_env_from_secret(secret=secret), check_none=True)
env += to_list(get_connection_env_var(connection=connection,
secret=secret),
check_none=True)
config_map = connection.get_config_map()
if config_map:
volume_mounts += to_list(get_mount_from_resource(resource=config_map),
check_none=True)
env += to_list(get_items_from_config_map(config_map=config_map),
check_none=True)
env_from = to_list(get_env_from_config_map(config_map=config_map),
check_none=True)
container_name = container.name or generate_container_name(
INIT_CUSTOM_CONTAINER_PREFIX, connection.name)
return patch_container(
container=container,
name=container_name,
env=env,
env_from=env_from,
volume_mounts=volume_mounts,
)
def _wait_for_condition(self, statuses: List[str] = None):
statuses = to_list(statuses, check_none=True)
def condition():
if statuses:
return last_status in statuses
return LifeCycle.is_done(last_status)
last_status = None
while not condition():
if last_status:
time.sleep(settings.CLIENT_CONFIG.watch_interval)
last_status, _conditions = self.get_statuses(last_status)
yield last_status, _conditions
def get_container_command_args(config):
def sanitize_str(value):
if not value:
return
value = strip_spaces(value=value, join=False)
value = [c.strip().strip("\\") for c in value if (c and c != "\\")]
value = [c for c in value if (c and c != "\\")]
return " ".join(value)
def sanitize(value):
return ([sanitize_str(v) for v in value] if isinstance(value, list)
else to_list(sanitize_str(value), check_none=True))
return to_list(config.command, check_none=True), sanitize(config.args)
def _resolve_init_contexts(
cls,
contexts: Dict,
init: List[V1Init],
connection_by_names: Dict[str, V1ConnectionType],
):
init = to_list(init, check_none=True)
connections = [i.connection for i in init if i.connection]
return cls._resolve_connections_contexts(
contexts=contexts,
connections=connections,
connection_by_names=connection_by_names,
key="init",
)