def patch_container(
container: k8s_schemas.V1Container,
name: str = None,
command: List[str] = None,
args: List[str] = None,
image: str = None,
image_pull_policy: str = None,
env: List[k8s_schemas.V1EnvVar] = None,
env_from: List[k8s_schemas.V1EnvFromSource] = None,
volume_mounts: List[k8s_schemas.V1VolumeMount] = None,
ports: List[k8s_schemas.V1ContainerPort] = None,
resources: k8s_schemas.V1ResourceRequirements = None,
) -> k8s_schemas.V1Container:
container.name = name or container.name
container.env = to_list(container.env, check_none=True) + to_list(
env, check_none=True)
container.env_from = to_list(container.env_from,
check_none=True) + to_list(env_from,
check_none=True)
container.volume_mounts = to_list(
container.volume_mounts, check_none=True) + to_list(volume_mounts,
check_none=True)
container.ports = to_list(container.ports, check_none=True) + to_list(
ports, check_none=True)
container.resources = container.resources or resources
container.image_pull_policy = container.image_pull_policy or image_pull_policy
container.image = container.image or image
if not any([container.command, container.args]):
container.command = command
container.args = args
return sanitize_container(container)
def sanitize_container_command_args(
container: k8s_schemas.V1Container,
) -> k8s_schemas.V1Container:
# Sanitize container command/args
if container.command:
container.command = [
str(c) for c in to_list(container.command, check_none=True) if c
]
if container.args:
container.args = [str(c) for c in to_list(container.args, check_none=True) if c]
return container
def get_default_notification_container():
return V1Container(
name=MAIN_JOB_CONTAINER,
image="polyaxon/polyaxon-events-handlers:{}".format("dev"),
image_pull_policy=PullPolicy.ALWAYS.value,
command=["polyaxon", "notify"],
args=[
"--kind={{kind}}",
"--owner={{owner}}",
"--project={{project}}",
"--run_uuid={{run_uuid}}",
"--run_name={{run_name}}",
"--condition={{condition}}",
],
resources=k8s_schemas.V1ResourceRequirements(
limits={
"cpu": "0.5",
"memory": "100Mi"
},
requests={
"cpu": "0.1",
"memory": "20Mi"
},
),
)
def get_default_notification_container():
return V1Container(
name=MAIN_JOB_CONTAINER,
image="polyaxon/polyaxon-events-handlers:{}".format(pkg.VERSION),
image_pull_policy=PullPolicy.IF_NOT_PRESENT.value,
command=["polyaxon", "notify"],
args=[
"--kind={{kind}}",
"--owner={{owner}}",
"--project={{project}}",
"--run-uuid={{run_uuid}}",
"{{params.condition.as_arg}}",
"{{params.run_name.as_arg}}",
],
resources=k8s_schemas.V1ResourceRequirements(
limits={
"cpu": "0.5",
"memory": "100Mi"
},
requests={
"cpu": "0.1",
"memory": "20Mi"
},
),
)
def get_batch_cleaner_container(
store: V1ConnectionType,
paths: List[str],
):
subpaths = [os.path.join(store.store_path, subpath) for subpath in paths]
subpaths = " ".join(["-sp={}".format(sp) for sp in subpaths])
clean_args = "polyaxon clean-artifacts {} {}".format(
store.kind.replace("_", "-"), subpaths)
return V1Container(
name=MAIN_JOB_CONTAINER,
image="polyaxon/polyaxon-init:{}".format(pkg.VERSION),
image_pull_policy=PullPolicy.IF_NOT_PRESENT.value,
command=["/bin/bash", "-c"],
args=[clean_args],
resources=k8s_schemas.V1ResourceRequirements(
limits={
"cpu": "0.5",
"memory": "160Mi"
},
requests={
"cpu": "0.1",
"memory": "80Mi"
},
),
)
def sanitize_container_env(
container: k8s_schemas.V1Container, ) -> k8s_schemas.V1Container:
def sanitize_env_dict(d: Dict):
return {
d_k: sanitize_value(d_v, handle_dict=False) if d_k
in ["name", "value"] else sanitize_value(d_v, handle_dict=True)
for d_k, d_v in d.items()
}
def sanitize_value(d, handle_dict: bool = False):
if isinstance(d, str):
return d
if not isinstance(d, Mapping):
return json.dumps(d)
if not handle_dict:
return json.dumps(d)
return {
d_k: sanitize_value(d_v, handle_dict=True)
for d_k, d_v in d.items()
}
if container.env:
env = []
for e in container.env:
if isinstance(e, Mapping):
e = sanitize_env_dict(e)
env.append(e)
elif isinstance(e, k8s_schemas.V1EnvVar):
if e.value is not None:
e.value = sanitize_value(e.value, handle_dict=False)
env.append(e)
container.env = env
return container
def ensure_container_name(container: k8s_schemas.V1Container,
prefix: str = None) -> k8s_schemas.V1Container:
if not container:
return container
name = container.name
if not name:
container.name = generate_container_name(prefix=prefix)
return container
def sanitize_container_env(
container: k8s_schemas.V1Container, ) -> k8s_schemas.V1Container:
if container.env:
env = []
for e in container.env:
if e:
if isinstance(e, Mapping):
e = {k: str(v) for k, v in e.items()}
elif e.value:
e.value = str(e.value)
env.append(e)
container.env = env
return container
def get_default_tuner_container(command):
return V1Container(
name=MAIN_JOB_CONTAINER,
image="polyaxon/polyaxon-hpsearch:{}".format("dev"),
image_pull_policy=PullPolicy.ALWAYS.value,
command=command,
args=[
"--parallel={{parallel}}",
"--configs={{configs}}",
"--metrics={{metrics}}",
],
resources=k8s_schemas.V1ResourceRequirements(
requests={"cpu": "0.1", "memory": "180Mi"},
),
)
def test_refs_pipeline(self):
run_config = V1CompiledOperation.read(
[
os.path.abspath("tests/fixtures/pipelines/ref_pipeline.yml"),
{"kind": "compiled_operation"},
]
)
with patch("polyaxon.config_reader.spec.ConfigSpec.read") as config_read:
config_read.return_value = V1Component(
kind="component",
version="1.05",
inputs=[V1IO(name="str-input", iotype="str")],
run=V1Job(container=V1Container(name="test")),
).to_dict()
compiled_op = CompiledOperationSpecification.apply_context(run_config)
assert compiled_op.run is not None
assert len(compiled_op.run.operations) == 2
assert compiled_op.run.operations[0].name == "ref-path-op"
assert compiled_op.run.operations[1].name == "ref-url-op"
def get_default_tuner_container(command, bracket_iteration: int = None):
args = [
"{{params.matrix.as_arg}}",
"{{params.search.as_arg}}",
"{{params.iteration.as_arg}}",
]
if bracket_iteration is not None:
args.append("{{params.bracket_iteration.as_arg}}")
return V1Container(
name=MAIN_JOB_CONTAINER,
image="polyaxon/polyaxon-hpsearch:{}".format(pkg.VERSION),
image_pull_policy=PullPolicy.IF_NOT_PRESENT.value,
command=command,
args=args,
resources=k8s_schemas.V1ResourceRequirements(requests={
"cpu": "0.1",
"memory": "180Mi"
}, ),
)
def get_default_cleaner_container(store: V1ConnectionType, run_path: str):
subpath = os.path.join(store.store_path, run_path)
return V1Container(
name=MAIN_JOB_CONTAINER,
image="polyaxon/polyaxon-init:{}".format(pkg.VERSION),
image_pull_policy=PullPolicy.ALWAYS.value,
command=["polyaxon", "clean-artifacts",
store.kind.replace('_', '-')],
args=["--subpath={}".format(subpath)],
resources=k8s_schemas.V1ResourceRequirements(
limits={
"cpu": "0.5",
"memory": "100Mi"
},
requests={
"cpu": "0.1",
"memory": "20Mi"
},
),
)
def get_default_cleaner_container(store: V1ConnectionType, run_uuid: str,
run_kind: str):
subpath = os.path.join(store.store_path, run_uuid)
clean_args = "polyaxon clean-artifacts {} --subpath={}".format(
store.kind.replace("_", "-"), subpath)
wait_args = "polyaxon wait --uuid={} --kind={}".format(run_uuid, run_kind)
return V1Container(
name=MAIN_JOB_CONTAINER,
image="polyaxon/polyaxon-init:{}".format(pkg.VERSION),
image_pull_policy=PullPolicy.IF_NOT_PRESENT.value,
command=["/bin/bash", "-c"],
args=["{} && {}".format(wait_args, clean_args)],
resources=k8s_schemas.V1ResourceRequirements(
limits={
"cpu": "0.5",
"memory": "160Mi"
},
requests={
"cpu": "0.1",
"memory": "80Mi"
},
),
)
def sanitize_container(
container: k8s_schemas.V1Container, ) -> k8s_schemas.V1Container:
container = sanitize_container_command_args(container)
container.resources = sanitize_resources(container.resources)
return sanitize_container_env(container)
outputs = [
V1IO(name="loss", type=types.FLOAT),
V1IO(name="accuracy", type=types.FLOAT),
]
job = V1Job(
init=[
V1Init(git=V1GitType(
url="https://github.com/polyaxon/polyaxon-quick-start"))
],
container=V1Container(
image="polyaxon/polyaxon-quick-start",
working_dir="{{ globals.artifacts_path }}",
command=["python3", "polyaxon-quick-start/model.py"],
args=[
"--conv1_size={{ conv1_size }}", "--conv2_size={{ conv2_size }}",
"--dropout={{ dropout }}", "--hidden1_size={{ hidden1_size }}",
"--optimizer={{ optimizer }}",
"--conv_activation={{ conv_activation }}",
"--dense_activation={{ dense_activation }}",
"--learning_rate={{ learning_rate }}", "--epochs={{ epochs }}"
]),
)
component = V1Component(
name="typed-experiment",
description="experiment with inputs",
tags=["examples"],
inputs=inputs,
outputs=outputs,
run=job,
)