def patch_container( container: k8s_schemas.V1Container, name: str = None, command: List[str] = None, args: List[str] = None, image: str = None, image_pull_policy: str = None, env: List[k8s_schemas.V1EnvVar] = None, env_from: List[k8s_schemas.V1EnvFromSource] = None, volume_mounts: List[k8s_schemas.V1VolumeMount] = None, ports: List[k8s_schemas.V1ContainerPort] = None, resources: k8s_schemas.V1ResourceRequirements = None, ) -> k8s_schemas.V1Container: container.name = name or container.name container.env = to_list(container.env, check_none=True) + to_list( env, check_none=True) container.env_from = to_list(container.env_from, check_none=True) + to_list(env_from, check_none=True) container.volume_mounts = to_list( container.volume_mounts, check_none=True) + to_list(volume_mounts, check_none=True) container.ports = to_list(container.ports, check_none=True) + to_list( ports, check_none=True) container.resources = container.resources or resources container.image_pull_policy = container.image_pull_policy or image_pull_policy container.image = container.image or image if not any([container.command, container.args]): container.command = command container.args = args return sanitize_container(container)
def sanitize_container_command_args( container: k8s_schemas.V1Container, ) -> k8s_schemas.V1Container: # Sanitize container command/args if container.command: container.command = [ str(c) for c in to_list(container.command, check_none=True) if c ] if container.args: container.args = [str(c) for c in to_list(container.args, check_none=True) if c] return container
def get_default_notification_container(): return V1Container( name=MAIN_JOB_CONTAINER, image="polyaxon/polyaxon-events-handlers:{}".format("dev"), image_pull_policy=PullPolicy.ALWAYS.value, command=["polyaxon", "notify"], args=[ "--kind={{kind}}", "--owner={{owner}}", "--project={{project}}", "--run_uuid={{run_uuid}}", "--run_name={{run_name}}", "--condition={{condition}}", ], resources=k8s_schemas.V1ResourceRequirements( limits={ "cpu": "0.5", "memory": "100Mi" }, requests={ "cpu": "0.1", "memory": "20Mi" }, ), )
def get_default_notification_container(): return V1Container( name=MAIN_JOB_CONTAINER, image="polyaxon/polyaxon-events-handlers:{}".format(pkg.VERSION), image_pull_policy=PullPolicy.IF_NOT_PRESENT.value, command=["polyaxon", "notify"], args=[ "--kind={{kind}}", "--owner={{owner}}", "--project={{project}}", "--run-uuid={{run_uuid}}", "{{params.condition.as_arg}}", "{{params.run_name.as_arg}}", ], resources=k8s_schemas.V1ResourceRequirements( limits={ "cpu": "0.5", "memory": "100Mi" }, requests={ "cpu": "0.1", "memory": "20Mi" }, ), )
def get_batch_cleaner_container( store: V1ConnectionType, paths: List[str], ): subpaths = [os.path.join(store.store_path, subpath) for subpath in paths] subpaths = " ".join(["-sp={}".format(sp) for sp in subpaths]) clean_args = "polyaxon clean-artifacts {} {}".format( store.kind.replace("_", "-"), subpaths) return V1Container( name=MAIN_JOB_CONTAINER, image="polyaxon/polyaxon-init:{}".format(pkg.VERSION), image_pull_policy=PullPolicy.IF_NOT_PRESENT.value, command=["/bin/bash", "-c"], args=[clean_args], resources=k8s_schemas.V1ResourceRequirements( limits={ "cpu": "0.5", "memory": "160Mi" }, requests={ "cpu": "0.1", "memory": "80Mi" }, ), )
def sanitize_container_env( container: k8s_schemas.V1Container, ) -> k8s_schemas.V1Container: def sanitize_env_dict(d: Dict): return { d_k: sanitize_value(d_v, handle_dict=False) if d_k in ["name", "value"] else sanitize_value(d_v, handle_dict=True) for d_k, d_v in d.items() } def sanitize_value(d, handle_dict: bool = False): if isinstance(d, str): return d if not isinstance(d, Mapping): return json.dumps(d) if not handle_dict: return json.dumps(d) return { d_k: sanitize_value(d_v, handle_dict=True) for d_k, d_v in d.items() } if container.env: env = [] for e in container.env: if isinstance(e, Mapping): e = sanitize_env_dict(e) env.append(e) elif isinstance(e, k8s_schemas.V1EnvVar): if e.value is not None: e.value = sanitize_value(e.value, handle_dict=False) env.append(e) container.env = env return container
def ensure_container_name(container: k8s_schemas.V1Container, prefix: str = None) -> k8s_schemas.V1Container: if not container: return container name = container.name if not name: container.name = generate_container_name(prefix=prefix) return container
def sanitize_container_env( container: k8s_schemas.V1Container, ) -> k8s_schemas.V1Container: if container.env: env = [] for e in container.env: if e: if isinstance(e, Mapping): e = {k: str(v) for k, v in e.items()} elif e.value: e.value = str(e.value) env.append(e) container.env = env return container
def get_default_tuner_container(command): return V1Container( name=MAIN_JOB_CONTAINER, image="polyaxon/polyaxon-hpsearch:{}".format("dev"), image_pull_policy=PullPolicy.ALWAYS.value, command=command, args=[ "--parallel={{parallel}}", "--configs={{configs}}", "--metrics={{metrics}}", ], resources=k8s_schemas.V1ResourceRequirements( requests={"cpu": "0.1", "memory": "180Mi"}, ), )
def test_refs_pipeline(self): run_config = V1CompiledOperation.read( [ os.path.abspath("tests/fixtures/pipelines/ref_pipeline.yml"), {"kind": "compiled_operation"}, ] ) with patch("polyaxon.config_reader.spec.ConfigSpec.read") as config_read: config_read.return_value = V1Component( kind="component", version="1.05", inputs=[V1IO(name="str-input", iotype="str")], run=V1Job(container=V1Container(name="test")), ).to_dict() compiled_op = CompiledOperationSpecification.apply_context(run_config) assert compiled_op.run is not None assert len(compiled_op.run.operations) == 2 assert compiled_op.run.operations[0].name == "ref-path-op" assert compiled_op.run.operations[1].name == "ref-url-op"
def get_default_tuner_container(command, bracket_iteration: int = None): args = [ "{{params.matrix.as_arg}}", "{{params.search.as_arg}}", "{{params.iteration.as_arg}}", ] if bracket_iteration is not None: args.append("{{params.bracket_iteration.as_arg}}") return V1Container( name=MAIN_JOB_CONTAINER, image="polyaxon/polyaxon-hpsearch:{}".format(pkg.VERSION), image_pull_policy=PullPolicy.IF_NOT_PRESENT.value, command=command, args=args, resources=k8s_schemas.V1ResourceRequirements(requests={ "cpu": "0.1", "memory": "180Mi" }, ), )
def get_default_cleaner_container(store: V1ConnectionType, run_path: str): subpath = os.path.join(store.store_path, run_path) return V1Container( name=MAIN_JOB_CONTAINER, image="polyaxon/polyaxon-init:{}".format(pkg.VERSION), image_pull_policy=PullPolicy.ALWAYS.value, command=["polyaxon", "clean-artifacts", store.kind.replace('_', '-')], args=["--subpath={}".format(subpath)], resources=k8s_schemas.V1ResourceRequirements( limits={ "cpu": "0.5", "memory": "100Mi" }, requests={ "cpu": "0.1", "memory": "20Mi" }, ), )
def get_default_cleaner_container(store: V1ConnectionType, run_uuid: str, run_kind: str): subpath = os.path.join(store.store_path, run_uuid) clean_args = "polyaxon clean-artifacts {} --subpath={}".format( store.kind.replace("_", "-"), subpath) wait_args = "polyaxon wait --uuid={} --kind={}".format(run_uuid, run_kind) return V1Container( name=MAIN_JOB_CONTAINER, image="polyaxon/polyaxon-init:{}".format(pkg.VERSION), image_pull_policy=PullPolicy.IF_NOT_PRESENT.value, command=["/bin/bash", "-c"], args=["{} && {}".format(wait_args, clean_args)], resources=k8s_schemas.V1ResourceRequirements( limits={ "cpu": "0.5", "memory": "160Mi" }, requests={ "cpu": "0.1", "memory": "80Mi" }, ), )
def sanitize_container( container: k8s_schemas.V1Container, ) -> k8s_schemas.V1Container: container = sanitize_container_command_args(container) container.resources = sanitize_resources(container.resources) return sanitize_container_env(container)
outputs = [ V1IO(name="loss", type=types.FLOAT), V1IO(name="accuracy", type=types.FLOAT), ] job = V1Job( init=[ V1Init(git=V1GitType( url="https://github.com/polyaxon/polyaxon-quick-start")) ], container=V1Container( image="polyaxon/polyaxon-quick-start", working_dir="{{ globals.artifacts_path }}", command=["python3", "polyaxon-quick-start/model.py"], args=[ "--conv1_size={{ conv1_size }}", "--conv2_size={{ conv2_size }}", "--dropout={{ dropout }}", "--hidden1_size={{ hidden1_size }}", "--optimizer={{ optimizer }}", "--conv_activation={{ conv_activation }}", "--dense_activation={{ dense_activation }}", "--learning_rate={{ learning_rate }}", "--epochs={{ epochs }}" ]), ) component = V1Component( name="typed-experiment", description="experiment with inputs", tags=["examples"], inputs=inputs, outputs=outputs, run=job, )