def get_tuner(
name: str,
container: V1Container,
matrix: V1Matrix,
search: V1ParamSearch,
iteration: int,
bracket_iteration: int = None,
) -> V1Operation:
params = {
"matrix": V1Param(value=matrix.to_dict()),
"search": V1Param(value=search.to_dict()),
"iteration": V1Param(value=iteration),
}
inputs = [
V1IO(name="matrix", iotype=types.DICT, is_list=False,
is_optional=True),
V1IO(name="search", iotype=types.DICT, is_list=False,
is_optional=True),
V1IO(name="iteration",
iotype=types.INT,
is_list=False,
is_optional=True),
]
if bracket_iteration is not None:
params["bracket_iteration"] = V1Param(value=bracket_iteration)
inputs.append(
V1IO(
name="bracket_iteration",
iotype=types.INT,
is_list=False,
is_optional=True,
))
return V1Operation(
params=params,
component=V1Component(
name=name,
plugins=V1Plugins(
auth=True,
collect_logs=True,
collect_artifacts=True,
collect_resources=False,
sync_statuses=False,
),
inputs=inputs,
outputs=[
V1IO(
name="suggestions",
iotype=types.DICT,
is_list=True,
is_optional=False,
),
],
run=V1Tuner(container=container, ),
),
)
def get_tuner(
name: str,
container: V1Container,
matrix: V1Matrix,
configs: List[Dict],
metrics: List[float],
iteration: int,
) -> V1Operation:
return V1Operation(
params={
"configs": V1Param(value=configs),
"metrics": V1Param(value=metrics),
"matrix": V1Param(value=matrix),
"iteration": V1Param(value=iteration),
},
termination=V1Termination(max_retries=3),
component=V1Component(
name=name,
plugins=V1Plugins(
auth=True,
collect_logs=False,
collect_artifacts=False,
collect_resources=False,
sync_statuses=True,
),
inputs=[
V1IO(name="configs",
iotype=types.DICT,
is_list=True,
is_optional=False),
V1IO(name="metrics",
iotype=types.FLOAT,
is_list=True,
is_optional=False),
V1IO(name="iteration",
iotype=types.INT,
is_list=True,
is_optional=True),
],
outputs=[
V1IO(
name="suggestions",
iotype=types.DICT,
is_list=True,
is_optional=False,
),
],
run=V1Tuner(container=container, ),
),
)
def get_context_io(c_name: str, c_io: V1Param, is_list=None):
if not c_io.context_only:
return
contexts.append(
V1IO(
name=c_name,
to_init=c_io.to_init,
connection=c_io.connection,
is_list=is_list,
))
def compile_operation(cls,
config: V1Operation,
override: Dict = None) -> V1CompiledOperation:
if override:
preset = OperationSpecification.read(override, is_preset=True)
config = config.patch(preset, preset.patch_strategy)
# Patch run
component = config.component # type: V1Component
if config.run_patch:
component.run = component.run.patch(
validate_run_patch(config.run_patch, component.run.kind),
strategy=config.patch_strategy,
)
# Gather contexts io
config_params = config.params or {}
contexts = [
V1IO(name=p) for p in config_params
if config_params[p].context_only
]
patch_compiled = V1CompiledOperation(
name=config.name,
description=config.description,
contexts=contexts,
tags=config.tags,
presets=config.presets,
queue=config.queue,
cache=config.cache,
hooks=config.hooks,
actions=config.actions,
events=config.events,
plugins=config.plugins,
termination=config.termination,
matrix=config.matrix,
schedule=config.schedule,
dependencies=config.dependencies,
trigger=config.trigger,
conditions=config.conditions,
skip_on_upstream_skip=config.skip_on_upstream_skip,
)
values = [
{
cls.VERSION: config.version
},
component.to_dict(),
{
cls.KIND: kinds.COMPILED_OPERATION
},
]
compiled = V1CompiledOperation.read(
values) # type: V1CompiledOperation
return compiled.patch(patch_compiled, strategy=config.patch_strategy)
def test_refs_pipeline(self):
run_config = V1CompiledOperation.read(
[
os.path.abspath("tests/fixtures/pipelines/ref_pipeline.yml"),
{"kind": "compiled_operation"},
]
)
with patch("polyaxon.config_reader.spec.ConfigSpec.read") as config_read:
config_read.return_value = V1Component(
kind="component",
version="1.05",
inputs=[V1IO(name="str-input", iotype="str")],
run=V1Job(container=V1Container(name="test")),
).to_dict()
compiled_op = CompiledOperationSpecification.apply_context(run_config)
assert compiled_op.run is not None
assert len(compiled_op.run.operations) == 2
assert compiled_op.run.operations[0].name == "ref-path-op"
assert compiled_op.run.operations[1].name == "ref-url-op"
def get_notifier_operation(
connection: str,
backend: str,
owner: str,
project: str,
run_uuid: str,
run_name: str,
condition: Union[str, Dict],
) -> V1Operation:
return V1Operation(
params={
"backend": V1Param(value=backend),
"owner": V1Param(value=owner),
"project": V1Param(value=project),
"uuid": V1Param(value=run_uuid),
"name": V1Param(value=run_name),
"condition": V1Param(value=condition),
},
termination=V1Termination(max_retries=3),
component=V1Component(
name="notifier",
plugins=V1Plugins(
auth=False,
collect_logs=False,
collect_artifacts=False,
collect_resources=False,
auto_resume=False,
sync_statuses=False,
external_host=True,
),
inputs=[
V1IO(name="backend", type=types.STR, is_optional=False),
V1IO(name="owner", type=types.STR, is_optional=False),
V1IO(name="project", type=types.STR, is_optional=False),
V1IO(name="uuid", type=types.STR, is_optional=False),
V1IO(name="name", type=types.STR, is_optional=True),
V1IO(name="condition", type=types.DICT, is_optional=True),
V1IO(name="connection", type=types.STR, is_optional=True),
],
run=V1NotifierJob(
connections=[connection],
container=get_default_notification_container(),
),
),
)
def get_notifier_operation(
connection: str,
kind: str,
owner: str,
project: str,
run_uuid: str,
run_name: str,
condition: str,
) -> V1Operation:
return V1Operation(
params={
"kind": V1Param(value=kind),
"owner": V1Param(value=owner),
"project": V1Param(value=project),
"run_uuid": V1Param(value=run_uuid),
"run_name": V1Param(value=run_name),
"condition": V1Param(value=condition),
},
termination=V1Termination(max_retries=3),
component=V1Component(
name="slack-notification",
plugins=V1Plugins(
auth=False,
collect_logs=False,
collect_artifacts=False,
collect_resources=False,
sync_statuses=False,
),
inputs=[
V1IO(name="kind", iotype=types.STR, is_optional=False),
V1IO(name="owner", iotype=types.STR, is_optional=False),
V1IO(name="project", iotype=types.STR, is_optional=False),
V1IO(name="run_uuid", iotype=types.STR, is_optional=False),
V1IO(name="run_name", iotype=types.STR, is_optional=True),
V1IO(name="condition", iotype=types.STR, is_optional=True),
V1IO(name="connection", iotype=types.STR, is_optional=True),
],
run=V1Notifier(
connections=[connection],
container=get_default_notification_container(),
),
),
)
async def notify_run(
namespace: str,
owner: str,
project: str,
run_uuid: str,
run_name: str,
condition: V1StatusCondition,
connections: List[str],
):
spawner = AsyncSpawner(namespace=namespace)
await spawner.k8s_manager.setup()
for connection in connections:
connection_type = settings.AGENT_CONFIG.notification_connections_by_names.get(
connection
)
if not connection_type:
logger.warning(
"Could not create notification using connection {}, "
"the connection was not found or not set correctly.".format(
connection_type
)
)
continue
operation = V1Operation(
params={
"kind": connection_type.kind,
"owner": owner,
"project": project,
"run_uuid": run_uuid,
"run_name": run_name,
"condition": ujson.dumps(condition.to_dict()),
},
termination=V1Termination(max_retries=3),
component=V1Component(
name="slack-notification",
plugins=V1Plugins(
auth=False,
collect_logs=False,
collect_artifacts=False,
collect_resources=False,
sync_statuses=False,
),
inputs=[
V1IO(name="kind", iotype=types.STR, is_optional=False),
V1IO(name="owner", iotype=types.STR, is_optional=False),
V1IO(name="project", iotype=types.STR, is_optional=False),
V1IO(name="run_uuid", iotype=types.STR, is_optional=False),
V1IO(name="run_name", iotype=types.STR, is_optional=True),
V1IO(name="condition", iotype=types.STR, is_optional=True),
V1IO(name="connection", iotype=types.STR, is_optional=True),
],
run=V1Notifier(
connections=[connection],
container=get_default_notification_container(),
),
),
)
compiled_operation = OperationSpecification.compile_operation(operation)
resource = compiler.make(
owner_name=owner,
project_name=project,
project_uuid=project,
run_uuid=run_uuid,
run_name=run_name,
run_path=run_uuid,
compiled_operation=compiled_operation,
params=operation.params,
)
await spawner.create(
run_uuid=run_uuid,
run_kind=compiled_operation.get_run_kind(),
resource=resource,
)
from polyaxon.schemas.types import V1GitType
from polyaxon.polyflow import V1Component, V1Init, V1IO, V1Job
"""
This is the same Polyaxonfile as in typed.yaml using the Python library.
Note: Running this file using CLI is similar as well:
```bash
polyaxon run -pm experimentation/typed.py:component -P epochs=5 -l
```
* -pm: --python-module
"""
inputs = [
V1IO(name="conv1_size", type=types.INT, value=32, is_optional=True),
V1IO(name="conv2_size", type=types.INT, value=64, is_optional=True),
V1IO(name="dropout", type=types.FLOAT, value=0.2, is_optional=True),
V1IO(name="hidden1_size", type=types.INT, value=500, is_optional=True),
V1IO(name="conv_activation",
type=types.STR,
value="relu",
is_optional=True),
V1IO(name="dense_activation",
type=types.STR,
value="relu",
is_optional=True),
V1IO(name="optimizer", type=types.STR, value="adam", is_optional=True),
V1IO(name="learning_rate", type=types.FLOAT, value=0.01, is_optional=True),
V1IO(name="epochs", type=types.INT),
]