def get_tuner(
name: str,
container: V1Container,
matrix: V1Matrix,
configs: List[Dict],
metrics: List[float],
iteration: int,
) -> V1Operation:
return V1Operation(
params={
"configs": V1Param(value=configs),
"metrics": V1Param(value=metrics),
"matrix": V1Param(value=matrix),
"iteration": V1Param(value=iteration),
},
termination=V1Termination(max_retries=3),
component=V1Component(
name=name,
plugins=V1Plugins(
auth=True,
collect_logs=False,
collect_artifacts=False,
collect_resources=False,
sync_statuses=True,
),
inputs=[
V1IO(name="configs",
iotype=types.DICT,
is_list=True,
is_optional=False),
V1IO(name="metrics",
iotype=types.FLOAT,
is_list=True,
is_optional=False),
V1IO(name="iteration",
iotype=types.INT,
is_list=True,
is_optional=True),
],
outputs=[
V1IO(
name="suggestions",
iotype=types.DICT,
is_list=True,
is_optional=False,
),
],
run=V1Tuner(container=container, ),
),
)
def get_notifier_operation(
connection: str,
backend: str,
owner: str,
project: str,
run_uuid: str,
run_name: str,
condition: Union[str, Dict],
) -> V1Operation:
return V1Operation(
params={
"backend": V1Param(value=backend),
"owner": V1Param(value=owner),
"project": V1Param(value=project),
"uuid": V1Param(value=run_uuid),
"name": V1Param(value=run_name),
"condition": V1Param(value=condition),
},
termination=V1Termination(max_retries=3),
component=V1Component(
name="notifier",
plugins=V1Plugins(
auth=False,
collect_logs=False,
collect_artifacts=False,
collect_resources=False,
auto_resume=False,
sync_statuses=False,
external_host=True,
),
inputs=[
V1IO(name="backend", type=types.STR, is_optional=False),
V1IO(name="owner", type=types.STR, is_optional=False),
V1IO(name="project", type=types.STR, is_optional=False),
V1IO(name="uuid", type=types.STR, is_optional=False),
V1IO(name="name", type=types.STR, is_optional=True),
V1IO(name="condition", type=types.DICT, is_optional=True),
V1IO(name="connection", type=types.STR, is_optional=True),
],
run=V1NotifierJob(
connections=[connection],
container=get_default_notification_container(),
),
),
)
def get_notifier_operation(
connection: str,
kind: str,
owner: str,
project: str,
run_uuid: str,
run_name: str,
condition: str,
) -> V1Operation:
return V1Operation(
params={
"kind": V1Param(value=kind),
"owner": V1Param(value=owner),
"project": V1Param(value=project),
"run_uuid": V1Param(value=run_uuid),
"run_name": V1Param(value=run_name),
"condition": V1Param(value=condition),
},
termination=V1Termination(max_retries=3),
component=V1Component(
name="slack-notification",
plugins=V1Plugins(
auth=False,
collect_logs=False,
collect_artifacts=False,
collect_resources=False,
sync_statuses=False,
),
inputs=[
V1IO(name="kind", iotype=types.STR, is_optional=False),
V1IO(name="owner", iotype=types.STR, is_optional=False),
V1IO(name="project", iotype=types.STR, is_optional=False),
V1IO(name="run_uuid", iotype=types.STR, is_optional=False),
V1IO(name="run_name", iotype=types.STR, is_optional=True),
V1IO(name="condition", iotype=types.STR, is_optional=True),
V1IO(name="connection", iotype=types.STR, is_optional=True),
],
run=V1Notifier(
connections=[connection],
container=get_default_notification_container(),
),
),
)
def get_cleaner_operation(connection: V1ConnectionType, run_uuid: str,
run_kind: str) -> V1Operation:
return V1Operation(
termination=V1Termination(max_retries=1),
component=V1Component(
name="cleaner",
plugins=V1Plugins(
auth=False,
collect_logs=False,
collect_artifacts=False,
collect_resources=False,
auto_resume=False,
sync_statuses=False,
),
run=V1CleanerJob(
connections=[connection.name],
container=get_default_cleaner_container(
connection, run_uuid, run_kind),
),
),
)
def get_batch_cleaner_operation(
connection: V1ConnectionType,
paths: List[str],
) -> V1Operation:
return V1Operation(
termination=V1Termination(max_retries=1),
component=V1Component(
name="cleaner",
plugins=V1Plugins(
auth=False,
collect_logs=False,
collect_artifacts=False,
collect_resources=False,
auto_resume=False,
sync_statuses=False,
),
run=V1CleanerJob(
connections=[connection.name],
container=get_batch_cleaner_container(connection, paths),
),
),
)
async def notify_run(
namespace: str,
owner: str,
project: str,
run_uuid: str,
run_name: str,
condition: V1StatusCondition,
connections: List[str],
):
spawner = AsyncSpawner(namespace=namespace)
await spawner.k8s_manager.setup()
for connection in connections:
connection_type = settings.AGENT_CONFIG.notification_connections_by_names.get(
connection
)
if not connection_type:
logger.warning(
"Could not create notification using connection {}, "
"the connection was not found or not set correctly.".format(
connection_type
)
)
continue
operation = V1Operation(
params={
"kind": connection_type.kind,
"owner": owner,
"project": project,
"run_uuid": run_uuid,
"run_name": run_name,
"condition": ujson.dumps(condition.to_dict()),
},
termination=V1Termination(max_retries=3),
component=V1Component(
name="slack-notification",
plugins=V1Plugins(
auth=False,
collect_logs=False,
collect_artifacts=False,
collect_resources=False,
sync_statuses=False,
),
inputs=[
V1IO(name="kind", iotype=types.STR, is_optional=False),
V1IO(name="owner", iotype=types.STR, is_optional=False),
V1IO(name="project", iotype=types.STR, is_optional=False),
V1IO(name="run_uuid", iotype=types.STR, is_optional=False),
V1IO(name="run_name", iotype=types.STR, is_optional=True),
V1IO(name="condition", iotype=types.STR, is_optional=True),
V1IO(name="connection", iotype=types.STR, is_optional=True),
],
run=V1Notifier(
connections=[connection],
container=get_default_notification_container(),
),
),
)
compiled_operation = OperationSpecification.compile_operation(operation)
resource = compiler.make(
owner_name=owner,
project_name=project,
project_uuid=project,
run_uuid=run_uuid,
run_name=run_name,
run_path=run_uuid,
compiled_operation=compiled_operation,
params=operation.params,
)
await spawner.create(
run_uuid=run_uuid,
run_kind=compiled_operation.get_run_kind(),
resource=resource,
)