def update_experiment_from_template(
self, experiment: schemas.ExperimentUpdate, experiment_id: str
):
"""
Recreates the operators of experiment using a template.
Parameters
----------
experiment : projects.schemas.experiment.ExperimentUpdate
experiment_id : str
"""
template = self.session.query(models.Template).get(experiment.template_id)
if template is None:
raise BadRequest(
code="InvalidTemplateId",
message="The specified template does not exist",
)
# remove operators
self.session.query(models.Operator).filter(
models.Operator.experiment_id == experiment_id
).delete()
# save the operators created to get the created_uuid to use on dependencies
operators_created = []
for task in template.tasks:
dependencies = []
task_dependencies = task["dependencies"]
if len(task_dependencies) > 0:
for d in task_dependencies:
op_created = next(
(o for o in operators_created if o["uuid"] == d), None
)
dependencies.append(op_created["created_uuid"])
operator_id = uuid_alpha()
objects = [
models.Operator(
uuid=operator_id,
experiment_id=experiment_id,
task_id=task["task_id"],
dependencies=dependencies,
position_x=task["position_x"],
position_y=task["position_y"],
)
]
self.session.bulk_save_objects(objects)
task["created_uuid"] = operator_id
operators_created.append(task)
self.session.commit()
experiment = self.session.query(models.Experiment).get(experiment_id)
return schemas.Experiment.from_orm(experiment)
def create_deployment_from_template(self, template_id: str,
project_id: str):
"""
Creates the operators of deployment using a template.
Parameters
----------
template_id : str
project_id : str
Returns
-------
list
A list of projects.models.deployment.Deployment.
"""
template = self.template_controller.get_template(template_id)
deployment = models.Deployment(uuid=uuid_alpha(),
name=template.name,
project_id=project_id,
created_at=now(),
updated_at=now())
self.session.add(deployment)
self.session.flush()
# save the operators created to get the created_uuid to use on dependencies
operators_created = []
for task in template.tasks:
dependencies = []
task_dependencies = task["dependencies"]
if len(task_dependencies) > 0:
for d in task_dependencies:
op_created = next(
(o for o in operators_created if o["uuid"] == d), None)
dependencies.append(op_created["created_uuid"])
operator_id = uuid_alpha()
objects = [
models.Operator(
uuid=operator_id,
deployment_id=deployment.uuid,
task_id=task["task_id"],
dependencies=dependencies,
position_x=task["position_x"],
position_y=task["position_y"],
status="Setted up",
)
]
self.session.bulk_save_objects(objects)
task["created_uuid"] = operator_id
operators_created.append(task)
return [deployment]
def create_operator(
self,
operator: schemas.OperatorCreate,
project_id: str,
experiment_id: Optional[str] = None,
deployment_id: Optional[str] = None,
):
"""
Creates a new operator in our database.
Parameters
----------
operator: projects.schemas.operator.OperatorCreate
project_id : str
experiment_id : str or None
deployment_id : str or None
Returns
-------
projects.schemas.operator.Operator
Raises
------
BadRequest
When the operator attributes are invalid.
"""
if not isinstance(operator.task_id, str):
raise BadRequest(code="MissingRequiredTaskId",
message="taskId is required")
try:
self.task_controller.raise_if_task_does_not_exist(operator.task_id)
except NotFound:
raise BadRequest(code="InvalidTaskId",
message="source task does not exist")
if operator.dependencies is None:
operator.dependencies = []
if experiment_id:
self.raise_if_dependencies_are_invalid(
project_id=project_id,
experiment_id=experiment_id,
deployment_id=deployment_id,
dependencies=operator.dependencies,
)
if experiment_id and deployment_id:
raise BadRequest(
code="InvalidOperatorRequestBody",
message=
"Operator cannot contain an experiment and a deployment simultaneously",
)
if operator.parameters is None:
operator.parameters = {}
self.raise_if_parameters_are_invalid(operator.parameters)
operator = models.Operator(uuid=uuid_alpha(),
name=operator.name,
experiment_id=experiment_id,
deployment_id=deployment_id,
task_id=operator.task_id,
dependencies=operator.dependencies,
status=operator.status,
parameters=operator.parameters,
position_x=operator.position_x,
position_y=operator.position_y,
created_at=now(),
updated_at=now())
self.session.add(operator)
self.session.commit()
self.session.refresh(operator)
return schemas.Operator.from_orm(operator)
def create_mocks():
"""
Inserts some mock records into test database.
"""
session = TestingSessionLocal()
objects = [
models.Project(
uuid=MOCK_UUID_1,
name=MOCK_PROJECT_NAME_1,
created_at=MOCK_CREATED_AT_1,
updated_at=MOCK_UPDATED_AT_1,
tenant=DB_TENANT,
),
models.Project(
uuid=MOCK_UUID_2,
name=MOCK_PROJECT_NAME_2,
created_at=MOCK_CREATED_AT_2,
updated_at=MOCK_UPDATED_AT_2,
tenant=DB_TENANT,
),
models.Project(
uuid=MOCK_UUID_3,
name=MOCK_PROJECT_NAME_3,
created_at=MOCK_CREATED_AT_3,
updated_at=MOCK_UPDATED_AT_3,
tenant=DB_TENANT,
),
]
session.bulk_save_objects(objects)
session.flush()
objects = [
models.Task(
uuid=MOCK_UUID_1,
name=MOCK_TASK_NAME_1,
category="DEFAULT",
tags=[],
experiment_notebook_path="Experiment.ipynb",
deployment_notebook_path="Deployment.ipynb",
created_at=MOCK_CREATED_AT_1,
updated_at=MOCK_UPDATED_AT_1,
),
models.Task(
uuid=MOCK_UUID_2,
name=MOCK_TASK_NAME_2,
category="DEFAULT",
tags=[],
created_at=MOCK_CREATED_AT_2,
updated_at=MOCK_UPDATED_AT_2,
),
models.Task(
uuid=MOCK_UUID_3,
name=MOCK_TASK_NAME_3,
category="DEFAULT",
tags=[],
created_at=MOCK_CREATED_AT_3,
updated_at=MOCK_UPDATED_AT_3,
),
models.Task(
uuid=MOCK_UUID_4,
name=MOCK_TASK_NAME_4,
category="DEFAULT",
tags=[],
created_at=MOCK_CREATED_AT_4,
updated_at=MOCK_UPDATED_AT_4,
),
models.Task(
uuid=MOCK_UUID_5,
name=MOCK_TASK_NAME_5,
category="MONITORING",
tags=[],
created_at=MOCK_CREATED_AT_4,
updated_at=MOCK_UPDATED_AT_4,
),
models.Task(
uuid=MOCK_UUID_6,
name=MOCK_TASK_NAME_6,
category="DEFAULT",
tags=[],
created_at=MOCK_CREATED_AT_6,
updated_at=MOCK_UPDATED_AT_6,
),
]
session.bulk_save_objects(objects)
session.flush()
objects = [
models.Experiment(
uuid=MOCK_UUID_1,
name=MOCK_EXPERIMENT_NAME_1,
project_id=MOCK_UUID_1,
position=0,
is_active=True,
created_at=MOCK_CREATED_AT_1,
updated_at=MOCK_UPDATED_AT_1,
),
models.Experiment(
uuid=MOCK_UUID_2,
name=MOCK_EXPERIMENT_NAME_2,
project_id=MOCK_UUID_1,
position=1,
is_active=False,
created_at=MOCK_CREATED_AT_2,
updated_at=MOCK_UPDATED_AT_2,
),
]
session.bulk_save_objects(objects)
session.flush()
objects = [
models.Deployment(
uuid=MOCK_UUID_1,
name=MOCK_DEPLOYMENT_NAME_1,
project_id=MOCK_UUID_1,
experiment_id=MOCK_UUID_1,
position=0,
is_active=True,
status="Pending",
created_at=MOCK_CREATED_AT_1,
updated_at=MOCK_UPDATED_AT_1,
),
models.Deployment(
uuid=MOCK_UUID_2,
name=MOCK_DEPLOYMENT_NAME_2,
project_id=MOCK_UUID_1,
experiment_id=MOCK_UUID_1,
is_active=False,
position=1,
status="Pending",
created_at=MOCK_CREATED_AT_2,
updated_at=MOCK_UPDATED_AT_2,
),
]
session.bulk_save_objects(objects)
session.flush()
objects = [
models.Operator(
uuid=MOCK_UUID_1,
experiment_id=MOCK_UUID_1,
task_id=MOCK_UUID_1,
dependencies=[],
parameters={"dataset": IRIS_DATASET_NAME},
status="Unset",
position_x=0,
position_y=0,
created_at=MOCK_CREATED_AT_1,
updated_at=MOCK_UPDATED_AT_1,
),
models.Operator(
uuid=MOCK_UUID_2,
deployment_id=MOCK_UUID_1,
task_id=MOCK_UUID_1,
dependencies=[],
parameters={},
status="Unset",
position_x=0,
position_y=0,
created_at=MOCK_CREATED_AT_2,
updated_at=MOCK_UPDATED_AT_2,
),
models.Operator(
uuid=MOCK_UUID_3,
deployment_id=MOCK_UUID_2,
task_id=MOCK_UUID_1,
dependencies=[],
parameters={},
status="Unset",
position_x=0,
position_y=0,
created_at=MOCK_CREATED_AT_3,
updated_at=MOCK_UPDATED_AT_3,
),
models.Operator(
uuid=MOCK_UUID_4,
experiment_id=MOCK_UUID_1,
task_id=MOCK_UUID_1,
dependencies=[MOCK_UUID_1],
parameters={},
status="Unset",
position_x=0,
position_y=0,
created_at=MOCK_CREATED_AT_1,
updated_at=MOCK_UPDATED_AT_1,
),
]
session.bulk_save_objects(objects)
session.flush()
objects = [
models.Template(
uuid=MOCK_UUID_1,
name=MOCK_TEMPLATE_NAME_1,
tasks=[{
"uuid": MOCK_UUID_1,
"task_id": MOCK_UUID_1,
"dependencies": [],
"position_x": 0.0,
"position_y": 0.0,
}],
experiment_id=MOCK_UUID_1,
deployment_id=None,
created_at=MOCK_CREATED_AT_1,
updated_at=MOCK_UPDATED_AT_1,
tenant=DB_TENANT,
),
models.Template(
uuid=MOCK_UUID_2,
name=MOCK_TEMPLATE_NAME_2,
tasks=[{
"uuid": MOCK_UUID_2,
"task_id": MOCK_UUID_1,
"dependencies": [],
"position_x": 0.0,
"position_y": 0.0,
}],
experiment_id=None,
deployment_id=MOCK_UUID_1,
created_at=MOCK_CREATED_AT_2,
updated_at=MOCK_UPDATED_AT_2,
tenant=DB_TENANT,
),
]
session.bulk_save_objects(objects)
session.flush()
objects = [
models.Comparison(
uuid=MOCK_UUID_1,
project_id=MOCK_UUID_1,
experiment_id=MOCK_UUID_1,
operator_id=MOCK_UUID_1,
active_tab="1",
run_id=MOCK_UUID_1,
layout={
"x": 0,
"y": 0,
"w": 0,
"h": 0
},
created_at=MOCK_CREATED_AT_1,
updated_at=MOCK_UPDATED_AT_1,
),
]
session.bulk_save_objects(objects)
session.flush()
objects = [
models.Monitoring(
uuid=MOCK_UUID_1,
deployment_id=MOCK_UUID_1,
task_id=MOCK_UUID_1,
created_at=MOCK_CREATED_AT_1,
),
]
session.bulk_save_objects(objects)
session.commit()
session.close()
def copy_operators(self, deployment_id: str,
stored_operators: schemas.OperatorList):
"""
Copies the operators to a deployment.
Creates new uuids and don't keep the experiment_id/deployment_id relationship.
Parameters
----------
stored_operators : projects.schemas.operator.OperatorsList
deployment_id : str
"""
if len(stored_operators) == 0:
raise BadRequest(
code="MissingRequiredOperatorId",
message="Necessary at least one operator.",
)
# Creates a dict to map source operator_id to its copy operator_id.
# This map will be used to build the dependencies using new operator_ids
copies_map = {}
# just a simple flag to detect the existence of a dataset operator
some_stored_operators_is_dataset = False
# default position, in case we have to create a dataset operator
leftmost_operator_position = (
stored_operators[0].position_x,
stored_operators[0].position_y,
)
for stored_operator in stored_operators:
# If we have to create a dataset operator, it is interesting that we put before the leftmost position
if stored_operator.position_x < leftmost_operator_position[0]:
leftmost_operator_position = (
stored_operator.position_x,
stored_operator.position_y,
)
if stored_operator.task.category == "DATASETS":
name = FONTE_DE_DADOS
parameters = {"type": "L", "dataset": None}
some_stored_operators_is_dataset = True
else:
name = None
parameters = stored_operator.parameters
operator_id = uuid_alpha()
operator = models.Operator(
uuid=operator_id,
name=name,
deployment_id=deployment_id,
task_id=stored_operator.task_id,
dependencies=[],
status="Setted up",
parameters=parameters,
position_x=stored_operator.position_x,
position_y=stored_operator.position_y,
)
self.session.add(operator)
self.session.flush()
copies_map[stored_operator.uuid] = {
"copy_uuid": operator_id,
"dependencies": stored_operator.dependencies,
}
# creates a DATASET type operator if doesn't exist any
if not some_stored_operators_is_dataset:
generated_dataset_operator_uuid = uuid_alpha()
operator = models.Operator(
uuid=generated_dataset_operator_uuid,
name=FONTE_DE_DADOS,
deployment_id=deployment_id,
task_id=self.task_controller.
get_or_create_dataset_task_if_not_exist(),
dependencies=[],
parameters={
"type": "L",
"dataset": None
},
position_x=leftmost_operator_position[0] -
DATASET_OPERATOR_DISTANCE,
position_y=leftmost_operator_position[1],
)
self.session.add(operator)
self.session.flush()
self.set_dependents_for_generated_dataset_operator(
copies_map, generated_dataset_operator_uuid)
self.set_dependencies_on_new_operators(copies_map)