def update_compare_result(uuid, project_id, **kwargs):
"""Updates a compare result in our database.
Args:
uuid (str): the a compare result uuid to look for in our database.
project_id (str): the project uuid.
**kwargs: arbitrary keyword arguments.
Returns:
The compare result info.
"""
raise_if_project_does_not_exist(project_id)
compare_result = CompareResult.query.get(uuid)
if compare_result is None:
raise NOT_FOUND
experiment_id = kwargs.get("experiment_id", None)
if experiment_id:
raise_if_experiment_does_not_exist(experiment_id)
data = {"updated_at": datetime.utcnow()}
data.update(kwargs)
try:
db_session.query(CompareResult).filter_by(uuid=uuid).update(data)
db_session.commit()
except (InvalidRequestError, ProgrammingError) as e:
raise BadRequest(str(e))
return compare_result.as_dict()
def delete_operator(uuid, project_id, experiment_id):
"""Delete an operator in our database.
Args:
uuid (str): the operator uuid to look for in our database.
project_id (str): the project uuid.
experiment_id (str): the experiment uuid.
Returns:
The deletion result.
"""
raise_if_project_does_not_exist(project_id)
raise_if_experiment_does_not_exist(experiment_id)
operator = Operator.query.get(uuid)
if operator is None:
raise NotFound("The specified operator does not exist")
# check if other operators contains the operator being deleted
# in dependencies and remove this operator from dependencies
operators = db_session.query(Operator) \
.filter_by(experiment_id=experiment_id) \
.filter(Operator.uuid != uuid)\
.all()
for op in operators:
if uuid in op.dependencies:
dependencies = op.dependencies.remove(uuid)
if dependencies is None:
dependencies = []
update_operator(op.uuid, project_id, experiment_id, dependencies=dependencies)
db_session.delete(operator)
db_session.commit()
return {"message": "Operator deleted"}
def fix_positions(project_id, experiment_id=None, new_position=None):
"""Reorders the experiments in a project when an experiment is updated/deleted.
Args:
project_id (str): the project uuid.
experiment_id (str): the experiment uuid.
new_position (int): the position where the experiment is shown.
"""
other_experiments = db_session.query(Experiment) \
.filter_by(project_id=project_id) \
.filter(Experiment.uuid != experiment_id)\
.order_by(Experiment.position.asc())\
.all()
if experiment_id is not None:
experiment = Experiment.query.get(experiment_id)
other_experiments.insert(new_position, experiment)
for index, experiment in enumerate(other_experiments):
data = {"position": index}
is_last = (index == len(other_experiments) - 1)
# if experiment_id WAS NOT informed, then set the higher position as is_active=True
if experiment_id is None and is_last:
data["is_active"] = True
# if experiment_id WAS informed, then set experiment.is_active=True
elif experiment_id is not None and experiment_id == experiment.uuid:
data["is_active"] = True
else:
data["is_active"] = False
db_session.query(Experiment).filter_by(uuid=experiment.uuid).update(data)
db_session.commit()
def delete_experiment(uuid, project_id):
"""Delete an experiment in our database and in the object storage.
Args:
uuid (str): the experiment uuid to look for in our database.
project_id (str): the project uuid.
Returns:
The deletion result.
"""
raise_if_project_does_not_exist(project_id)
experiment = Experiment.query.get(uuid)
if experiment is None:
raise NOTFOUND
# remove compare results
CompareResult.query.filter(CompareResult.experiment_id == uuid).delete()
# remove operators
Operator.query.filter(Operator.experiment_id == uuid).delete()
db_session.delete(experiment)
db_session.commit()
fix_positions(project_id=experiment.project_id)
prefix = join("experiments", uuid)
remove_objects(prefix=prefix)
return {"message": "Experiment deleted"}
def update_project(uuid, **kwargs):
"""Updates a project in our database.
Args:
uuid(str): the project uuid to look for in our database.
**kwargs: arbitrary keyword arguments.
Returns:
The project info.
"""
project = Project.query.get(uuid)
if project is None:
raise NOT_FOUND
if "name" in kwargs:
name = kwargs["name"]
if name != project.name:
check_project_name = db_session.query(Project).filter_by(
name=name).first()
if check_project_name:
raise BadRequest("a project with that name already exists")
data = {"updated_at": datetime.utcnow()}
data.update(kwargs)
try:
db_session.query(Project).filter_by(uuid=uuid).update(data)
db_session.commit()
except (InvalidRequestError, ProgrammingError) as e:
raise BadRequest(str(e))
return project.as_dict()
def update_template(uuid, **kwargs):
"""Updates a template in our database.
Args:
uuid (str): the template uuid to look for in our database.
**kwargs: arbitrary keyword arguments.
Returns:
The template info.
"""
template = Template.query.get(uuid)
if template is None:
raise NotFound("The specified template does not exist")
data = {"updated_at": datetime.utcnow()}
data.update(kwargs)
try:
db_session.query(Template).filter_by(uuid=uuid).update(data)
db_session.commit()
except (InvalidRequestError, ProgrammingError) as e:
raise BadRequest(str(e))
return template.as_dict()
def create_template(name=None, experiment_id=None, **kwargs):
"""Creates a new template in our database.
Args:
name (str): the template name.
Returns:
The template info.
"""
if not isinstance(name, str):
raise BadRequest("name is required")
if not isinstance(experiment_id, str):
raise BadRequest("experimentId is required")
try:
raise_if_experiment_does_not_exist(experiment_id)
except NotFound as e:
raise BadRequest(e.description)
operators = db_session.query(Operator) \
.filter_by(experiment_id=experiment_id) \
.order_by(Operator.created_at.asc()) \
.all()
# JSON array order of elements are preserved,
# so there is no need to save positions
tasks = [operator.task_id for operator in operators]
template = Template(uuid=uuid_alpha(), name=name, tasks=tasks)
db_session.add(template)
db_session.commit()
return template.as_dict()
def delete_project(uuid):
"""Delete a project in our database and in the object storage.
Args:
uuid (str): the project uuid to look for in our database.
Returns:
The deletion result.
"""
project = Project.query.get(uuid)
if project is None:
raise NOT_FOUND
experiments = Experiment.query.filter(Experiment.project_id == uuid).all()
for experiment in experiments:
# remove operators
Operator.query.filter(
Operator.experiment_id == experiment.uuid).delete()
Experiment.query.filter(Experiment.project_id == uuid).delete()
db_session.delete(project)
db_session.commit()
prefix = join("experiments", uuid)
remove_objects(prefix=prefix)
return {"message": "Project deleted"}
def update_task(uuid, **kwargs):
"""Updates a task in our database/object storage.
Args:
uuid (str): the task uuid to look for in our database.
**kwargs: arbitrary keyword arguments.
Returns:
The task info.
"""
task = Task.query.get(uuid)
if task is None:
raise NOT_FOUND
if "name" in kwargs:
name = kwargs["name"]
if name != task.name:
check_comp_name = db_session.query(Task).filter_by(name=name).first()
if check_comp_name:
raise BadRequest("a task with that name already exists")
if "tags" in kwargs:
tags = kwargs["tags"]
if any(tag not in VALID_TAGS for tag in tags):
valid_str = ",".join(VALID_TAGS)
raise BadRequest(f"Invalid tag. Choose any of {valid_str}")
if "experiment_notebook" in kwargs:
obj_name = f"{PREFIX}/{uuid}/Experiment.ipynb"
put_object(obj_name, dumps(kwargs["experiment_notebook"]).encode())
del kwargs["experiment_notebook"]
if "deployment_notebook" in kwargs:
obj_name = f"{PREFIX}/{uuid}/Deployment.ipynb"
put_object(obj_name, dumps(kwargs["deployment_notebook"]).encode())
del kwargs["deployment_notebook"]
# store the name to use it after update
old_name = task.name
try:
data = {"updated_at": datetime.utcnow()}
data.update(kwargs)
db_session.query(Task).filter_by(uuid=uuid).update(data)
db_session.commit()
except (InvalidRequestError, ProgrammingError) as e:
raise BadRequest(str(e))
# update jupyter folder name if name changed
if old_name != task.name:
update_folder_name(f"{PREFIX}/{old_name}", f"{PREFIX}/{task.name}")
return task.as_dict()
def create_operator(project_id, experiment_id, task_id=None,
parameters=None, dependencies=None, position_x=None,
position_y=None, **kwargs):
"""
Creates a new operator in our database.
The new operator is added to the end of the operator list.
Args:
project_id (str): the project uuid.
experiment_id (str): the experiment uuid.
task_id (str): the task uuid.
parameters (dict): the parameters dict.
dependencies (list): the dependencies array.
position_x (float): position x.
position_y (float): position y.
Returns:
The operator info.
"""
raise_if_project_does_not_exist(project_id)
raise_if_experiment_does_not_exist(experiment_id)
if not isinstance(task_id, str):
raise BadRequest("taskId is required")
try:
raise_if_task_does_not_exist(task_id)
except NotFound as e:
raise BadRequest(e.description)
if parameters is None:
parameters = {}
raise_if_parameters_are_invalid(parameters)
if dependencies is None:
dependencies = []
raise_if_dependencies_are_invalid(project_id, experiment_id, dependencies)
operator = Operator(uuid=uuid_alpha(),
experiment_id=experiment_id,
task_id=task_id,
dependencies=dependencies,
parameters=parameters,
position_x=position_x,
position_y=position_y)
db_session.add(operator)
db_session.commit()
check_status(operator)
return operator.as_dict()
def create_compare_result(project_id=None):
"""Creates a new compare result in our database.
Args:
project_id (str): the project uuid.
Returns:
The compare result info.
"""
raise_if_project_does_not_exist(project_id)
compare_result = CompareResult(uuid=uuid_alpha(), project_id=project_id)
db_session.add(compare_result)
db_session.commit()
return compare_result.as_dict()
def delete_template(uuid):
"""Delete a template in our database.
Args:
uuid (str): the template uuid to look for in our database.
Returns:
The deletion result.
"""
template = Template.query.get(uuid)
if template is None:
raise NotFound("The specified template does not exist")
db_session.delete(template)
db_session.commit()
return {"message": "Template deleted"}
def delete_compare_result(uuid, project_id):
"""Delete a compare result in our database.
Args:
uuid (str): the compare result uuid to look for in our database.
project_id (str): the project uuid.
Returns:
The deletion result.
"""
raise_if_project_does_not_exist(project_id)
compare_result = CompareResult.query.get(uuid)
if compare_result is None:
raise NOT_FOUND
db_session.delete(compare_result)
db_session.commit()
return {"message": "Compare result deleted"}
def delete_task(uuid):
"""Delete a task in our database/object storage.
Args:
uuid (str): the task uuid to look for in our database.
Returns:
The task info.
"""
task = Task.query.get(uuid)
if task is None:
raise NOT_FOUND
try:
db_session.query(Task).filter_by(uuid=uuid).delete()
# remove files and directory from jupyter notebook server
source_name = f"{PREFIX}/{task.name}"
jupyter_files = list_files(source_name)
if jupyter_files is not None:
for jupyter_file in jupyter_files["content"]:
delete_file(jupyter_file["path"])
delete_file(source_name)
# remove MinIO files and directory
source_name = f"{PREFIX}/{uuid}"
minio_files = list_objects(source_name)
for minio_file in minio_files:
remove_object(minio_file.object_name)
remove_object(source_name)
db_session.commit()
except IntegrityError as e:
raise Forbidden(str(e))
except (InvalidRequestError, ProgrammingError, ResponseError) as e:
raise BadRequest(str(e))
return {"message": "Task deleted"}
def create_project(name=None, **kwargs):
"""Creates a new project in our database.
Args:
name (str): the project name.
Returns:
The project info.
"""
if not isinstance(name, str):
raise BadRequest("name is required")
check_project_name = db_session.query(Project).filter_by(name=name).first()
if check_project_name:
raise BadRequest("a project with that name already exists")
project = Project(uuid=uuid_alpha(),
name=name,
description=kwargs.get("description"))
db_session.add(project)
db_session.commit()
create_experiment(name="Experimento 1", project_id=project.uuid)
return project.as_dict()
def update_operator(uuid, project_id, experiment_id, **kwargs):
"""Updates an operator in our database and adjusts the position of others.
Args:
uuid (str): the operator uuid to look for in our database.
project_id (str): the project uuid.
experiment_id (str): the experiment uuid.
**kwargs: arbitrary keyword arguments.
Returns:
The operator info.
"""
raise_if_project_does_not_exist(project_id)
raise_if_experiment_does_not_exist(experiment_id)
operator = Operator.query.get(uuid)
if operator is None:
raise NotFound("The specified operator does not exist")
raise_if_parameters_are_invalid(kwargs.get("parameters", {}))
dependencies = kwargs.get("dependencies")
if dependencies is not None:
raise_if_dependencies_are_invalid(project_id, experiment_id, dependencies, operator_id=uuid)
data = {"updated_at": datetime.utcnow()}
data.update(kwargs)
try:
db_session.query(Operator).filter_by(uuid=uuid).update(data)
db_session.commit()
except (InvalidRequestError, ProgrammingError) as e:
raise BadRequest(str(e))
check_status(operator)
return operator.as_dict()
def create_experiment(name=None, project_id=None, copy_from=None):
"""Creates a new experiment in our database and adjusts the position of others.
The new experiment is added to the end of the experiment list.
Args:
name (str): the experiment name.
project_id (str): the project uuid.
copy_from (str): the copy_from uuid.
Returns:
The experiment info.
"""
raise_if_project_does_not_exist(project_id)
if not isinstance(name, str):
raise BadRequest("name is required")
check_experiment_name = db_session.query(Experiment)\
.filter(Experiment.project_id == project_id)\
.filter(Experiment.name == name)\
.first()
if check_experiment_name:
raise BadRequest("an experiment with that name already exists")
experiment = Experiment(uuid=uuid_alpha(), name=name, project_id=project_id)
db_session.add(experiment)
db_session.commit()
if copy_from:
try:
experiment_find = find_by_experiment_id(experiment_id=copy_from)
source_operators = {}
for source_operator in experiment_find['operators']:
source_dependencies = source_operator.dependencies
kwargs = {
"task_id": source_operator.task_id,
"parameters": source_operator.parameters,
"dependencies": [],
"position_x": source_operator.position_x,
"position_y": source_operator.position_y
}
operator = create_operator(project_id, experiment.uuid, **kwargs)
source_operators[source_operator.uuid] = {
"copy_uuid": operator["uuid"],
"dependencies": source_dependencies
}
# update dependencies on new operators
for _, value in source_operators.items():
dependencies = [source_operators[d]['copy_uuid'] for d in value['dependencies']]
update_operator(value['copy_uuid'], project_id, experiment.uuid, dependencies=dependencies)
except NotFound:
delete_experiment(experiment.uuid, project_id)
raise BadRequest('Source experiment does not exist')
fix_positions(project_id=project_id,
experiment_id=experiment.uuid,
new_position=sys.maxsize) # will add to end of list
return experiment.as_dict()
def copy_task(name, description, tags, copy_from):
"""Makes a copy of a task in our database/object storage.
Args:
name (str): the task name.
description (str): the task description.
tags (list): the task tags list.
copy_from (str): the task_id from which the notebooks are copied.
Returns:
The task info.
"""
task = Task.query.get(copy_from)
if task is None:
raise BadRequest("Source task does not exist")
task_id = uuid_alpha()
image = task.image
commands = task.commands
arguments = task.arguments
# reads source notebooks from object storage
source_name = f"{PREFIX}/{copy_from}/Deployment.ipynb"
deployment_notebook = loads(get_object(source_name))
source_name = f"{PREFIX}/{copy_from}/Experiment.ipynb"
experiment_notebook = loads(get_object(source_name))
# Even though we are creating 'copies', the new task must have
# its own task_id, experiment_id and operator_id.
# We don't want to mix models and metrics of different tasks.
# Notice these values are ignored when a notebook is run in a pipeline.
# They are only used by JupyterLab interface.
init_notebook_metadata(task_id, deployment_notebook, experiment_notebook)
# saves new notebooks to object storage
destination_name = f"{PREFIX}/{task_id}/Deployment.ipynb"
deployment_notebook_path = f"minio://{BUCKET_NAME}/{destination_name}"
put_object(destination_name, dumps(deployment_notebook).encode())
destination_name = f"{PREFIX}/{task_id}/Experiment.ipynb"
experiment_notebook_path = f"minio://{BUCKET_NAME}/{destination_name}"
put_object(destination_name, dumps(experiment_notebook).encode())
# create deployment notebook and eperiment notebook on jupyter
create_jupyter_files(task_name=name,
deployment_notebook=dumps(deployment_notebook).encode(),
experiment_notebook=dumps(experiment_notebook).encode())
# saves task info to the database
task = Task(uuid=task_id,
name=name,
description=description,
tags=tags,
image=image,
commands=commands,
arguments=arguments,
deployment_notebook_path=deployment_notebook_path,
experiment_notebook_path=experiment_notebook_path,
is_default=False)
db_session.add(task)
db_session.commit()
return task.as_dict()
def create_task(**kwargs):
"""Creates a new task in our database/object storage.
Args:
**kwargs: arbitrary keyword arguments.
Returns:
The task info.
"""
name = kwargs.get('name', None)
description = kwargs.get('description', None)
tags = kwargs.get('tags', None)
image = kwargs.get('image', None)
commands = kwargs.get('commands', None)
arguments = kwargs.get('arguments', None)
experiment_notebook = kwargs.get('experiment_notebook', None)
deployment_notebook = kwargs.get('deployment_notebook', None)
is_default = kwargs.get('is_default', None)
copy_from = kwargs.get('copy_from', None)
if not isinstance(name, str):
raise BadRequest("name is required")
if copy_from and (experiment_notebook or deployment_notebook):
raise BadRequest("Either provide notebooks or a task to copy from")
if tags is None or len(tags) == 0:
tags = ["DEFAULT"]
if any(tag not in VALID_TAGS for tag in tags):
valid_str = ",".join(VALID_TAGS)
raise BadRequest(f"Invalid tag. Choose any of {valid_str}")
# check if image is a valid docker image
if image:
pattern = re.compile('[a-z0-9.-]+([/]{1}[a-z0-9.-]+)+([:]{1}[a-z0-9.-]+){0,1}$')
if pattern.match(image) is None:
raise BadRequest("invalid docker image name")
check_comp_name = db_session.query(Task).filter_by(name=name).first()
if check_comp_name:
raise BadRequest("a task with that name already exists")
# creates a task with specified name,
# but copies notebooks from a source task
if copy_from:
return copy_task(name, description, tags, copy_from)
task_id = str(uuid_alpha())
# loads a sample notebook if none was sent
if experiment_notebook is None and "DATASETS" not in tags:
experiment_notebook = EXPERIMENT_NOTEBOOK
if deployment_notebook is None and "DATASETS" not in tags:
deployment_notebook = DEPLOYMENT_NOTEBOOK
# The new task must have its own task_id, experiment_id and operator_id.
# Notice these values are ignored when a notebook is run in a pipeline.
# They are only used by JupyterLab interface.
init_notebook_metadata(task_id, deployment_notebook, experiment_notebook)
# saves new notebooks to object storage
if "DATASETS" not in tags:
obj_name = f"{PREFIX}/{task_id}/Experiment.ipynb"
experiment_notebook_path = f"minio://{BUCKET_NAME}/{obj_name}"
put_object(obj_name, dumps(experiment_notebook).encode())
obj_name = f"{PREFIX}/{task_id}/Deployment.ipynb"
deployment_notebook_path = f"minio://{BUCKET_NAME}/{obj_name}"
put_object(obj_name, dumps(deployment_notebook).encode())
# create deployment notebook and experiment_notebook on jupyter
create_jupyter_files(task_name=name,
deployment_notebook=dumps(deployment_notebook).encode(),
experiment_notebook=dumps(experiment_notebook).encode())
else:
experiment_notebook_path = None
deployment_notebook_path = None
if commands is None or len(commands) == 0:
commands = DEFAULT_COMMANDS
if arguments is None or len(arguments) == 0:
arguments = DEFAULT_ARGUMENTS
# saves task info to the database
task = Task(uuid=task_id,
name=name,
description=description,
tags=tags,
image=image,
commands=commands,
arguments=arguments,
experiment_notebook_path=experiment_notebook_path,
deployment_notebook_path=deployment_notebook_path,
is_default=is_default)
db_session.add(task)
db_session.commit()
return task.as_dict()
def update_experiment(uuid, project_id, **kwargs):
"""Updates an experiment in our database and adjusts the position of others.
Args:
uuid (str): the experiment uuid to look for in our database.
project_id (str): the project uuid.
**kwargs: arbitrary keyword arguments.
Returns:
The experiment info.
"""
raise_if_project_does_not_exist(project_id)
experiment = Experiment.query.get(uuid)
if experiment is None:
raise NOTFOUND
if "name" in kwargs:
name = kwargs["name"]
if name != experiment.name:
check_experiment_name = db_session.query(Experiment)\
.filter(Experiment.project_id == project_id)\
.filter(Experiment.name == name)\
.first()
if check_experiment_name:
raise BadRequest("an experiment with that name already exists")
# updates operators
if "template_id" in kwargs:
template_id = kwargs["template_id"]
del kwargs["template_id"]
template = Template.query.get(template_id)
if template is None:
raise BadRequest("The specified template does not exist")
# remove operators
Operator.query.filter(Operator.experiment_id == uuid).delete()
# save the last operator id created to create dependency on next operator
last_operator_id = None
for task_id in template.tasks:
operator_id = uuid_alpha()
dependencies = []
if last_operator_id is not None:
dependencies = [last_operator_id]
objects = [
Operator(uuid=operator_id,
experiment_id=uuid,
task_id=task_id,
dependencies=dependencies)
]
db_session.bulk_save_objects(objects)
last_operator_id = operator_id
data = {"updated_at": datetime.utcnow()}
data.update(kwargs)
try:
db_session.query(Experiment).filter_by(uuid=uuid).update(data)
db_session.commit()
except (InvalidRequestError, ProgrammingError) as e:
raise BadRequest(str(e))
fix_positions(project_id=experiment.project_id,
experiment_id=experiment.uuid,
new_position=experiment.position)
return experiment.as_dict()
