def update_template(uuid, **kwargs):
"""Updates a template in our database.
Args:
uuid (str): the template uuid to look for in our database.
**kwargs: arbitrary keyword arguments.
Returns:
The template info.
"""
template = Template.query.get(uuid)
if template is None:
raise NotFound("The specified template does not exist")
data = {"updated_at": datetime.utcnow()}
data.update(kwargs)
try:
db_session.query(Template).filter_by(uuid=uuid).update(data)
db_session.commit()
except (InvalidRequestError, ProgrammingError) as e:
raise BadRequest(str(e))
return template.as_dict()
def update_compare_result(uuid, project_id, **kwargs):
"""Updates a compare result in our database.
Args:
uuid (str): the a compare result uuid to look for in our database.
project_id (str): the project uuid.
**kwargs: arbitrary keyword arguments.
Returns:
The compare result info.
"""
raise_if_project_does_not_exist(project_id)
compare_result = CompareResult.query.get(uuid)
if compare_result is None:
raise NOT_FOUND
experiment_id = kwargs.get("experiment_id", None)
if experiment_id:
raise_if_experiment_does_not_exist(experiment_id)
data = {"updated_at": datetime.utcnow()}
data.update(kwargs)
try:
db_session.query(CompareResult).filter_by(uuid=uuid).update(data)
db_session.commit()
except (InvalidRequestError, ProgrammingError) as e:
raise BadRequest(str(e))
return compare_result.as_dict()
def update_project(uuid, **kwargs):
"""Updates a project in our database.
Args:
uuid(str): the project uuid to look for in our database.
**kwargs: arbitrary keyword arguments.
Returns:
The project info.
"""
project = Project.query.get(uuid)
if project is None:
raise NOT_FOUND
if "name" in kwargs:
name = kwargs["name"]
if name != project.name:
check_project_name = db_session.query(Project).filter_by(
name=name).first()
if check_project_name:
raise BadRequest("a project with that name already exists")
data = {"updated_at": datetime.utcnow()}
data.update(kwargs)
try:
db_session.query(Project).filter_by(uuid=uuid).update(data)
db_session.commit()
except (InvalidRequestError, ProgrammingError) as e:
raise BadRequest(str(e))
return project.as_dict()
def fix_positions(project_id, experiment_id=None, new_position=None):
"""Reorders the experiments in a project when an experiment is updated/deleted.
Args:
project_id (str): the project uuid.
experiment_id (str): the experiment uuid.
new_position (int): the position where the experiment is shown.
"""
other_experiments = db_session.query(Experiment) \
.filter_by(project_id=project_id) \
.filter(Experiment.uuid != experiment_id)\
.order_by(Experiment.position.asc())\
.all()
if experiment_id is not None:
experiment = Experiment.query.get(experiment_id)
other_experiments.insert(new_position, experiment)
for index, experiment in enumerate(other_experiments):
data = {"position": index}
is_last = (index == len(other_experiments) - 1)
# if experiment_id WAS NOT informed, then set the higher position as is_active=True
if experiment_id is None and is_last:
data["is_active"] = True
# if experiment_id WAS informed, then set experiment.is_active=True
elif experiment_id is not None and experiment_id == experiment.uuid:
data["is_active"] = True
else:
data["is_active"] = False
db_session.query(Experiment).filter_by(uuid=experiment.uuid).update(data)
db_session.commit()
def delete_multiple_projects(project_ids):
"""Delete multiple projects.
Args:
project_ids(str): list of projects
Returns:
message
"""
total_elements = len(project_ids)
all_projects_ids = list_objects(project_ids)
if total_elements < 1:
return {"message": "please inform the uuid of the project"}
projects = db_session.query(Project).filter(
Project.uuid.in_(all_projects_ids)).all()
experiments = db_session.query(Experiment).filter(
Experiment.project_id.in_(objects_uuid(projects))).all()
operators = db_session.query(Operator).filter(Operator.experiment_id.in_(objects_uuid(experiments))) \
.all()
session = pre_delete(db_session, projects, total_elements, operators,
experiments, all_projects_ids)
for experiment in experiments:
prefix = join("experiments", experiment.uuid)
try:
remove_objects(prefix=prefix)
except Exception:
pass
session.commit()
return {"message": "Successfully removed projects"}
def update_task(uuid, **kwargs):
"""Updates a task in our database/object storage.
Args:
uuid (str): the task uuid to look for in our database.
**kwargs: arbitrary keyword arguments.
Returns:
The task info.
"""
task = Task.query.get(uuid)
if task is None:
raise NOT_FOUND
if "name" in kwargs:
name = kwargs["name"]
if name != task.name:
check_comp_name = db_session.query(Task).filter_by(name=name).first()
if check_comp_name:
raise BadRequest("a task with that name already exists")
if "tags" in kwargs:
tags = kwargs["tags"]
if any(tag not in VALID_TAGS for tag in tags):
valid_str = ",".join(VALID_TAGS)
raise BadRequest(f"Invalid tag. Choose any of {valid_str}")
if "experiment_notebook" in kwargs:
obj_name = f"{PREFIX}/{uuid}/Experiment.ipynb"
put_object(obj_name, dumps(kwargs["experiment_notebook"]).encode())
del kwargs["experiment_notebook"]
if "deployment_notebook" in kwargs:
obj_name = f"{PREFIX}/{uuid}/Deployment.ipynb"
put_object(obj_name, dumps(kwargs["deployment_notebook"]).encode())
del kwargs["deployment_notebook"]
# store the name to use it after update
old_name = task.name
try:
data = {"updated_at": datetime.utcnow()}
data.update(kwargs)
db_session.query(Task).filter_by(uuid=uuid).update(data)
db_session.commit()
except (InvalidRequestError, ProgrammingError) as e:
raise BadRequest(str(e))
# update jupyter folder name if name changed
if old_name != task.name:
update_folder_name(f"{PREFIX}/{old_name}", f"{PREFIX}/{task.name}")
return task.as_dict()
def create_template(name=None, experiment_id=None, **kwargs):
"""Creates a new template in our database.
Args:
name (str): the template name.
Returns:
The template info.
"""
if not isinstance(name, str):
raise BadRequest("name is required")
if not isinstance(experiment_id, str):
raise BadRequest("experimentId is required")
try:
raise_if_experiment_does_not_exist(experiment_id)
except NotFound as e:
raise BadRequest(e.description)
operators = db_session.query(Operator) \
.filter_by(experiment_id=experiment_id) \
.order_by(Operator.created_at.asc()) \
.all()
# JSON array order of elements are preserved,
# so there is no need to save positions
tasks = [operator.task_id for operator in operators]
template = Template(uuid=uuid_alpha(), name=name, tasks=tasks)
db_session.add(template)
db_session.commit()
return template.as_dict()
def delete_operator(uuid, project_id, experiment_id):
"""Delete an operator in our database.
Args:
uuid (str): the operator uuid to look for in our database.
project_id (str): the project uuid.
experiment_id (str): the experiment uuid.
Returns:
The deletion result.
"""
raise_if_project_does_not_exist(project_id)
raise_if_experiment_does_not_exist(experiment_id)
operator = Operator.query.get(uuid)
if operator is None:
raise NotFound("The specified operator does not exist")
# check if other operators contains the operator being deleted
# in dependencies and remove this operator from dependencies
operators = db_session.query(Operator) \
.filter_by(experiment_id=experiment_id) \
.filter(Operator.uuid != uuid)\
.all()
for op in operators:
if uuid in op.dependencies:
dependencies = op.dependencies.remove(uuid)
if dependencies is None:
dependencies = []
update_operator(op.uuid, project_id, experiment_id, dependencies=dependencies)
db_session.delete(operator)
db_session.commit()
return {"message": "Operator deleted"}
def pagination_tasks(name, page, page_size, order):
"""Task pagination.
Args:
name (str): task name
page (int): records size
page_size (int): page number
order (str): order by
Raises:
InternalServerError: MySQL error
Returns:
dict: query response
"""
try:
query = db_session.query(Task)
if name:
query = query.filter(Task.name.ilike(func.lower(f"%{name}%")))
if page is not None and order is None:
if page == 0:
query = query.order_by(Task.name)
else:
query = query.order_by(text('name')).limit(page_size).offset((page - 1) * page_size)
else:
query = pagination_ordering(query, page_size, page, order)
tasks = query.all()
total_rows = total_rows_tasks(name)
return {'total': total_rows, 'tasks': [task.as_dict() for task in tasks]}
except (OperationalError, InternalError):
raise InternalServerError("The service is unavailable. Try your call again.")
def get_tasks_by_tag(tag):
"""Get all tasks with a specific tag.
Returns:
A list of tasks.
"""
tasks = db_session.query(Task).filter(Task.tags.contains([tag])).all()
return [task.as_dict() for task in tasks]
def raise_if_task_does_not_exist(task_id):
"""Raises an exception if the specified task does not exist.
Args:
task_id (str): the task uuid.
"""
exists = db_session.query(Task.uuid) \
.filter_by(uuid=task_id) \
.scalar() is not None
if not exists:
raise NotFound("The specified task does not exist")
def raise_if_project_does_not_exist(project_id):
"""Raises an exception if the specified project does not exist.
Args:
project_id (str): the project uuid.
"""
exists = db_session.query(Project.uuid) \
.filter_by(uuid=project_id) \
.scalar() is not None
if not exists:
raise NotFound("The specified project does not exist")
def raise_if_experiment_does_not_exist(experiment_id):
"""Raises an exception if the specified experiment does not exist.
Args:
experiment_id (str): the experiment uuid.
"""
exists = db_session.query(Experiment.uuid) \
.filter_by(uuid=experiment_id) \
.scalar() is not None
if not exists:
raise NotFound("The specified experiment does not exist")
def delete_task(uuid):
"""Delete a task in our database/object storage.
Args:
uuid (str): the task uuid to look for in our database.
Returns:
The task info.
"""
task = Task.query.get(uuid)
if task is None:
raise NOT_FOUND
try:
db_session.query(Task).filter_by(uuid=uuid).delete()
# remove files and directory from jupyter notebook server
source_name = f"{PREFIX}/{task.name}"
jupyter_files = list_files(source_name)
if jupyter_files is not None:
for jupyter_file in jupyter_files["content"]:
delete_file(jupyter_file["path"])
delete_file(source_name)
# remove MinIO files and directory
source_name = f"{PREFIX}/{uuid}"
minio_files = list_objects(source_name)
for minio_file in minio_files:
remove_object(minio_file.object_name)
remove_object(source_name)
db_session.commit()
except IntegrityError as e:
raise Forbidden(str(e))
except (InvalidRequestError, ProgrammingError, ResponseError) as e:
raise BadRequest(str(e))
return {"message": "Task deleted"}
def list_templates():
"""Lists all templates from our database.
Returns:
A list of all templates sorted by name in natural sort order.
"""
templates = db_session.query(Template) \
.all()
# sort the list in place, using natural sort
templates.sort(key=lambda o: [
int(t) if t.isdigit() else t.lower()
for t in re.split(r"(\d+)", o.name)
])
return [template.as_dict() for template in templates]
def total_rows_projects(name):
"""Returns the total number of records.
Args:
name(str):name to be searched
Returns:
total records.
"""
query = db_session.query(func.count(Project.uuid))
if name:
query = query.filter(Project.name.ilike(func.lower(f"%{name}%")))
rows = query.scalar()
return rows
def total_rows_tasks(name):
"""Counts the total number of records.
Args:
name(str): name
Returns:
rows
"""
query = db_session.query(func.count(Task.uuid))
if name:
query = query.filter(Task.name.ilike(func.lower(f"%{name}%")))
rows = query.scalar()
return rows
def update_operator(uuid, project_id, experiment_id, **kwargs):
"""Updates an operator in our database and adjusts the position of others.
Args:
uuid (str): the operator uuid to look for in our database.
project_id (str): the project uuid.
experiment_id (str): the experiment uuid.
**kwargs: arbitrary keyword arguments.
Returns:
The operator info.
"""
raise_if_project_does_not_exist(project_id)
raise_if_experiment_does_not_exist(experiment_id)
operator = Operator.query.get(uuid)
if operator is None:
raise NotFound("The specified operator does not exist")
raise_if_parameters_are_invalid(kwargs.get("parameters", {}))
dependencies = kwargs.get("dependencies")
if dependencies is not None:
raise_if_dependencies_are_invalid(project_id, experiment_id, dependencies, operator_id=uuid)
data = {"updated_at": datetime.utcnow()}
data.update(kwargs)
try:
db_session.query(Operator).filter_by(uuid=uuid).update(data)
db_session.commit()
except (InvalidRequestError, ProgrammingError) as e:
raise BadRequest(str(e))
check_status(operator)
return operator.as_dict()
def list_compare_results(project_id):
"""Lists all compare results under a project.
Args:
project_id (str): the project uuid.
Returns:
A list of all compare results.
"""
raise_if_project_does_not_exist(project_id)
compare_results = db_session.query(CompareResult) \
.filter_by(project_id=project_id) \
.order_by(CompareResult.created_at.asc()) \
.all()
return [compare_result.as_dict() for compare_result in compare_results]
def raise_if_operator_does_not_exist(operator_id, experiment_id=None):
"""Raises an exception if the specified operator does not exist.
Args:
operator_id (str): the operator uuid.
"""
operator = db_session.query(Operator) \
.filter_by(uuid=operator_id)
if operator.scalar() is None:
raise NotFound("The specified operator does not exist")
else:
# verify if operator is from the provided experiment
if experiment_id and operator.one().as_dict(
)["experimentId"] != experiment_id:
raise NotFound("The specified operator is from another experiment")
def list_experiments(project_id):
"""Lists all experiments under a project.
Args:
project_id (str): the project uuid.
Returns:
A list of all experiments.
"""
raise_if_project_does_not_exist(project_id)
experiments = db_session.query(Experiment) \
.filter_by(project_id=project_id) \
.order_by(Experiment.position.asc()) \
.all()
return [experiment.as_dict() for experiment in experiments]
def create_project(name=None, **kwargs):
"""Creates a new project in our database.
Args:
name (str): the project name.
Returns:
The project info.
"""
if not isinstance(name, str):
raise BadRequest("name is required")
check_project_name = db_session.query(Project).filter_by(name=name).first()
if check_project_name:
raise BadRequest("a project with that name already exists")
project = Project(uuid=uuid_alpha(),
name=name,
description=kwargs.get("description"))
db_session.add(project)
db_session.commit()
create_experiment(name="Experimento 1", project_id=project.uuid)
return project.as_dict()
def list_operators(project_id, experiment_id):
"""Lists all operators under an experiment.
Args:
project_id (str): the project uuid.
experiment_id (str): the experiment uuid.
Returns:
A list of all operator.
"""
raise_if_project_does_not_exist(project_id)
raise_if_experiment_does_not_exist(experiment_id)
operators = db_session.query(Operator) \
.filter_by(experiment_id=experiment_id) \
.all()
response = []
for operator in operators:
check_status(operator)
response.append(operator.as_dict())
return response
def update_experiment(uuid, project_id, **kwargs):
"""Updates an experiment in our database and adjusts the position of others.
Args:
uuid (str): the experiment uuid to look for in our database.
project_id (str): the project uuid.
**kwargs: arbitrary keyword arguments.
Returns:
The experiment info.
"""
raise_if_project_does_not_exist(project_id)
experiment = Experiment.query.get(uuid)
if experiment is None:
raise NOTFOUND
if "name" in kwargs:
name = kwargs["name"]
if name != experiment.name:
check_experiment_name = db_session.query(Experiment)\
.filter(Experiment.project_id == project_id)\
.filter(Experiment.name == name)\
.first()
if check_experiment_name:
raise BadRequest("an experiment with that name already exists")
# updates operators
if "template_id" in kwargs:
template_id = kwargs["template_id"]
del kwargs["template_id"]
template = Template.query.get(template_id)
if template is None:
raise BadRequest("The specified template does not exist")
# remove operators
Operator.query.filter(Operator.experiment_id == uuid).delete()
# save the last operator id created to create dependency on next operator
last_operator_id = None
for task_id in template.tasks:
operator_id = uuid_alpha()
dependencies = []
if last_operator_id is not None:
dependencies = [last_operator_id]
objects = [
Operator(uuid=operator_id,
experiment_id=uuid,
task_id=task_id,
dependencies=dependencies)
]
db_session.bulk_save_objects(objects)
last_operator_id = operator_id
data = {"updated_at": datetime.utcnow()}
data.update(kwargs)
try:
db_session.query(Experiment).filter_by(uuid=uuid).update(data)
db_session.commit()
except (InvalidRequestError, ProgrammingError) as e:
raise BadRequest(str(e))
fix_positions(project_id=experiment.project_id,
experiment_id=experiment.uuid,
new_position=experiment.position)
return experiment.as_dict()
def create_task(**kwargs):
"""Creates a new task in our database/object storage.
Args:
**kwargs: arbitrary keyword arguments.
Returns:
The task info.
"""
name = kwargs.get('name', None)
description = kwargs.get('description', None)
tags = kwargs.get('tags', None)
image = kwargs.get('image', None)
commands = kwargs.get('commands', None)
arguments = kwargs.get('arguments', None)
experiment_notebook = kwargs.get('experiment_notebook', None)
deployment_notebook = kwargs.get('deployment_notebook', None)
is_default = kwargs.get('is_default', None)
copy_from = kwargs.get('copy_from', None)
if not isinstance(name, str):
raise BadRequest("name is required")
if copy_from and (experiment_notebook or deployment_notebook):
raise BadRequest("Either provide notebooks or a task to copy from")
if tags is None or len(tags) == 0:
tags = ["DEFAULT"]
if any(tag not in VALID_TAGS for tag in tags):
valid_str = ",".join(VALID_TAGS)
raise BadRequest(f"Invalid tag. Choose any of {valid_str}")
# check if image is a valid docker image
if image:
pattern = re.compile('[a-z0-9.-]+([/]{1}[a-z0-9.-]+)+([:]{1}[a-z0-9.-]+){0,1}$')
if pattern.match(image) is None:
raise BadRequest("invalid docker image name")
check_comp_name = db_session.query(Task).filter_by(name=name).first()
if check_comp_name:
raise BadRequest("a task with that name already exists")
# creates a task with specified name,
# but copies notebooks from a source task
if copy_from:
return copy_task(name, description, tags, copy_from)
task_id = str(uuid_alpha())
# loads a sample notebook if none was sent
if experiment_notebook is None and "DATASETS" not in tags:
experiment_notebook = EXPERIMENT_NOTEBOOK
if deployment_notebook is None and "DATASETS" not in tags:
deployment_notebook = DEPLOYMENT_NOTEBOOK
# The new task must have its own task_id, experiment_id and operator_id.
# Notice these values are ignored when a notebook is run in a pipeline.
# They are only used by JupyterLab interface.
init_notebook_metadata(task_id, deployment_notebook, experiment_notebook)
# saves new notebooks to object storage
if "DATASETS" not in tags:
obj_name = f"{PREFIX}/{task_id}/Experiment.ipynb"
experiment_notebook_path = f"minio://{BUCKET_NAME}/{obj_name}"
put_object(obj_name, dumps(experiment_notebook).encode())
obj_name = f"{PREFIX}/{task_id}/Deployment.ipynb"
deployment_notebook_path = f"minio://{BUCKET_NAME}/{obj_name}"
put_object(obj_name, dumps(deployment_notebook).encode())
# create deployment notebook and experiment_notebook on jupyter
create_jupyter_files(task_name=name,
deployment_notebook=dumps(deployment_notebook).encode(),
experiment_notebook=dumps(experiment_notebook).encode())
else:
experiment_notebook_path = None
deployment_notebook_path = None
if commands is None or len(commands) == 0:
commands = DEFAULT_COMMANDS
if arguments is None or len(arguments) == 0:
arguments = DEFAULT_ARGUMENTS
# saves task info to the database
task = Task(uuid=task_id,
name=name,
description=description,
tags=tags,
image=image,
commands=commands,
arguments=arguments,
experiment_notebook_path=experiment_notebook_path,
deployment_notebook_path=deployment_notebook_path,
is_default=is_default)
db_session.add(task)
db_session.commit()
return task.as_dict()
def create_experiment(name=None, project_id=None, copy_from=None):
"""Creates a new experiment in our database and adjusts the position of others.
The new experiment is added to the end of the experiment list.
Args:
name (str): the experiment name.
project_id (str): the project uuid.
copy_from (str): the copy_from uuid.
Returns:
The experiment info.
"""
raise_if_project_does_not_exist(project_id)
if not isinstance(name, str):
raise BadRequest("name is required")
check_experiment_name = db_session.query(Experiment)\
.filter(Experiment.project_id == project_id)\
.filter(Experiment.name == name)\
.first()
if check_experiment_name:
raise BadRequest("an experiment with that name already exists")
experiment = Experiment(uuid=uuid_alpha(), name=name, project_id=project_id)
db_session.add(experiment)
db_session.commit()
if copy_from:
try:
experiment_find = find_by_experiment_id(experiment_id=copy_from)
source_operators = {}
for source_operator in experiment_find['operators']:
source_dependencies = source_operator.dependencies
kwargs = {
"task_id": source_operator.task_id,
"parameters": source_operator.parameters,
"dependencies": [],
"position_x": source_operator.position_x,
"position_y": source_operator.position_y
}
operator = create_operator(project_id, experiment.uuid, **kwargs)
source_operators[source_operator.uuid] = {
"copy_uuid": operator["uuid"],
"dependencies": source_dependencies
}
# update dependencies on new operators
for _, value in source_operators.items():
dependencies = [source_operators[d]['copy_uuid'] for d in value['dependencies']]
update_operator(value['copy_uuid'], project_id, experiment.uuid, dependencies=dependencies)
except NotFound:
delete_experiment(experiment.uuid, project_id)
raise BadRequest('Source experiment does not exist')
fix_positions(project_id=project_id,
experiment_id=experiment.uuid,
new_position=sys.maxsize) # will add to end of list
return experiment.as_dict()
