def update_template(uuid, **kwargs): """Updates a template in our database. Args: uuid (str): the template uuid to look for in our database. **kwargs: arbitrary keyword arguments. Returns: The template info. """ template = Template.query.get(uuid) if template is None: raise NotFound("The specified template does not exist") data = {"updated_at": datetime.utcnow()} data.update(kwargs) try: db_session.query(Template).filter_by(uuid=uuid).update(data) db_session.commit() except (InvalidRequestError, ProgrammingError) as e: raise BadRequest(str(e)) return template.as_dict()
def update_compare_result(uuid, project_id, **kwargs): """Updates a compare result in our database. Args: uuid (str): the a compare result uuid to look for in our database. project_id (str): the project uuid. **kwargs: arbitrary keyword arguments. Returns: The compare result info. """ raise_if_project_does_not_exist(project_id) compare_result = CompareResult.query.get(uuid) if compare_result is None: raise NOT_FOUND experiment_id = kwargs.get("experiment_id", None) if experiment_id: raise_if_experiment_does_not_exist(experiment_id) data = {"updated_at": datetime.utcnow()} data.update(kwargs) try: db_session.query(CompareResult).filter_by(uuid=uuid).update(data) db_session.commit() except (InvalidRequestError, ProgrammingError) as e: raise BadRequest(str(e)) return compare_result.as_dict()
def update_project(uuid, **kwargs): """Updates a project in our database. Args: uuid(str): the project uuid to look for in our database. **kwargs: arbitrary keyword arguments. Returns: The project info. """ project = Project.query.get(uuid) if project is None: raise NOT_FOUND if "name" in kwargs: name = kwargs["name"] if name != project.name: check_project_name = db_session.query(Project).filter_by( name=name).first() if check_project_name: raise BadRequest("a project with that name already exists") data = {"updated_at": datetime.utcnow()} data.update(kwargs) try: db_session.query(Project).filter_by(uuid=uuid).update(data) db_session.commit() except (InvalidRequestError, ProgrammingError) as e: raise BadRequest(str(e)) return project.as_dict()
def fix_positions(project_id, experiment_id=None, new_position=None): """Reorders the experiments in a project when an experiment is updated/deleted. Args: project_id (str): the project uuid. experiment_id (str): the experiment uuid. new_position (int): the position where the experiment is shown. """ other_experiments = db_session.query(Experiment) \ .filter_by(project_id=project_id) \ .filter(Experiment.uuid != experiment_id)\ .order_by(Experiment.position.asc())\ .all() if experiment_id is not None: experiment = Experiment.query.get(experiment_id) other_experiments.insert(new_position, experiment) for index, experiment in enumerate(other_experiments): data = {"position": index} is_last = (index == len(other_experiments) - 1) # if experiment_id WAS NOT informed, then set the higher position as is_active=True if experiment_id is None and is_last: data["is_active"] = True # if experiment_id WAS informed, then set experiment.is_active=True elif experiment_id is not None and experiment_id == experiment.uuid: data["is_active"] = True else: data["is_active"] = False db_session.query(Experiment).filter_by(uuid=experiment.uuid).update(data) db_session.commit()
def delete_multiple_projects(project_ids): """Delete multiple projects. Args: project_ids(str): list of projects Returns: message """ total_elements = len(project_ids) all_projects_ids = list_objects(project_ids) if total_elements < 1: return {"message": "please inform the uuid of the project"} projects = db_session.query(Project).filter( Project.uuid.in_(all_projects_ids)).all() experiments = db_session.query(Experiment).filter( Experiment.project_id.in_(objects_uuid(projects))).all() operators = db_session.query(Operator).filter(Operator.experiment_id.in_(objects_uuid(experiments))) \ .all() session = pre_delete(db_session, projects, total_elements, operators, experiments, all_projects_ids) for experiment in experiments: prefix = join("experiments", experiment.uuid) try: remove_objects(prefix=prefix) except Exception: pass session.commit() return {"message": "Successfully removed projects"}
def update_task(uuid, **kwargs): """Updates a task in our database/object storage. Args: uuid (str): the task uuid to look for in our database. **kwargs: arbitrary keyword arguments. Returns: The task info. """ task = Task.query.get(uuid) if task is None: raise NOT_FOUND if "name" in kwargs: name = kwargs["name"] if name != task.name: check_comp_name = db_session.query(Task).filter_by(name=name).first() if check_comp_name: raise BadRequest("a task with that name already exists") if "tags" in kwargs: tags = kwargs["tags"] if any(tag not in VALID_TAGS for tag in tags): valid_str = ",".join(VALID_TAGS) raise BadRequest(f"Invalid tag. Choose any of {valid_str}") if "experiment_notebook" in kwargs: obj_name = f"{PREFIX}/{uuid}/Experiment.ipynb" put_object(obj_name, dumps(kwargs["experiment_notebook"]).encode()) del kwargs["experiment_notebook"] if "deployment_notebook" in kwargs: obj_name = f"{PREFIX}/{uuid}/Deployment.ipynb" put_object(obj_name, dumps(kwargs["deployment_notebook"]).encode()) del kwargs["deployment_notebook"] # store the name to use it after update old_name = task.name try: data = {"updated_at": datetime.utcnow()} data.update(kwargs) db_session.query(Task).filter_by(uuid=uuid).update(data) db_session.commit() except (InvalidRequestError, ProgrammingError) as e: raise BadRequest(str(e)) # update jupyter folder name if name changed if old_name != task.name: update_folder_name(f"{PREFIX}/{old_name}", f"{PREFIX}/{task.name}") return task.as_dict()
def create_template(name=None, experiment_id=None, **kwargs): """Creates a new template in our database. Args: name (str): the template name. Returns: The template info. """ if not isinstance(name, str): raise BadRequest("name is required") if not isinstance(experiment_id, str): raise BadRequest("experimentId is required") try: raise_if_experiment_does_not_exist(experiment_id) except NotFound as e: raise BadRequest(e.description) operators = db_session.query(Operator) \ .filter_by(experiment_id=experiment_id) \ .order_by(Operator.created_at.asc()) \ .all() # JSON array order of elements are preserved, # so there is no need to save positions tasks = [operator.task_id for operator in operators] template = Template(uuid=uuid_alpha(), name=name, tasks=tasks) db_session.add(template) db_session.commit() return template.as_dict()
def delete_operator(uuid, project_id, experiment_id): """Delete an operator in our database. Args: uuid (str): the operator uuid to look for in our database. project_id (str): the project uuid. experiment_id (str): the experiment uuid. Returns: The deletion result. """ raise_if_project_does_not_exist(project_id) raise_if_experiment_does_not_exist(experiment_id) operator = Operator.query.get(uuid) if operator is None: raise NotFound("The specified operator does not exist") # check if other operators contains the operator being deleted # in dependencies and remove this operator from dependencies operators = db_session.query(Operator) \ .filter_by(experiment_id=experiment_id) \ .filter(Operator.uuid != uuid)\ .all() for op in operators: if uuid in op.dependencies: dependencies = op.dependencies.remove(uuid) if dependencies is None: dependencies = [] update_operator(op.uuid, project_id, experiment_id, dependencies=dependencies) db_session.delete(operator) db_session.commit() return {"message": "Operator deleted"}
def pagination_tasks(name, page, page_size, order): """Task pagination. Args: name (str): task name page (int): records size page_size (int): page number order (str): order by Raises: InternalServerError: MySQL error Returns: dict: query response """ try: query = db_session.query(Task) if name: query = query.filter(Task.name.ilike(func.lower(f"%{name}%"))) if page is not None and order is None: if page == 0: query = query.order_by(Task.name) else: query = query.order_by(text('name')).limit(page_size).offset((page - 1) * page_size) else: query = pagination_ordering(query, page_size, page, order) tasks = query.all() total_rows = total_rows_tasks(name) return {'total': total_rows, 'tasks': [task.as_dict() for task in tasks]} except (OperationalError, InternalError): raise InternalServerError("The service is unavailable. Try your call again.")
def get_tasks_by_tag(tag): """Get all tasks with a specific tag. Returns: A list of tasks. """ tasks = db_session.query(Task).filter(Task.tags.contains([tag])).all() return [task.as_dict() for task in tasks]
def raise_if_task_does_not_exist(task_id): """Raises an exception if the specified task does not exist. Args: task_id (str): the task uuid. """ exists = db_session.query(Task.uuid) \ .filter_by(uuid=task_id) \ .scalar() is not None if not exists: raise NotFound("The specified task does not exist")
def raise_if_project_does_not_exist(project_id): """Raises an exception if the specified project does not exist. Args: project_id (str): the project uuid. """ exists = db_session.query(Project.uuid) \ .filter_by(uuid=project_id) \ .scalar() is not None if not exists: raise NotFound("The specified project does not exist")
def raise_if_experiment_does_not_exist(experiment_id): """Raises an exception if the specified experiment does not exist. Args: experiment_id (str): the experiment uuid. """ exists = db_session.query(Experiment.uuid) \ .filter_by(uuid=experiment_id) \ .scalar() is not None if not exists: raise NotFound("The specified experiment does not exist")
def delete_task(uuid): """Delete a task in our database/object storage. Args: uuid (str): the task uuid to look for in our database. Returns: The task info. """ task = Task.query.get(uuid) if task is None: raise NOT_FOUND try: db_session.query(Task).filter_by(uuid=uuid).delete() # remove files and directory from jupyter notebook server source_name = f"{PREFIX}/{task.name}" jupyter_files = list_files(source_name) if jupyter_files is not None: for jupyter_file in jupyter_files["content"]: delete_file(jupyter_file["path"]) delete_file(source_name) # remove MinIO files and directory source_name = f"{PREFIX}/{uuid}" minio_files = list_objects(source_name) for minio_file in minio_files: remove_object(minio_file.object_name) remove_object(source_name) db_session.commit() except IntegrityError as e: raise Forbidden(str(e)) except (InvalidRequestError, ProgrammingError, ResponseError) as e: raise BadRequest(str(e)) return {"message": "Task deleted"}
def list_templates(): """Lists all templates from our database. Returns: A list of all templates sorted by name in natural sort order. """ templates = db_session.query(Template) \ .all() # sort the list in place, using natural sort templates.sort(key=lambda o: [ int(t) if t.isdigit() else t.lower() for t in re.split(r"(\d+)", o.name) ]) return [template.as_dict() for template in templates]
def total_rows_projects(name): """Returns the total number of records. Args: name(str):name to be searched Returns: total records. """ query = db_session.query(func.count(Project.uuid)) if name: query = query.filter(Project.name.ilike(func.lower(f"%{name}%"))) rows = query.scalar() return rows
def total_rows_tasks(name): """Counts the total number of records. Args: name(str): name Returns: rows """ query = db_session.query(func.count(Task.uuid)) if name: query = query.filter(Task.name.ilike(func.lower(f"%{name}%"))) rows = query.scalar() return rows
def update_operator(uuid, project_id, experiment_id, **kwargs): """Updates an operator in our database and adjusts the position of others. Args: uuid (str): the operator uuid to look for in our database. project_id (str): the project uuid. experiment_id (str): the experiment uuid. **kwargs: arbitrary keyword arguments. Returns: The operator info. """ raise_if_project_does_not_exist(project_id) raise_if_experiment_does_not_exist(experiment_id) operator = Operator.query.get(uuid) if operator is None: raise NotFound("The specified operator does not exist") raise_if_parameters_are_invalid(kwargs.get("parameters", {})) dependencies = kwargs.get("dependencies") if dependencies is not None: raise_if_dependencies_are_invalid(project_id, experiment_id, dependencies, operator_id=uuid) data = {"updated_at": datetime.utcnow()} data.update(kwargs) try: db_session.query(Operator).filter_by(uuid=uuid).update(data) db_session.commit() except (InvalidRequestError, ProgrammingError) as e: raise BadRequest(str(e)) check_status(operator) return operator.as_dict()
def list_compare_results(project_id): """Lists all compare results under a project. Args: project_id (str): the project uuid. Returns: A list of all compare results. """ raise_if_project_does_not_exist(project_id) compare_results = db_session.query(CompareResult) \ .filter_by(project_id=project_id) \ .order_by(CompareResult.created_at.asc()) \ .all() return [compare_result.as_dict() for compare_result in compare_results]
def raise_if_operator_does_not_exist(operator_id, experiment_id=None): """Raises an exception if the specified operator does not exist. Args: operator_id (str): the operator uuid. """ operator = db_session.query(Operator) \ .filter_by(uuid=operator_id) if operator.scalar() is None: raise NotFound("The specified operator does not exist") else: # verify if operator is from the provided experiment if experiment_id and operator.one().as_dict( )["experimentId"] != experiment_id: raise NotFound("The specified operator is from another experiment")
def list_experiments(project_id): """Lists all experiments under a project. Args: project_id (str): the project uuid. Returns: A list of all experiments. """ raise_if_project_does_not_exist(project_id) experiments = db_session.query(Experiment) \ .filter_by(project_id=project_id) \ .order_by(Experiment.position.asc()) \ .all() return [experiment.as_dict() for experiment in experiments]
def create_project(name=None, **kwargs): """Creates a new project in our database. Args: name (str): the project name. Returns: The project info. """ if not isinstance(name, str): raise BadRequest("name is required") check_project_name = db_session.query(Project).filter_by(name=name).first() if check_project_name: raise BadRequest("a project with that name already exists") project = Project(uuid=uuid_alpha(), name=name, description=kwargs.get("description")) db_session.add(project) db_session.commit() create_experiment(name="Experimento 1", project_id=project.uuid) return project.as_dict()
def list_operators(project_id, experiment_id): """Lists all operators under an experiment. Args: project_id (str): the project uuid. experiment_id (str): the experiment uuid. Returns: A list of all operator. """ raise_if_project_does_not_exist(project_id) raise_if_experiment_does_not_exist(experiment_id) operators = db_session.query(Operator) \ .filter_by(experiment_id=experiment_id) \ .all() response = [] for operator in operators: check_status(operator) response.append(operator.as_dict()) return response
def update_experiment(uuid, project_id, **kwargs): """Updates an experiment in our database and adjusts the position of others. Args: uuid (str): the experiment uuid to look for in our database. project_id (str): the project uuid. **kwargs: arbitrary keyword arguments. Returns: The experiment info. """ raise_if_project_does_not_exist(project_id) experiment = Experiment.query.get(uuid) if experiment is None: raise NOTFOUND if "name" in kwargs: name = kwargs["name"] if name != experiment.name: check_experiment_name = db_session.query(Experiment)\ .filter(Experiment.project_id == project_id)\ .filter(Experiment.name == name)\ .first() if check_experiment_name: raise BadRequest("an experiment with that name already exists") # updates operators if "template_id" in kwargs: template_id = kwargs["template_id"] del kwargs["template_id"] template = Template.query.get(template_id) if template is None: raise BadRequest("The specified template does not exist") # remove operators Operator.query.filter(Operator.experiment_id == uuid).delete() # save the last operator id created to create dependency on next operator last_operator_id = None for task_id in template.tasks: operator_id = uuid_alpha() dependencies = [] if last_operator_id is not None: dependencies = [last_operator_id] objects = [ Operator(uuid=operator_id, experiment_id=uuid, task_id=task_id, dependencies=dependencies) ] db_session.bulk_save_objects(objects) last_operator_id = operator_id data = {"updated_at": datetime.utcnow()} data.update(kwargs) try: db_session.query(Experiment).filter_by(uuid=uuid).update(data) db_session.commit() except (InvalidRequestError, ProgrammingError) as e: raise BadRequest(str(e)) fix_positions(project_id=experiment.project_id, experiment_id=experiment.uuid, new_position=experiment.position) return experiment.as_dict()
def create_task(**kwargs): """Creates a new task in our database/object storage. Args: **kwargs: arbitrary keyword arguments. Returns: The task info. """ name = kwargs.get('name', None) description = kwargs.get('description', None) tags = kwargs.get('tags', None) image = kwargs.get('image', None) commands = kwargs.get('commands', None) arguments = kwargs.get('arguments', None) experiment_notebook = kwargs.get('experiment_notebook', None) deployment_notebook = kwargs.get('deployment_notebook', None) is_default = kwargs.get('is_default', None) copy_from = kwargs.get('copy_from', None) if not isinstance(name, str): raise BadRequest("name is required") if copy_from and (experiment_notebook or deployment_notebook): raise BadRequest("Either provide notebooks or a task to copy from") if tags is None or len(tags) == 0: tags = ["DEFAULT"] if any(tag not in VALID_TAGS for tag in tags): valid_str = ",".join(VALID_TAGS) raise BadRequest(f"Invalid tag. Choose any of {valid_str}") # check if image is a valid docker image if image: pattern = re.compile('[a-z0-9.-]+([/]{1}[a-z0-9.-]+)+([:]{1}[a-z0-9.-]+){0,1}$') if pattern.match(image) is None: raise BadRequest("invalid docker image name") check_comp_name = db_session.query(Task).filter_by(name=name).first() if check_comp_name: raise BadRequest("a task with that name already exists") # creates a task with specified name, # but copies notebooks from a source task if copy_from: return copy_task(name, description, tags, copy_from) task_id = str(uuid_alpha()) # loads a sample notebook if none was sent if experiment_notebook is None and "DATASETS" not in tags: experiment_notebook = EXPERIMENT_NOTEBOOK if deployment_notebook is None and "DATASETS" not in tags: deployment_notebook = DEPLOYMENT_NOTEBOOK # The new task must have its own task_id, experiment_id and operator_id. # Notice these values are ignored when a notebook is run in a pipeline. # They are only used by JupyterLab interface. init_notebook_metadata(task_id, deployment_notebook, experiment_notebook) # saves new notebooks to object storage if "DATASETS" not in tags: obj_name = f"{PREFIX}/{task_id}/Experiment.ipynb" experiment_notebook_path = f"minio://{BUCKET_NAME}/{obj_name}" put_object(obj_name, dumps(experiment_notebook).encode()) obj_name = f"{PREFIX}/{task_id}/Deployment.ipynb" deployment_notebook_path = f"minio://{BUCKET_NAME}/{obj_name}" put_object(obj_name, dumps(deployment_notebook).encode()) # create deployment notebook and experiment_notebook on jupyter create_jupyter_files(task_name=name, deployment_notebook=dumps(deployment_notebook).encode(), experiment_notebook=dumps(experiment_notebook).encode()) else: experiment_notebook_path = None deployment_notebook_path = None if commands is None or len(commands) == 0: commands = DEFAULT_COMMANDS if arguments is None or len(arguments) == 0: arguments = DEFAULT_ARGUMENTS # saves task info to the database task = Task(uuid=task_id, name=name, description=description, tags=tags, image=image, commands=commands, arguments=arguments, experiment_notebook_path=experiment_notebook_path, deployment_notebook_path=deployment_notebook_path, is_default=is_default) db_session.add(task) db_session.commit() return task.as_dict()
def create_experiment(name=None, project_id=None, copy_from=None): """Creates a new experiment in our database and adjusts the position of others. The new experiment is added to the end of the experiment list. Args: name (str): the experiment name. project_id (str): the project uuid. copy_from (str): the copy_from uuid. Returns: The experiment info. """ raise_if_project_does_not_exist(project_id) if not isinstance(name, str): raise BadRequest("name is required") check_experiment_name = db_session.query(Experiment)\ .filter(Experiment.project_id == project_id)\ .filter(Experiment.name == name)\ .first() if check_experiment_name: raise BadRequest("an experiment with that name already exists") experiment = Experiment(uuid=uuid_alpha(), name=name, project_id=project_id) db_session.add(experiment) db_session.commit() if copy_from: try: experiment_find = find_by_experiment_id(experiment_id=copy_from) source_operators = {} for source_operator in experiment_find['operators']: source_dependencies = source_operator.dependencies kwargs = { "task_id": source_operator.task_id, "parameters": source_operator.parameters, "dependencies": [], "position_x": source_operator.position_x, "position_y": source_operator.position_y } operator = create_operator(project_id, experiment.uuid, **kwargs) source_operators[source_operator.uuid] = { "copy_uuid": operator["uuid"], "dependencies": source_dependencies } # update dependencies on new operators for _, value in source_operators.items(): dependencies = [source_operators[d]['copy_uuid'] for d in value['dependencies']] update_operator(value['copy_uuid'], project_id, experiment.uuid, dependencies=dependencies) except NotFound: delete_experiment(experiment.uuid, project_id) raise BadRequest('Source experiment does not exist') fix_positions(project_id=project_id, experiment_id=experiment.uuid, new_position=sys.maxsize) # will add to end of list return experiment.as_dict()