def create_run(self, project_id: str, deployment_id: str): """ Starts a new run in Kubeflow Pipelines. Parameters ---------- project_id : str deployment_id : str Returns ------- projects.schemas.run.Run Raises ------ NotFound When any of project_id, or deployment_id does not exist. """ deployment = self.session.query(models.Deployment).get(deployment_id) if deployment is None: raise NotFound("The specified deployment does not exist") # Removes operators that don't have a deployment_notebook (eg. Upload de Dados). # Then, fix dependencies in their children. operators = self.remove_non_deployable_operators(deployment.operators) try: run = kfp_runs.start_run(operators=operators, project_id=deployment.project_id, experiment_id=deployment.experiment_id, deployment_id=deployment_id, deployment_name=deployment.name) except ValueError as e: raise BadRequest(str(e)) # Deploy monitoring tasks monitorings = self.monitoring_controller.list_monitorings(project_id=project_id, deployment_id=deployment_id).monitorings if monitorings: for monitoring in monitorings: self.background_tasks.add_task( deploy_monitoring, deployment_id=deployment_id, experiment_id=deployment.experiment_id, run_id=run["uuid"], task_id=monitoring.task_id, monitoring_id=monitoring.uuid ) update_data = {"status": "Pending"} self.session.query(models.Operator) \ .filter_by(deployment_id=deployment_id) \ .update(update_data) self.session.commit() run["deploymentId"] = deployment_id return run
def undeploy_monitoring(monitoring_id): """ Undeploy the service and trigger of a given monitoring_id. Parameters ---------- monitoring_id : str Raises ------ NotFound When monitoring resources do not exist. """ load_kube_config() api = client.CustomObjectsApi() try: # Undeploy service service_name = f"service-{monitoring_id}" service_custom_object = api.get_namespaced_custom_object( group="serving.knative.dev", version="v1", namespace=KF_PIPELINES_NAMESPACE, plural="services", name=service_name, ) undeploy_pipeline( name=service_custom_object["metadata"]["name"], kind=service_custom_object["kind"], namespace=service_custom_object["metadata"]["namespace"], ) # Undeploy trigger trigger_name = f"trigger-{monitoring_id}" trigger_custom_object = api.get_namespaced_custom_object( group="eventing.knative.dev", version="v1", namespace=KF_PIPELINES_NAMESPACE, plural="triggers", name=trigger_name, ) undeploy_pipeline( name=trigger_custom_object["metadata"]["name"], kind=trigger_custom_object["kind"], namespace=trigger_custom_object["metadata"]["namespace"], ) except ApiException: raise NotFound( code="MonitoringNotFound", message="The specified monitoring does not exist.", )
def deploy_run(self, deployment): """ Starts a new run in Kubeflow Pipelines. Parameters ---------- deployment : projects.models.deployment.Deployment Returns ------- projects.schemas.run.Run Raises ------ NotFound When any of project_id, or deployment_id does not exist. """ if deployment is None: raise NotFound( code="DeploymentNotFound", message="The specified deployment does not exist", ) # Removes operators that don't have a deployment_notebook (eg. Upload de Dados). # Then, fix dependencies in their children. operators = self.remove_non_deployable_operators(deployment.operators) try: run = kfp_runs.start_run( operators=operators, project_id=deployment.project_id, experiment_id=deployment.experiment_id, deployment_id=deployment.uuid, deployment_name=deployment.name, ) except ValueError: raise BadRequest( code="MissingRequiredOperatorId", message=f"Necessary at least one operator.", ) # Remove the object from the operator session in order not to update the database, # Just need to remove the dependencies for the runs. for operator in deployment.operators: self.session.expunge(operator) return schemas.Run.from_orm(run)
def terminate_run(self, deployment_id): """ Terminates a run in Kubeflow Pipelines. Parameters ---------- deployment_id : str Returns ------- projects.schemas.message.Message Raises ------ NotFound When deployment run does not exist. """ load_kube_config() api = client.CustomObjectsApi() custom_objects = api.list_namespaced_custom_object( "machinelearning.seldon.io", "v1", KF_PIPELINES_NAMESPACE, "seldondeployments", ) deployments_objects = custom_objects["items"] if deployments_objects: for deployment in deployments_objects: if deployment["metadata"]["name"] == deployment_id: undeploy_pipeline( name=deployment["metadata"]["name"], kind=deployment["kind"], namespace=deployment["metadata"]["namespace"], ) deployment_run = get_deployment_runs(deployment_id) if not deployment_run: raise NotFound(code="RunNotFound", message="The specified run does not exist.") kfp_client().runs.delete_run(deployment_run["runId"]) return schemas.Message(message="Deployment deleted")
def raise_if_experiment_does_not_exist(self, experiment_id: str): """ Raises an exception if the specified experiment does not exist. Parameters ---------- experiment_id : str Raises ------ NotFound """ exists = self.session.query(models.Experiment.uuid) \ .filter_by(uuid=experiment_id) \ .scalar() is not None if not exists: raise NotFound("The specified experiment does not exist")
def get_dataset_name(self, operator_id, experiment_id): """ Get operator's dataset name. Parameters ---------- operator_id : str experiment_id: str Returns ------- str The dataset name. Raises ------ NotFound When a run does not have a dataset. """ operator = self.session.query(models.Operator).get(operator_id) dataset_name = operator.parameters.get("dataset") if dataset_name is None: operators = ( self.session.query(models.Operator) .filter_by(experiment_id=experiment_id) .filter(models.Operator.uuid != operator_id) .all() ) for operator in operators: dataset_name = operator.parameters.get("dataset") if dataset_name: break if dataset_name is None: raise NotFound( code="DatasetNotFound", message="No dataset assigned to the run" ) return dataset_name
def raise_if_monitoring_does_not_exist(self, monitoring_id: str): """ Raises an exception if the specified monitoring does not exist. Parameters ---------- monitoring_id : str Raises ------ NotFound """ exists = (self.session.query( models.Monitoring.uuid).filter_by(uuid=monitoring_id).scalar() is not None) if not exists: raise NotFound( code="MonitoringNotFound", message="The specified monitoring does not exist", )
def create_run(self, project_id: str, experiment_id: str): """ Starts a new run in Kubeflow Pipelines. Parameters ---------- project_id : str experiment_id : str Returns ------- dict The run attributes. Raises ------ NotFound When experiment_id does not exist. """ experiment = self.session.query(models.Experiment).get(experiment_id) if experiment is None: raise NotFound( code="ExperimentNotFound", message="The specified experiment does not exist", ) run = kfp_runs.start_run( project_id=project_id, experiment_id=experiment_id, operators=experiment.operators, ) run["experimentId"] = experiment_id update_data = {"status": "Pending", "status_message": None} self.session.query(models.Operator).filter_by( experiment_id=experiment_id).update(update_data) self.session.commit() return schemas.Run.from_orm(run)
def list_parameters(self, task_id: str): """ Lists all parameters from the experiment notebook of a task. Parameters ---------- task_id : str Returns ------- list A list of all parameters. Raises ------ NotFound When task_id does not exist. """ task = self.session.query(models.Task).get(task_id) if task is None: raise NotFound("The specified task does not exist") return task.parameters
def list_metrics(self, project_id: str, experiment_id: str, run_id: str, operator_id: str): """ Lists all metrics from object storage. Parameters ---------- project_id : str experiment_id : str run_id : str The run_id. If `run_id=latest`, then returns metrics from the latest run_id. operator_id : str Returns ------- list A list of metrics. """ try: return platiagro.list_metrics(experiment_id=experiment_id, operator_id=operator_id, run_id=run_id) except FileNotFoundError as e: raise NotFound(str(e))
# -*- coding: utf-8 -*- """Monitorings controller.""" import warnings from sqlalchemy import event from projects import models, schemas from projects.controllers.deployments.runs.runs import RunController from projects.controllers.tasks import TaskController from projects.controllers.utils import uuid_alpha from projects.exceptions import BadRequest, NotFound from projects.kfp.monitorings import deploy_monitoring, undeploy_monitoring NOT_FOUND = NotFound(code="MonitoringNotFound", message="The specified monitoring does not exist") class MonitoringController: def __init__(self, session, background_tasks=None): self.session = session self.background_tasks = background_tasks self.run_controller = RunController(session) self.task_controller = TaskController(session) @staticmethod @event.listens_for(models.Monitoring, "after_delete") def after_delete(_mapper, _connection, target): """ Starts a pipeline that deletes K8s resources associated with target monitoring. Parameters ----------
def create_template(self, template: schemas.TemplateCreate): """ Creates a new template in our database. Parameters ---------- template : projects.schemas.template.TemplateCreate Returns ------- projects.schemas.template.Template Raises ------ BadRequest When the project attributes are invalid. """ if not isinstance(template.name, str): raise BadRequest("name is required") if template.experiment_id: exists = self.session.query(models.Experiment.uuid) \ .filter_by(uuid=template.experiment_id) \ .scalar() is not None if not exists: raise NotFound("The specified experiment does not exist") operators = self.session.query(models.Operator) \ .filter_by(experiment_id=template.experiment_id) \ .all() elif template.deployment_id: exists = self.session.query(models.Deployment.uuid) \ .filter_by(uuid=template.deployment_id) \ .scalar() is not None if not exists: raise NotFound("The specified deployment does not exist") operators = self.session.query(models.Operator) \ .filter_by(deployment_id=template.deployment_id) \ .all() else: raise BadRequest( "experimentId or deploymentId needed to create template.") stored_template = self.session.query(models.Template) \ .filter_by(name=template.name) \ .first() if stored_template: raise BadRequest("a template with that name already exists") # order operators by dependencies operators_ordered = [] while len(operators) != len(operators_ordered): for operator in operators: self.order_operators_by_dependencies(operators_ordered, operator) # JSON array order of elements are preserved, so there is no need to save positions tasks = [] for uuid in operators_ordered: operator = next((op for op in operators if op.uuid == uuid), None) task = { "uuid": operator.uuid, "task_id": operator.task_id, "dependencies": operator.dependencies, "position_x": operator.position_x, "position_y": operator.position_y, } tasks.append(task) template = models.Template(uuid=uuid_alpha(), name=template.name, tasks=tasks) self.session.add(template) self.session.commit() self.session.refresh(template) return schemas.Template.from_orm(template)
# -*- coding: utf-8 -*- """Templates controller.""" import re from datetime import datetime from projects import models, schemas from projects.controllers.utils import uuid_alpha from projects.exceptions import BadRequest, NotFound NOT_FOUND = NotFound("The specified template does not exist") class TemplateController: def __init__(self, session): self.session = session def raise_if_template_does_not_exist(self, template_id: str): """ Raises an exception if the specified template does not exist. Parameters ---------- template_id :str Raises ------ NotFound """ exists = self.session.query(models.Template.uuid) \ .filter_by(uuid=template_id) \ .scalar() is not None
# -*- coding: utf-8 -*- """Projects controller.""" from datetime import datetime from os.path import join from typing import Optional from sqlalchemy import asc, desc, func from projects import models, schemas from projects.controllers.experiments import ExperimentController from projects.controllers.utils import uuid_alpha from projects.exceptions import BadRequest, NotFound from projects.utils import now NOT_FOUND = NotFound(code="ProjectNotFound", message="The specified project does not exist") class ProjectController: def __init__(self, session, kubeflow_userid=None): self.session = session self.experiment_controller = ExperimentController(session) self.kubeflow_userid = kubeflow_userid def raise_if_project_does_not_exist(self, project_id: str): """ Raises an exception if the specified project does not exist. Parameters ---------- project_id : str
# -*- coding: utf-8 -*- """Experiments Runs controller.""" from kfp_server_api.rest import ApiException from projects import models, schemas from projects.exceptions import NotFound from projects.kfp import runs as kfp_runs NOT_FOUND = NotFound("The specified run does not exist") class RunController: def __init__(self, session): self.session = session def raise_if_run_does_not_exist(self, run_id: str, experiment_id: str): """ Raises an exception if the specified run does not exist. Parameters ---------- run_id : str experiment_id : str Raises ------ NotFound """ try: kfp_runs.get_run(experiment_id=experiment_id, run_id=run_id) except (ApiException, ValueError):
# -*- coding: utf-8 -*- """Projects controller.""" from datetime import datetime from os.path import join from typing import Optional from sqlalchemy import asc, desc, func from projects import models, schemas from projects.controllers.experiments import ExperimentController from projects.controllers.utils import uuid_alpha from projects.exceptions import BadRequest, NotFound from projects.object_storage import remove_objects NOT_FOUND = NotFound("The specified project does not exist") class ProjectController: def __init__(self, session): self.session = session self.experiment_controller = ExperimentController(session) def raise_if_project_does_not_exist(self, project_id: str): """ Raises an exception if the specified project does not exist. Parameters ---------- project_id : str Raises
# -*- coding: utf-8 -*- """Comparison controller.""" from datetime import datetime from projects import models, schemas from projects.controllers.experiments import ExperimentController from projects.controllers.utils import uuid_alpha from projects.exceptions import BadRequest, NotFound NOT_FOUND = NotFound("The specified comparison does not exist") class ComparisonController: def __init__(self, session): self.session = session self.experiment_controller = ExperimentController(session) def raise_if_comparison_does_not_exist(self, comparison_id: str): """ Raises an exception if the specified comparison does not exist. Parameters ---------- comparison_id : str Raises ------ NotFound """ exists = self.session.query(models.Comparison.uuid) \ .filter_by(uuid=comparison_id) \
# -*- coding: utf-8 -*- """Experiments controller.""" import sys from datetime import datetime from typing import Optional from projects import models, schemas from projects.controllers.operators import OperatorController from projects.controllers.utils import uuid_alpha from projects.exceptions import BadRequest, NotFound NOT_FOUND = NotFound("The specified experiment does not exist") class ExperimentController: def __init__(self, session): self.session = session self.operator_controller = OperatorController(session) def raise_if_experiment_does_not_exist(self, experiment_id: str): """ Raises an exception if the specified experiment does not exist. Parameters ---------- experiment_id : str Raises ------ NotFound """
# -*- coding: utf-8 -*- """Deployments controller.""" import sys from datetime import datetime from projects import models, schemas from projects.controllers.deployments.runs import RunController from projects.controllers.experiments import ExperimentController from projects.controllers.operators import OperatorController from projects.controllers.templates import TemplateController from projects.controllers.utils import uuid_alpha from projects.exceptions import BadRequest, NotFound from projects.kfp.deployments import get_deployment_runs, list_deployments_runs from projects.kfp.monitorings import undeploy_monitoring NOT_FOUND = NotFound("The specified deployment does not exist") class DeploymentController: def __init__(self, session, background_tasks=None): self.session = session self.experiment_controller = ExperimentController(session) self.operator_controller = OperatorController(session) self.run_controller = RunController(session) self.template_controller = TemplateController(session) self.background_tasks = background_tasks def raise_if_deployment_does_not_exist(self, deployment_id: str): """ Raises an exception if the specified deployment does not exist.
# -*- coding: utf-8 -*- """Templates controller.""" import re from datetime import datetime from projects import models, schemas from projects.controllers.utils import uuid_alpha from projects.exceptions import BadRequest, NotFound from projects.utils import now NOT_FOUND = NotFound(code="TemplateNotFound", message="The specified template does not exist") class TemplateController: def __init__(self, session, kubeflow_userid=None): self.session = session self.kubeflow_userid = kubeflow_userid def raise_if_template_does_not_exist(self, template_id: str): """ Raises an exception if the specified template does not exist. Parameters ---------- template_id :str Raises ------ NotFound """
f"platiagro/platiagro-experiment-image:{__version__}", ) TASK_DEFAULT_CPU_LIMIT = os.getenv("TASK_DEFAULT_CPU_LIMIT", "2000m") TASK_DEFAULT_CPU_REQUEST = os.getenv("TASK_DEFAULT_CPU_REQUEST", "100m") TASK_DEFAULT_MEMORY_LIMIT = os.getenv("TASK_DEFAULT_MEMORY_LIMIT", "10Gi") TASK_DEFAULT_MEMORY_REQUEST = os.getenv("TASK_DEFAULT_MEMORY_REQUEST", "2Gi") TASK_DEFAULT_READINESS_INITIAL_DELAY_SECONDS = int( os.getenv( "TASK_DEFAULT_READINESS_INITIAL_DELAY_SECONDS", "60", )) EMAIL_MESSAGE_TEMPLATE = pkgutil.get_data("projects", "config/email-template.html") NOT_FOUND = NotFound(code="TaskNotFound", message="The specified task does not exist") class TaskController: def __init__(self, session, background_tasks=None): self.session = session self.background_tasks = background_tasks def raise_if_task_does_not_exist(self, task_id: str): """ Raises an exception if the specified task does not exist. Parameters ---------- task_id : str
"""Deployments controller.""" import sys import warnings from datetime import datetime from sqlalchemy import event from projects import models, schemas from projects.controllers.deployments.runs import RunController from projects.controllers.templates import TemplateController from projects.controllers.utils import uuid_alpha from projects.controllers.tasks import TaskController from projects.exceptions import BadRequest, NotFound from projects.utils import now NOT_FOUND = NotFound(code="DeploymentNotFound", message="The specified deployment does not exist") # Distance on the X axis from the leftmost operator DATASET_OPERATOR_DISTANCE = 300 FONTE_DE_DADOS = "Fonte de dados" class DeploymentController: def __init__(self, session, background_tasks=None, kubeflow_userid=None): self.session = session self.run_controller = RunController(session) self.template_controller = TemplateController( session, kubeflow_userid=kubeflow_userid) self.task_controller = TaskController(session, background_tasks) self.background_tasks = background_tasks
# -*- coding: utf-8 -*- """Deployments Runs controller.""" from kubernetes import client from kubernetes.client.rest import ApiException from projects import models, schemas from projects.exceptions import BadRequest, NotFound from projects.kfp import KF_PIPELINES_NAMESPACE, kfp_client from projects.kfp import runs as kfp_runs from projects.kfp.deployments import get_deployment_runs from projects.kfp.pipeline import undeploy_pipeline from projects.kubernetes.kube_config import load_kube_config from projects.kubernetes.seldon import get_seldon_deployment_url NOT_FOUND = NotFound(code="RunNotFound", message="The specified run does not exist") class RunController: def __init__(self, session): self.session = session def raise_if_run_does_not_exist(self, run_id: str, deployment_id: str): """ Raises an exception if the specified run does not exist. Parameters ---------- run_id : str deployment_id : str
# -*- coding: utf-8 -*- """Monitorings controller.""" from projects import models, schemas from projects.controllers.tasks import TaskController from projects.controllers.utils import uuid_alpha from projects.exceptions import NotFound NOT_FOUND = NotFound("The specified monitoring does not exist") class MonitoringController: def __init__(self, session): self.session = session self.task_controller = TaskController(session) def list_monitorings(self, project_id: str, deployment_id: str): """ Lists all monitorings under a deployment. Parameters ---------- project_id : str deployment_id : str Returns ------- projects.schemas.monitoring.MonitoringList """ monitorings = self.session.query(models.Monitoring) \ .filter_by(deployment_id=deployment_id) \ .order_by(models.Monitoring.created_at.asc()) \
import logging from typing import Optional import requests from platiagro import load_dataset from projects import models, schemas from projects.controllers.utils import ( parse_dataframe_to_seldon_request, parse_file_buffer_to_seldon_request, uuid_alpha, ) from projects.exceptions import BadRequest, NotFound from projects.kubernetes.seldon import get_seldon_deployment_url NOT_FOUND = NotFound(code="PredictionNotFound", message="The specified prediction does not exist") class PredictionController: def __init__(self, session, background_tasks=None): self.session = session self.background_tasks = background_tasks def create_prediction( self, deployment_id: str, upload_file: Optional[bytes] = None, dataset: Optional[str] = None, ): """ POST a prediction file to seldon deployment.
# -*- coding: utf-8 -*- """Comparison controller.""" from datetime import datetime from projects import models, schemas from projects.controllers.experiments import ExperimentController from projects.controllers.utils import uuid_alpha from projects.exceptions import BadRequest, NotFound from projects.utils import now NOT_FOUND = NotFound( code="ComparisonNotFound", message="The specified comparison does not exist" ) class ComparisonController: def __init__(self, session): self.session = session self.experiment_controller = ExperimentController(session) def raise_if_comparison_does_not_exist(self, comparison_id: str): """ Raises an exception if the specified comparison does not exist. Parameters ---------- comparison_id : str Raises ------
# -*- coding: utf-8 -*- """Operator controller.""" from datetime import datetime from typing import Dict, List, Optional from projects import models, schemas from projects.controllers.tasks import TaskController from projects.controllers.utils import uuid_alpha from projects.exceptions import BadRequest, NotFound NOT_FOUND = NotFound("The specified operator does not exist") PARAMETERS_EXCEPTION_MSG = "The specified parameters are not valid" DEPENDENCIES_EXCEPTION_MSG = "The specified dependencies are not valid." class OperatorController: def __init__(self, session): self.session = session self.task_controller = TaskController(session) def raise_if_operator_does_not_exist(self, operator_id: str): """ Raises an exception if the specified operator does not exist. Parameters ---------- operator_id : str Raises ------ NotFound
def get_dataset( self, project_id: str, experiment_id: str, run_id: str, operator_id: str, page: Optional[int] = 1, page_size: Optional[int] = 10, accept: Optional[str] = None, ): """ Get dataset records from a run. Supports pagination. Parameters ---------- project_id : str experiment_id : str run_id : str The run_id. If `run_id=latest`, then returns datasets from the latest run_id. operator_id : str page : int The page number. First page is 1. page_size : int The page size. Default value is 10. accept : str Whether dataset should be returned as csv file. Default to None. Returns ------- list A list of dataset records. Raises ------ NotFound When any of project_id, experiment_id, run_id, or operator_id does not exist. """ if run_id == "latest": run_id = get_latest_run_id(experiment_id) name = self.get_dataset_name(operator_id, experiment_id) try: metadata = stat_dataset(name=name, operator_id=operator_id, run_id=run_id) except FileNotFoundError: raise NotFound( code="DatasetNotFound", message="The specified run does not contain dataset", ) dataset = load_dataset( name=name, run_id=run_id, operator_id=operator_id, page=page, page_size=page_size, ) if isinstance(dataset, pd.DataFrame): # Replaces NaN value by a text "NaN" so JSON encode doesn't fail dataset.replace(np.nan, "NaN", inplace=True, regex=True) data = dataset.to_dict(orient="split") total = metadata.get("total", len(dataset.index)) return {"columns": data["columns"], "data": data["data"], "total": total} return StreamingResponse( dataset, media_type="application/octet-stream", )
import time from projects import models, schemas from projects.controllers.tasks import TaskController from projects.controllers.utils import uuid_alpha from projects.exceptions import BadRequest, NotFound from projects.kubernetes.kube_config import load_kube_config from projects.agent.utils import list_resource_version from projects.kfp import KF_PIPELINES_NAMESPACE from projects.kfp.runs import get_latest_run_id from projects.utils import now from kubernetes import client from kubernetes.watch import Watch NOT_FOUND = NotFound(code="OperatorNotFound", message="The specified operator does not exist") INVALID_PARAMETERS = BadRequest( code="InvalidParameters", message="The specified parameters are not valid") INVALID_DEPENDENCIES = BadRequest( code="InvalidDependencies", message="The specified dependencies are not valid.") class OperatorController: def __init__(self, session): self.session = session self.task_controller = TaskController(session) def raise_if_operator_does_not_exist(self, operator_id: str): """ Raises an exception if the specified operator does not exist.
# -*- coding: utf-8 -*- """Experiments controller.""" import sys from datetime import datetime from typing import Optional from projects import models, schemas from projects.controllers.operators import OperatorController from projects.controllers.utils import uuid_alpha from projects.exceptions import BadRequest, NotFound from projects.utils import now NOT_FOUND = NotFound( code="ExperimentNotFound", message="The specified experiment does not exist" ) class ExperimentController: def __init__(self, session): self.session = session self.operator_controller = OperatorController(session) def raise_if_experiment_does_not_exist(self, experiment_id: str): """ Raises an exception if the specified experiment does not exist. Parameters ---------- experiment_id : str Raises