Beispiel #1
0
    def create_run(self, project_id: str, deployment_id: str):
        """
        Starts a new run in Kubeflow Pipelines.

        Parameters
        ----------
        project_id : str
        deployment_id : str

        Returns
        -------
        projects.schemas.run.Run

        Raises
        ------
        NotFound
            When any of project_id, or deployment_id does not exist.
        """
        deployment = self.session.query(models.Deployment).get(deployment_id)

        if deployment is None:
            raise NotFound("The specified deployment does not exist")

        # Removes operators that don't have a deployment_notebook (eg. Upload de Dados).
        # Then, fix dependencies in their children.
        operators = self.remove_non_deployable_operators(deployment.operators)

        try:
            run = kfp_runs.start_run(operators=operators,
                                     project_id=deployment.project_id,
                                     experiment_id=deployment.experiment_id,
                                     deployment_id=deployment_id,
                                     deployment_name=deployment.name)
        except ValueError as e:
            raise BadRequest(str(e))

        # Deploy monitoring tasks
        monitorings = self.monitoring_controller.list_monitorings(project_id=project_id,
                                                                  deployment_id=deployment_id).monitorings
        if monitorings:
            for monitoring in monitorings:
                self.background_tasks.add_task(
                    deploy_monitoring,
                    deployment_id=deployment_id,
                    experiment_id=deployment.experiment_id,
                    run_id=run["uuid"],
                    task_id=monitoring.task_id,
                    monitoring_id=monitoring.uuid
                )

        update_data = {"status": "Pending"}
        self.session.query(models.Operator) \
            .filter_by(deployment_id=deployment_id) \
            .update(update_data)
        self.session.commit()

        run["deploymentId"] = deployment_id
        return run
Beispiel #2
0
def undeploy_monitoring(monitoring_id):
    """
    Undeploy the service and trigger of a given monitoring_id.

    Parameters
    ----------
    monitoring_id : str

    Raises
    ------
    NotFound
        When monitoring resources do not exist.
    """
    load_kube_config()
    api = client.CustomObjectsApi()

    try:
        # Undeploy service
        service_name = f"service-{monitoring_id}"
        service_custom_object = api.get_namespaced_custom_object(
            group="serving.knative.dev",
            version="v1",
            namespace=KF_PIPELINES_NAMESPACE,
            plural="services",
            name=service_name,
        )
        undeploy_pipeline(
            name=service_custom_object["metadata"]["name"],
            kind=service_custom_object["kind"],
            namespace=service_custom_object["metadata"]["namespace"],
        )

        # Undeploy trigger
        trigger_name = f"trigger-{monitoring_id}"
        trigger_custom_object = api.get_namespaced_custom_object(
            group="eventing.knative.dev",
            version="v1",
            namespace=KF_PIPELINES_NAMESPACE,
            plural="triggers",
            name=trigger_name,
        )
        undeploy_pipeline(
            name=trigger_custom_object["metadata"]["name"],
            kind=trigger_custom_object["kind"],
            namespace=trigger_custom_object["metadata"]["namespace"],
        )
    except ApiException:
        raise NotFound(
            code="MonitoringNotFound",
            message="The specified monitoring does not exist.",
        )
Beispiel #3
0
    def deploy_run(self, deployment):
        """
        Starts a new run in Kubeflow Pipelines.

        Parameters
        ----------
        deployment : projects.models.deployment.Deployment

        Returns
        -------
        projects.schemas.run.Run

        Raises
        ------
        NotFound
            When any of project_id, or deployment_id does not exist.
        """
        if deployment is None:
            raise NotFound(
                code="DeploymentNotFound",
                message="The specified deployment does not exist",
            )

        # Removes operators that don't have a deployment_notebook (eg. Upload de Dados).
        # Then, fix dependencies in their children.
        operators = self.remove_non_deployable_operators(deployment.operators)

        try:
            run = kfp_runs.start_run(
                operators=operators,
                project_id=deployment.project_id,
                experiment_id=deployment.experiment_id,
                deployment_id=deployment.uuid,
                deployment_name=deployment.name,
            )
        except ValueError:
            raise BadRequest(
                code="MissingRequiredOperatorId",
                message=f"Necessary at least one operator.",
            )

        # Remove the object from the operator session in order not to update the database,
        # Just need to remove the dependencies for the runs.
        for operator in deployment.operators:
            self.session.expunge(operator)

        return schemas.Run.from_orm(run)
Beispiel #4
0
    def terminate_run(self, deployment_id):
        """
        Terminates a run in Kubeflow Pipelines.

        Parameters
        ----------
        deployment_id : str

        Returns
        -------
        projects.schemas.message.Message

        Raises
        ------
        NotFound
            When deployment run does not exist.
        """
        load_kube_config()
        api = client.CustomObjectsApi()
        custom_objects = api.list_namespaced_custom_object(
            "machinelearning.seldon.io",
            "v1",
            KF_PIPELINES_NAMESPACE,
            "seldondeployments",
        )
        deployments_objects = custom_objects["items"]

        if deployments_objects:
            for deployment in deployments_objects:
                if deployment["metadata"]["name"] == deployment_id:
                    undeploy_pipeline(
                        name=deployment["metadata"]["name"],
                        kind=deployment["kind"],
                        namespace=deployment["metadata"]["namespace"],
                    )

        deployment_run = get_deployment_runs(deployment_id)

        if not deployment_run:
            raise NotFound(code="RunNotFound",
                           message="The specified run does not exist.")

        kfp_client().runs.delete_run(deployment_run["runId"])

        return schemas.Message(message="Deployment deleted")
Beispiel #5
0
    def raise_if_experiment_does_not_exist(self, experiment_id: str):
        """
        Raises an exception if the specified experiment does not exist.

        Parameters
        ----------
        experiment_id : str

        Raises
        ------
        NotFound
        """
        exists = self.session.query(models.Experiment.uuid) \
            .filter_by(uuid=experiment_id) \
            .scalar() is not None

        if not exists:
            raise NotFound("The specified experiment does not exist")
Beispiel #6
0
    def get_dataset_name(self, operator_id, experiment_id):
        """
        Get operator's dataset name.

        Parameters
        ----------
        operator_id : str
        experiment_id: str

        Returns
        -------
        str
            The dataset name.

        Raises
        ------
        NotFound
            When a run does not have a dataset.
        """
        operator = self.session.query(models.Operator).get(operator_id)
        dataset_name = operator.parameters.get("dataset")

        if dataset_name is None:
            operators = (
                self.session.query(models.Operator)
                .filter_by(experiment_id=experiment_id)
                .filter(models.Operator.uuid != operator_id)
                .all()
            )

            for operator in operators:
                dataset_name = operator.parameters.get("dataset")
                if dataset_name:
                    break

            if dataset_name is None:
                raise NotFound(
                    code="DatasetNotFound", message="No dataset assigned to the run"
                )

        return dataset_name
Beispiel #7
0
    def raise_if_monitoring_does_not_exist(self, monitoring_id: str):
        """
        Raises an exception if the specified monitoring does not exist.

        Parameters
        ----------
        monitoring_id : str

        Raises
        ------
        NotFound
        """
        exists = (self.session.query(
            models.Monitoring.uuid).filter_by(uuid=monitoring_id).scalar()
                  is not None)

        if not exists:
            raise NotFound(
                code="MonitoringNotFound",
                message="The specified monitoring does not exist",
            )
Beispiel #8
0
    def create_run(self, project_id: str, experiment_id: str):
        """
        Starts a new run in Kubeflow Pipelines.

        Parameters
        ----------
        project_id : str
        experiment_id : str

        Returns
        -------
        dict
            The run attributes.

        Raises
        ------
        NotFound
            When experiment_id does not exist.
        """
        experiment = self.session.query(models.Experiment).get(experiment_id)

        if experiment is None:
            raise NotFound(
                code="ExperimentNotFound",
                message="The specified experiment does not exist",
            )

        run = kfp_runs.start_run(
            project_id=project_id,
            experiment_id=experiment_id,
            operators=experiment.operators,
        )
        run["experimentId"] = experiment_id

        update_data = {"status": "Pending", "status_message": None}
        self.session.query(models.Operator).filter_by(
            experiment_id=experiment_id).update(update_data)
        self.session.commit()

        return schemas.Run.from_orm(run)
Beispiel #9
0
    def list_parameters(self, task_id: str):
        """
        Lists all parameters from the experiment notebook of a task.

        Parameters
        ----------
        task_id : str

        Returns
        -------
        list
            A list of all parameters.

        Raises
        ------
        NotFound
            When task_id does not exist.
        """
        task = self.session.query(models.Task).get(task_id)
        if task is None:
            raise NotFound("The specified task does not exist")

        return task.parameters
Beispiel #10
0
    def list_metrics(self, project_id: str, experiment_id: str, run_id: str,
                     operator_id: str):
        """
        Lists all metrics from object storage.

        Parameters
        ----------
        project_id : str
        experiment_id : str
        run_id : str
            The run_id. If `run_id=latest`, then returns metrics from the latest run_id.
        operator_id : str

        Returns
        -------
        list
            A list of metrics.
        """
        try:
            return platiagro.list_metrics(experiment_id=experiment_id,
                                          operator_id=operator_id,
                                          run_id=run_id)
        except FileNotFoundError as e:
            raise NotFound(str(e))
Beispiel #11
0
# -*- coding: utf-8 -*-
"""Monitorings controller."""
import warnings

from sqlalchemy import event

from projects import models, schemas
from projects.controllers.deployments.runs.runs import RunController
from projects.controllers.tasks import TaskController
from projects.controllers.utils import uuid_alpha
from projects.exceptions import BadRequest, NotFound
from projects.kfp.monitorings import deploy_monitoring, undeploy_monitoring

NOT_FOUND = NotFound(code="MonitoringNotFound",
                     message="The specified monitoring does not exist")


class MonitoringController:
    def __init__(self, session, background_tasks=None):
        self.session = session
        self.background_tasks = background_tasks
        self.run_controller = RunController(session)
        self.task_controller = TaskController(session)

    @staticmethod
    @event.listens_for(models.Monitoring, "after_delete")
    def after_delete(_mapper, _connection, target):
        """
        Starts a pipeline that deletes K8s resources associated with target monitoring.
        Parameters
        ----------
Beispiel #12
0
    def create_template(self, template: schemas.TemplateCreate):
        """
        Creates a new template in our database.

        Parameters
        ----------
        template : projects.schemas.template.TemplateCreate

        Returns
        -------
        projects.schemas.template.Template

        Raises
        ------
        BadRequest
            When the project attributes are invalid.
        """
        if not isinstance(template.name, str):
            raise BadRequest("name is required")

        if template.experiment_id:

            exists = self.session.query(models.Experiment.uuid) \
                .filter_by(uuid=template.experiment_id) \
                .scalar() is not None

            if not exists:
                raise NotFound("The specified experiment does not exist")

            operators = self.session.query(models.Operator) \
                .filter_by(experiment_id=template.experiment_id) \
                .all()
        elif template.deployment_id:

            exists = self.session.query(models.Deployment.uuid) \
                .filter_by(uuid=template.deployment_id) \
                .scalar() is not None

            if not exists:
                raise NotFound("The specified deployment does not exist")

            operators = self.session.query(models.Operator) \
                .filter_by(deployment_id=template.deployment_id) \
                .all()
        else:
            raise BadRequest(
                "experimentId or deploymentId needed to create template.")

        stored_template = self.session.query(models.Template) \
            .filter_by(name=template.name) \
            .first()
        if stored_template:
            raise BadRequest("a template with that name already exists")

        # order operators by dependencies
        operators_ordered = []
        while len(operators) != len(operators_ordered):
            for operator in operators:
                self.order_operators_by_dependencies(operators_ordered,
                                                     operator)

        # JSON array order of elements are preserved, so there is no need to save positions
        tasks = []
        for uuid in operators_ordered:
            operator = next((op for op in operators if op.uuid == uuid), None)
            task = {
                "uuid": operator.uuid,
                "task_id": operator.task_id,
                "dependencies": operator.dependencies,
                "position_x": operator.position_x,
                "position_y": operator.position_y,
            }
            tasks.append(task)

        template = models.Template(uuid=uuid_alpha(),
                                   name=template.name,
                                   tasks=tasks)
        self.session.add(template)
        self.session.commit()
        self.session.refresh(template)

        return schemas.Template.from_orm(template)
Beispiel #13
0
# -*- coding: utf-8 -*-
"""Templates controller."""
import re
from datetime import datetime

from projects import models, schemas
from projects.controllers.utils import uuid_alpha
from projects.exceptions import BadRequest, NotFound

NOT_FOUND = NotFound("The specified template does not exist")


class TemplateController:
    def __init__(self, session):
        self.session = session

    def raise_if_template_does_not_exist(self, template_id: str):
        """
        Raises an exception if the specified template does not exist.

        Parameters
        ----------
        template_id :str

        Raises
        ------
        NotFound
        """
        exists = self.session.query(models.Template.uuid) \
            .filter_by(uuid=template_id) \
            .scalar() is not None
Beispiel #14
0
# -*- coding: utf-8 -*-
"""Projects controller."""
from datetime import datetime
from os.path import join
from typing import Optional

from sqlalchemy import asc, desc, func

from projects import models, schemas
from projects.controllers.experiments import ExperimentController
from projects.controllers.utils import uuid_alpha
from projects.exceptions import BadRequest, NotFound
from projects.utils import now
NOT_FOUND = NotFound(code="ProjectNotFound",
                     message="The specified project does not exist")


class ProjectController:
    def __init__(self, session, kubeflow_userid=None):
        self.session = session
        self.experiment_controller = ExperimentController(session)
        self.kubeflow_userid = kubeflow_userid

    def raise_if_project_does_not_exist(self, project_id: str):
        """
        Raises an exception if the specified project does not exist.

        Parameters
        ----------
        project_id : str
Beispiel #15
0
# -*- coding: utf-8 -*-
"""Experiments Runs controller."""
from kfp_server_api.rest import ApiException

from projects import models, schemas
from projects.exceptions import NotFound
from projects.kfp import runs as kfp_runs

NOT_FOUND = NotFound("The specified run does not exist")


class RunController:
    def __init__(self, session):
        self.session = session

    def raise_if_run_does_not_exist(self, run_id: str, experiment_id: str):
        """
        Raises an exception if the specified run does not exist.

        Parameters
        ----------
        run_id : str
        experiment_id : str

        Raises
        ------
        NotFound
        """
        try:
            kfp_runs.get_run(experiment_id=experiment_id, run_id=run_id)
        except (ApiException, ValueError):
Beispiel #16
0
# -*- coding: utf-8 -*-
"""Projects controller."""
from datetime import datetime
from os.path import join
from typing import Optional

from sqlalchemy import asc, desc, func

from projects import models, schemas
from projects.controllers.experiments import ExperimentController
from projects.controllers.utils import uuid_alpha
from projects.exceptions import BadRequest, NotFound
from projects.object_storage import remove_objects

NOT_FOUND = NotFound("The specified project does not exist")


class ProjectController:
    def __init__(self, session):
        self.session = session
        self.experiment_controller = ExperimentController(session)

    def raise_if_project_does_not_exist(self, project_id: str):
        """
        Raises an exception if the specified project does not exist.

        Parameters
        ----------
        project_id : str

        Raises
Beispiel #17
0
# -*- coding: utf-8 -*-
"""Comparison controller."""
from datetime import datetime

from projects import models, schemas
from projects.controllers.experiments import ExperimentController
from projects.controllers.utils import uuid_alpha
from projects.exceptions import BadRequest, NotFound

NOT_FOUND = NotFound("The specified comparison does not exist")


class ComparisonController:
    def __init__(self, session):
        self.session = session
        self.experiment_controller = ExperimentController(session)

    def raise_if_comparison_does_not_exist(self, comparison_id: str):
        """
        Raises an exception if the specified comparison does not exist.

        Parameters
        ----------
        comparison_id : str

        Raises
        ------
        NotFound
        """
        exists = self.session.query(models.Comparison.uuid) \
            .filter_by(uuid=comparison_id) \
Beispiel #18
0
# -*- coding: utf-8 -*-
"""Experiments controller."""
import sys
from datetime import datetime
from typing import Optional

from projects import models, schemas
from projects.controllers.operators import OperatorController
from projects.controllers.utils import uuid_alpha
from projects.exceptions import BadRequest, NotFound

NOT_FOUND = NotFound("The specified experiment does not exist")


class ExperimentController:
    def __init__(self, session):
        self.session = session
        self.operator_controller = OperatorController(session)

    def raise_if_experiment_does_not_exist(self, experiment_id: str):
        """
        Raises an exception if the specified experiment does not exist.

        Parameters
        ----------
        experiment_id : str

        Raises
        ------
        NotFound
        """
Beispiel #19
0
# -*- coding: utf-8 -*-
"""Deployments controller."""
import sys
from datetime import datetime

from projects import models, schemas
from projects.controllers.deployments.runs import RunController
from projects.controllers.experiments import ExperimentController
from projects.controllers.operators import OperatorController
from projects.controllers.templates import TemplateController
from projects.controllers.utils import uuid_alpha
from projects.exceptions import BadRequest, NotFound
from projects.kfp.deployments import get_deployment_runs, list_deployments_runs
from projects.kfp.monitorings import undeploy_monitoring

NOT_FOUND = NotFound("The specified deployment does not exist")


class DeploymentController:
    def __init__(self, session, background_tasks=None):
        self.session = session
        self.experiment_controller = ExperimentController(session)
        self.operator_controller = OperatorController(session)
        self.run_controller = RunController(session)
        self.template_controller = TemplateController(session)
        self.background_tasks = background_tasks

    def raise_if_deployment_does_not_exist(self, deployment_id: str):
        """
        Raises an exception if the specified deployment does not exist.
Beispiel #20
0
# -*- coding: utf-8 -*-
"""Templates controller."""
import re
from datetime import datetime

from projects import models, schemas
from projects.controllers.utils import uuid_alpha
from projects.exceptions import BadRequest, NotFound
from projects.utils import now

NOT_FOUND = NotFound(code="TemplateNotFound",
                     message="The specified template does not exist")


class TemplateController:
    def __init__(self, session, kubeflow_userid=None):
        self.session = session
        self.kubeflow_userid = kubeflow_userid

    def raise_if_template_does_not_exist(self, template_id: str):
        """
        Raises an exception if the specified template does not exist.

        Parameters
        ----------
        template_id :str

        Raises
        ------
        NotFound
        """
Beispiel #21
0
    f"platiagro/platiagro-experiment-image:{__version__}",
)
TASK_DEFAULT_CPU_LIMIT = os.getenv("TASK_DEFAULT_CPU_LIMIT", "2000m")
TASK_DEFAULT_CPU_REQUEST = os.getenv("TASK_DEFAULT_CPU_REQUEST", "100m")
TASK_DEFAULT_MEMORY_LIMIT = os.getenv("TASK_DEFAULT_MEMORY_LIMIT", "10Gi")
TASK_DEFAULT_MEMORY_REQUEST = os.getenv("TASK_DEFAULT_MEMORY_REQUEST", "2Gi")
TASK_DEFAULT_READINESS_INITIAL_DELAY_SECONDS = int(
    os.getenv(
        "TASK_DEFAULT_READINESS_INITIAL_DELAY_SECONDS",
        "60",
    ))

EMAIL_MESSAGE_TEMPLATE = pkgutil.get_data("projects",
                                          "config/email-template.html")

NOT_FOUND = NotFound(code="TaskNotFound",
                     message="The specified task does not exist")


class TaskController:
    def __init__(self, session, background_tasks=None):
        self.session = session
        self.background_tasks = background_tasks

    def raise_if_task_does_not_exist(self, task_id: str):
        """
        Raises an exception if the specified task does not exist.

        Parameters
        ----------
        task_id : str
Beispiel #22
0
"""Deployments controller."""
import sys
import warnings
from datetime import datetime

from sqlalchemy import event

from projects import models, schemas
from projects.controllers.deployments.runs import RunController
from projects.controllers.templates import TemplateController
from projects.controllers.utils import uuid_alpha
from projects.controllers.tasks import TaskController
from projects.exceptions import BadRequest, NotFound
from projects.utils import now

NOT_FOUND = NotFound(code="DeploymentNotFound",
                     message="The specified deployment does not exist")

# Distance on the X axis from the leftmost operator
DATASET_OPERATOR_DISTANCE = 300
FONTE_DE_DADOS = "Fonte de dados"


class DeploymentController:
    def __init__(self, session, background_tasks=None, kubeflow_userid=None):
        self.session = session
        self.run_controller = RunController(session)
        self.template_controller = TemplateController(
            session, kubeflow_userid=kubeflow_userid)
        self.task_controller = TaskController(session, background_tasks)
        self.background_tasks = background_tasks
Beispiel #23
0
# -*- coding: utf-8 -*-
"""Deployments Runs controller."""
from kubernetes import client
from kubernetes.client.rest import ApiException

from projects import models, schemas
from projects.exceptions import BadRequest, NotFound
from projects.kfp import KF_PIPELINES_NAMESPACE, kfp_client
from projects.kfp import runs as kfp_runs
from projects.kfp.deployments import get_deployment_runs
from projects.kfp.pipeline import undeploy_pipeline
from projects.kubernetes.kube_config import load_kube_config
from projects.kubernetes.seldon import get_seldon_deployment_url

NOT_FOUND = NotFound(code="RunNotFound",
                     message="The specified run does not exist")


class RunController:
    def __init__(self, session):
        self.session = session

    def raise_if_run_does_not_exist(self, run_id: str, deployment_id: str):
        """
        Raises an exception if the specified run does not exist.

        Parameters
        ----------
        run_id : str
        deployment_id : str
Beispiel #24
0
# -*- coding: utf-8 -*-
"""Monitorings controller."""
from projects import models, schemas
from projects.controllers.tasks import TaskController
from projects.controllers.utils import uuid_alpha
from projects.exceptions import NotFound

NOT_FOUND = NotFound("The specified monitoring does not exist")


class MonitoringController:
    def __init__(self, session):
        self.session = session
        self.task_controller = TaskController(session)

    def list_monitorings(self, project_id: str, deployment_id: str):
        """
        Lists all monitorings under a deployment.

        Parameters
        ----------
        project_id : str
        deployment_id : str

        Returns
        -------
        projects.schemas.monitoring.MonitoringList
        """
        monitorings = self.session.query(models.Monitoring) \
            .filter_by(deployment_id=deployment_id) \
            .order_by(models.Monitoring.created_at.asc()) \
Beispiel #25
0
import logging
from typing import Optional

import requests
from platiagro import load_dataset

from projects import models, schemas
from projects.controllers.utils import (
    parse_dataframe_to_seldon_request,
    parse_file_buffer_to_seldon_request,
    uuid_alpha,
)
from projects.exceptions import BadRequest, NotFound
from projects.kubernetes.seldon import get_seldon_deployment_url

NOT_FOUND = NotFound(code="PredictionNotFound",
                     message="The specified prediction does not exist")


class PredictionController:
    def __init__(self, session, background_tasks=None):
        self.session = session
        self.background_tasks = background_tasks

    def create_prediction(
        self,
        deployment_id: str,
        upload_file: Optional[bytes] = None,
        dataset: Optional[str] = None,
    ):
        """
        POST a prediction file to seldon deployment.
Beispiel #26
0
# -*- coding: utf-8 -*-
"""Comparison controller."""
from datetime import datetime

from projects import models, schemas
from projects.controllers.experiments import ExperimentController
from projects.controllers.utils import uuid_alpha
from projects.exceptions import BadRequest, NotFound
from projects.utils import now


NOT_FOUND = NotFound(
    code="ComparisonNotFound", message="The specified comparison does not exist"
)


class ComparisonController:
    def __init__(self, session):
        self.session = session
        self.experiment_controller = ExperimentController(session)

    def raise_if_comparison_does_not_exist(self, comparison_id: str):
        """
        Raises an exception if the specified comparison does not exist.

        Parameters
        ----------
        comparison_id : str

        Raises
        ------
Beispiel #27
0
# -*- coding: utf-8 -*-
"""Operator controller."""
from datetime import datetime
from typing import Dict, List, Optional

from projects import models, schemas
from projects.controllers.tasks import TaskController
from projects.controllers.utils import uuid_alpha
from projects.exceptions import BadRequest, NotFound

NOT_FOUND = NotFound("The specified operator does not exist")
PARAMETERS_EXCEPTION_MSG = "The specified parameters are not valid"
DEPENDENCIES_EXCEPTION_MSG = "The specified dependencies are not valid."


class OperatorController:
    def __init__(self, session):
        self.session = session
        self.task_controller = TaskController(session)

    def raise_if_operator_does_not_exist(self, operator_id: str):
        """
        Raises an exception if the specified operator does not exist.

        Parameters
        ----------
        operator_id : str

        Raises
        ------
        NotFound
Beispiel #28
0
    def get_dataset(
        self,
        project_id: str,
        experiment_id: str,
        run_id: str,
        operator_id: str,
        page: Optional[int] = 1,
        page_size: Optional[int] = 10,
        accept: Optional[str] = None,
    ):
        """
        Get dataset records from a run. Supports pagination.

        Parameters
        ----------
        project_id : str
        experiment_id : str
        run_id : str
            The run_id. If `run_id=latest`, then returns datasets from the latest run_id.
        operator_id : str
        page : int
            The page number. First page is 1.
        page_size : int
            The page size. Default value is 10.
        accept : str
            Whether dataset should be returned as csv file. Default to None.

        Returns
        -------
        list
            A list of dataset records.

        Raises
        ------
        NotFound
            When any of project_id, experiment_id, run_id, or operator_id does not exist.
        """
        if run_id == "latest":
            run_id = get_latest_run_id(experiment_id)

        name = self.get_dataset_name(operator_id, experiment_id)

        try:
            metadata = stat_dataset(name=name, operator_id=operator_id, run_id=run_id)
        except FileNotFoundError:
            raise NotFound(
                code="DatasetNotFound",
                message="The specified run does not contain dataset",
            )

        dataset = load_dataset(
            name=name,
            run_id=run_id,
            operator_id=operator_id,
            page=page,
            page_size=page_size,
        )
        if isinstance(dataset, pd.DataFrame):
            # Replaces NaN value by a text "NaN" so JSON encode doesn't fail
            dataset.replace(np.nan, "NaN", inplace=True, regex=True)
            data = dataset.to_dict(orient="split")
            total = metadata.get("total", len(dataset.index))
            return {"columns": data["columns"], "data": data["data"], "total": total}

        return StreamingResponse(
            dataset,
            media_type="application/octet-stream",
        )
Beispiel #29
0
import time

from projects import models, schemas
from projects.controllers.tasks import TaskController
from projects.controllers.utils import uuid_alpha
from projects.exceptions import BadRequest, NotFound
from projects.kubernetes.kube_config import load_kube_config
from projects.agent.utils import list_resource_version
from projects.kfp import KF_PIPELINES_NAMESPACE
from projects.kfp.runs import get_latest_run_id
from projects.utils import now

from kubernetes import client
from kubernetes.watch import Watch

NOT_FOUND = NotFound(code="OperatorNotFound",
                     message="The specified operator does not exist")
INVALID_PARAMETERS = BadRequest(
    code="InvalidParameters", message="The specified parameters are not valid")
INVALID_DEPENDENCIES = BadRequest(
    code="InvalidDependencies",
    message="The specified dependencies are not valid.")


class OperatorController:
    def __init__(self, session):
        self.session = session
        self.task_controller = TaskController(session)

    def raise_if_operator_does_not_exist(self, operator_id: str):
        """
        Raises an exception if the specified operator does not exist.
Beispiel #30
0
# -*- coding: utf-8 -*-
"""Experiments controller."""
import sys
from datetime import datetime
from typing import Optional

from projects import models, schemas
from projects.controllers.operators import OperatorController
from projects.controllers.utils import uuid_alpha
from projects.exceptions import BadRequest, NotFound
from projects.utils import now

NOT_FOUND = NotFound(
    code="ExperimentNotFound", message="The specified experiment does not exist"
)


class ExperimentController:
    def __init__(self, session):
        self.session = session
        self.operator_controller = OperatorController(session)

    def raise_if_experiment_does_not_exist(self, experiment_id: str):
        """
        Raises an exception if the specified experiment does not exist.

        Parameters
        ----------
        experiment_id : str

        Raises