def create_project(self, project: schemas.ProjectCreate):
"""
Creates a new project in our database.
Parameters
----------
project: projects.schemas.project.ProjectCreate
Returns
-------
project: projects.schemas.project.Project
Raises
------
BadRequest
When the project attributes are invalid.
"""
store_project = (self.session.query(models.Project).filter_by(
name=project.name).filter_by(tenant=self.kubeflow_userid).first())
if store_project:
raise BadRequest(
code="ProjectNameExists",
message="a project with that name already exists",
)
project = models.Project(uuid=uuid_alpha(),
name=project.name,
description=project.description,
tenant=self.kubeflow_userid,
created_at=now(),
updated_at=now())
self.session.add(project)
self.session.flush()
experiment = schemas.ExperimentCreate(name="Experimento 1")
self.experiment_controller.create_experiment(experiment=experiment,
project_id=project.uuid)
self.session.commit()
self.session.refresh(project)
return schemas.Project.from_orm(project)
def create_comparison(self, project_id: str):
"""
Creates a new comparison in our database.
Parameters
----------
project_id : str
Returns
-------
projects.schemas.comparison.Comparison
"""
comparison = models.Comparison(uuid=uuid_alpha(),
project_id=project_id,
created_at=now(),
updated_at=now())
self.session.add(comparison)
self.session.commit()
self.session.refresh(comparison)
return schemas.Comparison.from_orm(comparison)
def create_project(name=None, **kwargs):
"""Creates a new project in our database.
Args:
name (str): the project name.
Returns:
The project info.
"""
if not isinstance(name, str):
raise BadRequest("name is required")
check_project_name = db_session.query(Project).filter_by(name=name).first()
if check_project_name:
raise BadRequest("a project with that name already exists")
project = Project(uuid=uuid_alpha(),
name=name,
description=kwargs.get("description"))
db_session.add(project)
db_session.commit()
create_experiment(name="Experimento 1", project_id=project.uuid)
return project.as_dict()
def create_project(self, project: schemas.ProjectCreate):
"""
Creates a new project in our database.
Parameters
----------
project: projects.schemas.project.ProjectCreate
Returns
-------
project: projects.schemas.project.Project
Raises
------
BadRequest
When the project attributes are invalid.
"""
if not isinstance(project.name, str):
raise BadRequest("name is required")
store_project = self.session.query(models.Project) \
.filter_by(name=project.name) \
.first()
if store_project:
raise BadRequest("a project with that name already exists")
project = models.Project(uuid=uuid_alpha(), name=project.name, description=project.description)
self.session.add(project)
self.session.flush()
experiment = schemas.ExperimentCreate(name="Experimento 1")
self.experiment_controller.create_experiment(experiment=experiment, project_id=project.uuid)
self.session.commit()
self.session.refresh(project)
return schemas.Project.from_orm(project)
# -*- coding: utf-8 -*-
from io import BytesIO
from json import dumps
from unittest import TestCase
from minio.error import BucketAlreadyOwnedByYou
from platiagro import CATEGORICAL, DATETIME, NUMERICAL
from projects.api.main import app
from projects.controllers.utils import uuid_alpha
from projects.database import engine
from projects.object_storage import BUCKET_NAME, MINIO_CLIENT
PROJECT_ID = str(uuid_alpha())
EXPERIMENT_ID = str(uuid_alpha())
COMPONENT_ID = str(uuid_alpha())
NAME = "foo"
DESCRIPTION = "long foo"
DATASET = "mock.csv"
TARGET = "col4"
POSITION = 0
PARAMETERS = {"dataset": DATASET}
COMMANDS = ["CMD"]
COMMANDS_JSON = dumps(COMMANDS)
TAGS = ["PREDICTOR"]
TAGS_JSON = dumps(TAGS)
PARAMETERS_JSON = dumps(PARAMETERS)
EXPERIMENT_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/components/{COMPONENT_ID}/Experiment.ipynb"
DEPLOYMENT_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/components/{COMPONENT_ID}/Deployment.ipynb"
RUN_ID = str(uuid_alpha())
OPERATOR_ID = str(uuid_alpha())
def create_template(self, template: schemas.TemplateCreate):
"""
Creates a new template in our database.
Parameters
----------
template : projects.schemas.template.TemplateCreate
Returns
-------
projects.schemas.template.Template
Raises
------
BadRequest
When the project attributes are invalid.
"""
if not isinstance(template.name, str):
raise BadRequest("name is required")
if template.experiment_id:
exists = self.session.query(models.Experiment.uuid) \
.filter_by(uuid=template.experiment_id) \
.scalar() is not None
if not exists:
raise NotFound("The specified experiment does not exist")
operators = self.session.query(models.Operator) \
.filter_by(experiment_id=template.experiment_id) \
.all()
elif template.deployment_id:
exists = self.session.query(models.Deployment.uuid) \
.filter_by(uuid=template.deployment_id) \
.scalar() is not None
if not exists:
raise NotFound("The specified deployment does not exist")
operators = self.session.query(models.Operator) \
.filter_by(deployment_id=template.deployment_id) \
.all()
else:
raise BadRequest(
"experimentId or deploymentId needed to create template.")
stored_template = self.session.query(models.Template) \
.filter_by(name=template.name) \
.first()
if stored_template:
raise BadRequest("a template with that name already exists")
# order operators by dependencies
operators_ordered = []
while len(operators) != len(operators_ordered):
for operator in operators:
self.order_operators_by_dependencies(operators_ordered,
operator)
# JSON array order of elements are preserved, so there is no need to save positions
tasks = []
for uuid in operators_ordered:
operator = next((op for op in operators if op.uuid == uuid), None)
task = {
"uuid": operator.uuid,
"task_id": operator.task_id,
"dependencies": operator.dependencies,
"position_x": operator.position_x,
"position_y": operator.position_y,
}
tasks.append(task)
template = models.Template(uuid=uuid_alpha(),
name=template.name,
tasks=tasks)
self.session.add(template)
self.session.commit()
self.session.refresh(template)
return schemas.Template.from_orm(template)
# -*- coding: utf-8 -*-
from unittest import TestCase
from fastapi.testclient import TestClient
from projects.api.main import app
from projects.controllers.utils import uuid_alpha
from projects.database import engine
TEST_CLIENT = TestClient(app)
EXPERIMENT_ID = str(uuid_alpha())
PROJECT_ID = str(uuid_alpha())
COMPARISON_ID = str(uuid_alpha())
NAME = "foo"
POSITION = 0
CREATED_AT = "2000-01-01 00:00:00"
CREATED_AT_ISO = "2000-01-01T00:00:00"
UPDATED_AT = "2000-01-01 00:00:00"
UPDATED_AT_ISO = "2000-01-01T00:00:00"
class TestComparisons(TestCase):
def setUp(self):
self.maxDiff = None
conn = engine.connect()
text = (f"INSERT INTO projects (uuid, name, created_at, updated_at) "
f"VALUES (%s, %s, %s, %s)")
conn.execute(text, (
PROJECT_ID,
NAME,
# -*- coding: utf-8 -*-
from json import dumps
from unittest import TestCase
from projects.api.main import app
from projects.controllers.utils import uuid_alpha
from projects.database import engine
from projects.object_storage import BUCKET_NAME
TEMPLATE_ID = str(uuid_alpha())
NAME = "foo"
TASK_ID = str(uuid_alpha())
PROJECT_ID = str(uuid_alpha())
EXPERIMENT_ID = str(uuid_alpha())
OPERATOR_ID = str(uuid_alpha())
POSITION = 0
PARAMETERS = {"coef": 0.1}
OPERATORS = [{"taskId": TASK_ID}]
DESCRIPTION = "long foo"
IMAGE = "platiagro/platiagro-notebook-image-test:0.1.0"
COMMANDS = ["CMD"]
COMMANDS_JSON = dumps(COMMANDS)
ARGUMENTS = ["ARG"]
ARGUMENTS_JSON = dumps(ARGUMENTS)
TAGS = ["PREDICTOR"]
TAGS_JSON = dumps(TAGS)
TASKS_JSON = dumps([TASK_ID])
PARAMETERS_JSON = dumps(PARAMETERS)
EXPERIMENT_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/tasks/{TASK_ID}/Experiment.ipynb"
DEPLOYMENT_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/tasks/{TASK_ID}/Deployment.ipynb"
CREATED_AT = "2000-01-01 00:00:00"
def create_template(self, template: schemas.TemplateCreate):
"""
Creates a new template in our database.
Parameters
----------
template : projects.schemas.template.TemplateCreate
Returns
-------
projects.schemas.template.Template
Raises
------
BadRequest
When the project attributes are invalid.
"""
if not isinstance(template.name, str):
raise BadRequest(code="MissingRequiredTemplateName",
message="name is required")
if template.experiment_id:
exists = (self.session.query(models.Experiment.uuid).filter_by(
uuid=template.experiment_id).scalar() is not None)
if not exists:
raise BadRequest(
code="InvalidExperimentId",
message="source experiment does not exist",
)
operators = (self.session.query(models.Operator).filter_by(
experiment_id=template.experiment_id).all())
elif template.deployment_id:
exists = (self.session.query(models.Deployment.uuid).filter_by(
uuid=template.deployment_id).scalar() is not None)
if not exists:
raise BadRequest(
code="InvalidDeploymentId",
message="source deployment does not exist",
)
operators = (self.session.query(models.Operator).filter_by(
deployment_id=template.deployment_id).all())
else:
raise BadRequest(
code="MissingRequiredExperimentIdOrDeploymentId",
message=
"experimentId or deploymentId needed to create template.",
)
stored_template = (self.session.query(models.Template).filter_by(
name=template.name).filter_by(tenant=self.kubeflow_userid).first())
if stored_template:
raise BadRequest(
code="TemplateNameExists",
message="a template with that name already exists",
)
# order operators by dependencies
operators_ordered = []
while len(operators) != len(operators_ordered):
for operator in operators:
self.order_operators_by_dependencies(operators_ordered,
operator)
# JSON array order of elements are preserved, so there is no need to save positions
tasks = []
for uuid in operators_ordered:
operator = next((op for op in operators if op.uuid == uuid), None)
task = {
"uuid": operator.uuid,
"task_id": operator.task_id,
"dependencies": operator.dependencies,
"position_x": operator.position_x,
"position_y": operator.position_y,
}
tasks.append(task)
template = models.Template(uuid=uuid_alpha(),
name=template.name,
tasks=tasks,
experiment_id=template.experiment_id,
deployment_id=template.deployment_id,
tenant=self.kubeflow_userid,
created_at=now(),
updated_at=now())
self.session.add(template)
self.session.commit()
self.session.refresh(template)
return schemas.Template.from_orm(template)
from os import environ, path, remove
from requests import get
from pytest import fixture
from papermill import execute_notebook
from .utils import creates_boston_metadata, creates_titanic_metadata, \
creates_mock_dataset, delete_mock_dataset
from projects.controllers.utils import uuid_alpha
EXPERIMENT_ID = str(uuid_alpha())
OPERATOR_ID = str(uuid_alpha())
RUN_ID = str(uuid_alpha())
BOSTON_DATASET = "boston_mock"
BOSTON_TARGET = "medv"
TITANIC_DATASET = "titanic_mock"
TITANIC_TARGET = "Fare"
@fixture(scope="module", autouse=True)
def setup(request):
# Set environment variables needed to run notebooks
environ["EXPERIMENT_ID"] = EXPERIMENT_ID
environ["OPERATOR_ID"] = OPERATOR_ID
environ["RUN_ID"] = RUN_ID
boston_content = \
get('https://raw.githubusercontent.com/platiagro/datasets/master/samples/boston.csv').content
# -*- coding: utf-8 -*-
from json import dumps
from unittest import TestCase
from projects.api.main import app
from projects.controllers.utils import uuid_alpha
from projects.database import engine
from projects.object_storage import BUCKET_NAME
OPERATOR_ID = str(uuid_alpha())
OPERATOR_ID_2 = str(uuid_alpha())
OPERATOR_ID_3 = str(uuid_alpha())
OPERATOR_ID_4 = str(uuid_alpha())
DEPENDENCY_ID = str(uuid_alpha())
DEPENDENCY_ID_2 = str(uuid_alpha())
NAME = "foo"
DESCRIPTION = "long foo"
PROJECT_ID = str(uuid_alpha())
EXPERIMENT_ID = str(uuid_alpha())
COMPONENT_ID = str(uuid_alpha())
PARAMETERS = {"coef": 0.1}
POSITION = 0
PARAMETERS = {}
COMMANDS = ["CMD"]
COMMANDS_JSON = dumps(COMMANDS)
TAGS = ["PREDICTOR"]
TAGS_JSON = dumps(TAGS)
PARAMETERS_JSON = dumps(PARAMETERS)
EXPERIMENT_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/components/{COMPONENT_ID}/Experiment.ipynb"
DEPLOYMENT_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/components/{COMPONENT_ID}/Deployment.ipynb"
CREATED_AT = "2000-01-01 00:00:00"
def create_experiment(name=None, project_id=None, copy_from=None):
"""Creates a new experiment in our database and adjusts the position of others.
The new experiment is added to the end of the experiment list.
Args:
name (str): the experiment name.
project_id (str): the project uuid.
copy_from (str): the copy_from uuid.
Returns:
The experiment info.
"""
raise_if_project_does_not_exist(project_id)
if not isinstance(name, str):
raise BadRequest("name is required")
check_experiment_name = db_session.query(Experiment)\
.filter(Experiment.project_id == project_id)\
.filter(Experiment.name == name)\
.first()
if check_experiment_name:
raise BadRequest("an experiment with that name already exists")
experiment = Experiment(uuid=uuid_alpha(), name=name, project_id=project_id)
db_session.add(experiment)
db_session.commit()
if copy_from:
try:
experiment_find = find_by_experiment_id(experiment_id=copy_from)
source_operators = {}
for source_operator in experiment_find['operators']:
source_dependencies = source_operator.dependencies
kwargs = {
"task_id": source_operator.task_id,
"parameters": source_operator.parameters,
"dependencies": [],
"position_x": source_operator.position_x,
"position_y": source_operator.position_y
}
operator = create_operator(project_id, experiment.uuid, **kwargs)
source_operators[source_operator.uuid] = {
"copy_uuid": operator["uuid"],
"dependencies": source_dependencies
}
# update dependencies on new operators
for _, value in source_operators.items():
dependencies = [source_operators[d]['copy_uuid'] for d in value['dependencies']]
update_operator(value['copy_uuid'], project_id, experiment.uuid, dependencies=dependencies)
except NotFound:
delete_experiment(experiment.uuid, project_id)
raise BadRequest('Source experiment does not exist')
fix_positions(project_id=project_id,
experiment_id=experiment.uuid,
new_position=sys.maxsize) # will add to end of list
return experiment.as_dict()
from json import dumps, loads
from unittest import TestCase
import requests
from fastapi.testclient import TestClient
from minio.error import BucketAlreadyOwnedByYou
from projects.api.main import app
from projects.controllers.utils import uuid_alpha
from projects.database import engine
from projects.jupyter import COOKIES, HEADERS, JUPYTER_ENDPOINT
from projects.object_storage import BUCKET_NAME, MINIO_CLIENT
TEST_CLIENT = TestClient(app)
TASK_ID = str(uuid_alpha())
NAME = "name foo"
DESCRIPTION = "long foo"
IMAGE = "platiagro/platiagro-experiment-image:0.2.0"
COMMANDS = ["CMD"]
COMMANDS_JSON = dumps(COMMANDS)
ARGUMENTS = ["ARG"]
ARGUMENTS_JSON = dumps(ARGUMENTS)
TAGS = ["PREDICTOR"]
TAGS_JSON = dumps(TAGS)
EXPERIMENT_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/tasks/{TASK_ID}/Experiment.ipynb"
DEPLOYMENT_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/tasks/{TASK_ID}/Deployment.ipynb"
IS_DEFAULT = False
PARAMETERS = [{"default": True, "name": "shuffle", "type": "boolean"}]
CREATED_AT = "2000-01-01 00:00:00"
CREATED_AT_ISO = "2000-01-01T00:00:00"
def copy_experiment(self, experiment: schemas.ExperimentCreate,
project_id: str):
"""
Makes a copy of an experiment in our database.
Parameters
----------
task: projects.schemas.experiment.ExperimentCreate
project_id: str
Returns
-------
projects.schemas.experiment.Experiment
Raises
------
BadRequest
When copy_from does not exist.
"""
stored_experiment = self.session.query(models.Experiment).get(
experiment.copy_from)
if stored_experiment is None:
raise BadRequest("source experiment does not exist")
experiment = models.Experiment(uuid=uuid_alpha(),
name=experiment.name,
project_id=project_id)
self.session.add(experiment)
self.session.flush()
# Creates a dict to map source operator_id to its copy operator_id.
# This map will be used to build the dependencies using new operator_ids
copies_map = {}
for stored_operator in stored_experiment.operators:
operator = schemas.OperatorCreate(
task_id=stored_operator.task_id,
experiment_id=experiment.uuid,
parameters=stored_operator.parameters,
position_x=stored_operator.position_x,
position_y=stored_operator.position_y,
)
operator = self.operator_controller.create_operator(
operator=operator,
project_id=project_id,
experiment_id=experiment.uuid)
copies_map[stored_operator.uuid] = {
"copy_uuid": operator.uuid,
"dependencies": stored_operator.dependencies,
}
# sets dependencies on new operators
for _, value in copies_map.items():
operator = schemas.OperatorUpdate(dependencies=[
copies_map[d]["copy_uuid"] for d in value["dependencies"]
], )
self.operator_controller.update_operator(
project_id=project_id,
experiment_id=experiment.uuid,
operator_id=value["copy_uuid"],
operator=operator)
return experiment
def update_experiment(uuid, project_id, **kwargs):
"""Updates an experiment in our database and adjusts the position of others.
Args:
uuid (str): the experiment uuid to look for in our database.
project_id (str): the project uuid.
**kwargs: arbitrary keyword arguments.
Returns:
The experiment info.
"""
raise_if_project_does_not_exist(project_id)
experiment = Experiment.query.get(uuid)
if experiment is None:
raise NOTFOUND
if "name" in kwargs:
name = kwargs["name"]
if name != experiment.name:
check_experiment_name = db_session.query(Experiment)\
.filter(Experiment.project_id == project_id)\
.filter(Experiment.name == name)\
.first()
if check_experiment_name:
raise BadRequest("an experiment with that name already exists")
# updates operators
if "template_id" in kwargs:
template_id = kwargs["template_id"]
del kwargs["template_id"]
template = Template.query.get(template_id)
if template is None:
raise BadRequest("The specified template does not exist")
# remove operators
Operator.query.filter(Operator.experiment_id == uuid).delete()
# save the last operator id created to create dependency on next operator
last_operator_id = None
for task_id in template.tasks:
operator_id = uuid_alpha()
dependencies = []
if last_operator_id is not None:
dependencies = [last_operator_id]
objects = [
Operator(uuid=operator_id,
experiment_id=uuid,
task_id=task_id,
dependencies=dependencies)
]
db_session.bulk_save_objects(objects)
last_operator_id = operator_id
data = {"updated_at": datetime.utcnow()}
data.update(kwargs)
try:
db_session.query(Experiment).filter_by(uuid=uuid).update(data)
db_session.commit()
except (InvalidRequestError, ProgrammingError) as e:
raise BadRequest(str(e))
fix_positions(project_id=experiment.project_id,
experiment_id=experiment.uuid,
new_position=experiment.position)
return experiment.as_dict()
from unittest import TestCase
from fastapi.testclient import TestClient
import matplotlib.pyplot as plt
import numpy as np
import platiagro
from projects.api.main import app
from projects.controllers.utils import uuid_alpha
from projects.database import engine
from projects.object_storage import BUCKET_NAME, MINIO_CLIENT
TEST_CLIENT = TestClient(app)
PROJECT_ID = str(uuid_alpha())
EXPERIMENT_ID = str(uuid_alpha())
TASK_ID = str(uuid_alpha())
OPERATOR_ID = str(uuid_alpha())
RUN_ID = str(uuid_alpha())
NAME = "foo"
DESCRIPTION = "long foo"
TARGET = "col4"
POSITION = 0
PARAMETERS = {}
COMMANDS = ["CMD"]
COMMANDS_JSON = dumps(COMMANDS)
ARGUMENTS = ["ARG"]
ARGUMENTS_JSON = dumps(ARGUMENTS)
IMAGE = "platiagro/platiagro-experiment-image:0.2.0"
TAGS = ["PREDICTOR"]
def create_deployment(self, deployment: schemas.DeploymentCreate,
project_id: str):
"""
Creates new deployments in our database and adjusts the position of others.
Parameters
----------
deployment : DeploymentCreate
project_id : str
Returns
-------
projects.schemas.deployment.Deployment
Raises
------
NotFound
When project_id does not exist.
BadRequest
When any experiment does not exist.
"""
# ^ is xor operator. it's equivalent to (a and not b) or (not a and b)
if not bool(deployment.experiments) ^ bool(deployment.template_id):
raise BadRequest("either experiments or templateId is required")
if deployment.template_id:
return self.create_deployment_from_template(
template_id=deployment.template_id, project_id=project_id)
experiments_dict = {
e.uuid: e
for e in self.session.query(models.Experiment).filter_by(
project_id=project_id)
}
for experiment_id in deployment.experiments:
if experiment_id not in experiments_dict:
raise BadRequest("some experiments do not exist")
deployments = []
for experiment_id in deployment.experiments:
experiment = experiments_dict[experiment_id]
deployment = models.Deployment(uuid=uuid_alpha(),
experiment_id=experiment_id,
name=experiment.name,
project_id=project_id)
self.session.add(deployment)
self.session.flush()
deployments.append(deployment)
self.copy_operators(project_id=project_id,
experiment_id=experiment_id,
deployment_id=deployment.uuid)
self.fix_positions(
project_id=project_id,
deployment_id=deployment.uuid,
new_position=sys.maxsize) # will add to end of list
# Temporary: also run deployment (while web-ui isn't ready)
for deployment in deployments:
self.run_controller.create_run(project_id=project_id,
deployment_id=deployment.uuid)
self.session.commit()
self.session.refresh(deployment)
return schemas.Deployment.from_orm(deployment)
# -*- coding: utf-8 -*-
from json import dumps
from unittest import TestCase
from fastapi.testclient import TestClient
from projects.api.main import app
from projects.controllers.utils import uuid_alpha
from projects.database import engine
from projects.object_storage import BUCKET_NAME
TEST_CLIENT = TestClient(app)
OPERATOR_ID = str(uuid_alpha())
OPERATOR_ID_2 = str(uuid_alpha())
NAME = "foo"
NAME_2 = "bar"
DEPLOYMENT_MOCK_NAME = "Foo Deployment"
DESCRIPTION = "long foo"
PROJECT_ID = str(uuid_alpha())
EXPERIMENT_ID = str(uuid_alpha())
EXPERIMENT_ID_2 = str(uuid_alpha())
DEPLOYMENT_ID = str(uuid_alpha())
DEPLOYMENT_ID_2 = str(uuid_alpha())
TEMPLATE_ID = str(uuid_alpha())
TASK_ID = str(uuid_alpha())
TASK_ID_2 = str(uuid_alpha())
RUN_ID = str(uuid_alpha())
PARAMETERS = {"coef": 0.1, "dataset": "dataset_name.csv"}
POSITION = 0
POSITION_2 = 1
# -*- coding: utf-8 -*-
from json import dumps
from unittest import TestCase
from projects.api.main import app
from projects.controllers.utils import uuid_alpha
from projects.database import engine
from projects.object_storage import BUCKET_NAME
EXPERIMENT_ID = str(uuid_alpha())
NAME = "foo"
POSITION = 0
EXPERIMENT_ID_2 = str(uuid_alpha())
NAME_2 = "foo 2"
POSITION_2 = 1
PROJECT_ID = str(uuid_alpha())
TEMPLATE_ID = str(uuid_alpha())
TASK_ID = str(uuid_alpha())
OPERATOR_ID = str(uuid_alpha())
OPERATOR_ID_2 = str(uuid_alpha())
OPERATOR_ID_3 = str(uuid_alpha())
OPERATOR_ID_4 = str(uuid_alpha())
DEPENDENCY_ID = str(uuid_alpha())
IS_ACTIVE = True
PARAMETERS = {"coef": 0.1}
PARAMETERS_JSON = dumps(PARAMETERS)
DESCRIPTION = "long foo"
IMAGE = "platiagro/platiagro-notebook-image-test:0.1.0"
COMMANDS = ["CMD"]
COMMANDS_JSON = dumps(COMMANDS)
ARGUMENTS = ["ARG"]
def create_response(self, project_id: str, deployment_id: str, body: dict):
"""
Creates a response entry in logs file.
Parameters
----------
project_id : str
deployment_id : str
body : dict
"""
if "data" in body:
ndarray = pd.DataFrame(body["data"]["ndarray"])
if "names" in body["data"]:
names = body["data"]["names"]
ndarray.columns = names
body = ndarray.to_dict(orient="records")
if isinstance(body, dict):
body = [body]
responses = []
for record in body:
responses.append(
models.Response(
uuid=uuid_alpha(),
deployment_id=deployment_id,
body=record,
))
self.session.bulk_save_objects(responses)
self.session.commit()
responses = (self.session.query(
models.Response).filter_by(deployment_id=deployment_id).order_by(
models.Response.created_at.asc()).all())
d = []
if len(responses) > 0:
for response in responses:
d.append(response.body)
data = pd.DataFrame(d)
# sends latest data to broker
response = requests.post(
BROKER_URL,
json={
"data": {
"ndarray": data.values.tolist(),
"names": data.columns.tolist(),
},
},
headers={
"Ce-Id": str(uuid.uuid4()),
"Ce-Specversion": "1.0",
"Ce-Type": f"deployment.{deployment_id}",
"Ce-Source": "logger.anonymous",
},
)
response.raise_for_status()
# -*- coding: utf-8 -*-
from io import BytesIO
from json import dumps
from unittest import TestCase
from minio.error import BucketAlreadyOwnedByYou
from projects.api.main import app
from projects.controllers.utils import uuid_alpha
from projects.database import engine
from projects.object_storage import BUCKET_NAME, MINIO_CLIENT
TASK_ID = str(uuid_alpha())
NAME = "foo"
DESCRIPTION = "long foo"
IMAGE = "platiagro/platiagro-notebook-image-test:0.1.0"
COMMANDS = ["CMD"]
COMMANDS_JSON = dumps(COMMANDS)
ARGUMENTS = ["ARG"]
ARGUMENTS_JSON = dumps(ARGUMENTS)
TAGS = ["PREDICTOR"]
TAGS_JSON = dumps(TAGS)
EXPERIMENT_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/tasks/{TASK_ID}/Experiment.ipynb"
DEPLOYMENT_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/tasks/{TASK_ID}/Deployment.ipynb"
CREATED_AT = "2000-01-01 00:00:00"
UPDATED_AT = "2000-01-01 00:00:00"
class TestParameters(TestCase):
def setUp(self):
self.maxDiff = None
def create_operator(
self,
operator: schemas.OperatorCreate,
project_id: str,
experiment_id: Optional[str] = None,
deployment_id: Optional[str] = None,
):
"""
Creates a new operator in our database.
Parameters
----------
operator: projects.schemas.operator.OperatorCreate
project_id : str
experiment_id : str or None
deployment_id : str or None
Returns
-------
projects.schemas.operator.Operator
Raises
------
BadRequest
When the operator attributes are invalid.
"""
if not isinstance(operator.task_id, str):
raise BadRequest(code="MissingRequiredTaskId",
message="taskId is required")
try:
self.task_controller.raise_if_task_does_not_exist(operator.task_id)
except NotFound:
raise BadRequest(code="InvalidTaskId",
message="source task does not exist")
if operator.dependencies is None:
operator.dependencies = []
if experiment_id:
self.raise_if_dependencies_are_invalid(
project_id=project_id,
experiment_id=experiment_id,
deployment_id=deployment_id,
dependencies=operator.dependencies,
)
if experiment_id and deployment_id:
raise BadRequest(
code="InvalidOperatorRequestBody",
message=
"Operator cannot contain an experiment and a deployment simultaneously",
)
if operator.parameters is None:
operator.parameters = {}
self.raise_if_parameters_are_invalid(operator.parameters)
operator = models.Operator(uuid=uuid_alpha(),
name=operator.name,
experiment_id=experiment_id,
deployment_id=deployment_id,
task_id=operator.task_id,
dependencies=operator.dependencies,
status=operator.status,
parameters=operator.parameters,
position_x=operator.position_x,
position_y=operator.position_y,
created_at=now(),
updated_at=now())
self.session.add(operator)
self.session.commit()
self.session.refresh(operator)
return schemas.Operator.from_orm(operator)
# -*- coding: utf-8 -*-
from io import BytesIO
from json import dumps, loads
from unittest import TestCase
import requests
from minio.error import BucketAlreadyOwnedByYou
from projects.api.main import app
from projects.controllers.utils import uuid_alpha
from projects.database import engine
from projects.jupyter import JUPYTER_ENDPOINT, COOKIES, HEADERS
from projects.object_storage import BUCKET_NAME, MINIO_CLIENT
COMPONENT_ID = str(uuid_alpha())
NAME = "foo"
DESCRIPTION = "long foo"
TAGS = ["PREDICTOR"]
TAGS_JSON = dumps(TAGS)
TRAINING_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/components/{COMPONENT_ID}/Training.ipynb"
INFERENCE_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/components/{COMPONENT_ID}/Inference.ipynb"
IS_DEFAULT = False
PARAMETERS = []
CREATED_AT = "2000-01-01 00:00:00"
CREATED_AT_ISO = "2000-01-01T00:00:00"
UPDATED_AT = "2000-01-01 00:00:00"
UPDATED_AT_ISO = "2000-01-01T00:00:00"
SAMPLE_NOTEBOOK = '{"cells":[{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.9"}},"nbformat":4,"nbformat_minor":4}'
class TestComponents(TestCase):
def create_task(self, task: schemas.TaskCreate):
"""
Creates a new task in our database and a volume claim in the cluster.
Parameters
----------
task: projects.schemas.task.TaskCreate
Returns
-------
projects.schemas.task.Task
Raises
------
BadRequest
When task attributes are invalid.
"""
has_notebook = task.experiment_notebook or task.deployment_notebook
if task.copy_from and has_notebook:
raise BadRequest(
code="MissingRequiredNotebookOrTaskId",
message="Either provide notebooks or a task to copy from",
)
if not task.tags:
task.tags = [CATEGORY_DEFAULT]
self.task_category_is_not_none(task)
if not task.category:
task.category = CATEGORY_DEFAULT
# check if image is a valid docker image
self.raise_if_invalid_docker_image(task.image)
check_comp_name = (self.session.query(
models.Task).filter_by(name=task.name).first())
if check_comp_name:
raise BadRequest(code="TaskNameExists",
message="a task with that name already exists")
# creates a task with specified name,
# but copies notebooks from a source task
stored_task = None
if task.copy_from:
stored_task = self.session.query(models.Task).get(task.copy_from)
if stored_task is None:
raise BadRequest(code="InvalidTaskId",
message="source task does not exist")
task.image = stored_task.image
task.commands = stored_task.commands
task.arguments = stored_task.arguments
task.parameters = stored_task.parameters
experiment_notebook_path = stored_task.experiment_notebook_path
deployment_notebook_path = stored_task.deployment_notebook_path
task.cpu_limit = stored_task.cpu_limit
task.cpu_request = stored_task.cpu_request
task.memory_limit = stored_task.memory_limit
task.memory_request = stored_task.memory_request
# Adding the task name if it is a copy.
task.name = self.generate_name_task(f"{stored_task.name} - Cópia")
else:
if not isinstance(task.name, str):
task.name = self.generate_name_task("Tarefa em branco")
if task.image is not None:
experiment_notebook_path = None
deployment_notebook_path = None
task.experiment_notebook = None
task.deployment_notebook = None
else:
# relative path to the mount_path
experiment_notebook_path = "Experiment.ipynb"
deployment_notebook_path = "Deployment.ipynb"
task.experiment_notebook = EXPERIMENT_NOTEBOOK
task.deployment_notebook = DEPLOYMENT_NOTEBOOK
task_id = str(uuid_alpha())
task_dict = task.dict(exclude_unset=True)
task_dict.pop("copy_from", None)
task_dict.pop("experiment_notebook", None)
task_dict.pop("deployment_notebook", None)
task_dict["uuid"] = task_id
task_dict["experiment_notebook_path"] = experiment_notebook_path
task_dict["deployment_notebook_path"] = deployment_notebook_path
# saves task info to the database
task = models.Task(**task_dict)
self.session.add(task)
self.session.commit()
self.session.refresh(task)
all_tasks = self.session.query(models.Task).all()
make_task_creation_job(
task=task,
namespace=KF_PIPELINES_NAMESPACE,
all_tasks=all_tasks,
copy_from=stored_task,
)
return schemas.Task.from_orm(task)
# -*- coding: utf-8 -*-
from unittest import TestCase
from projects.api.main import app
from projects.controllers.utils import uuid_alpha
from projects.database import engine
PROJECT_ID = str(uuid_alpha())
NAME = "foo"
CREATED_AT = "2000-01-01 00:00:00"
CREATED_AT_ISO = "2000-01-01T00:00:00"
UPDATED_AT = "2000-01-01 00:00:00"
UPDATED_AT_ISO = "2000-01-01T00:00:00"
EXPERIMENT_ID = str(uuid_alpha())
EXPERIMENT_NAME = "Novo experimento"
DESCRIPTION = "Description"
class TestProjects(TestCase):
def setUp(self):
self.maxDiff = None
conn = engine.connect()
text = (
f"INSERT INTO projects (uuid, name, created_at, updated_at, description) "
f"VALUES ('{PROJECT_ID}', '{NAME}', '{CREATED_AT}', '{UPDATED_AT}', '{DESCRIPTION}')"
)
conn.execute(text)
text = (
f"INSERT INTO experiments (uuid, name, project_id, dataset, target, position, is_active, created_at, updated_at) "
f"VALUES ('{EXPERIMENT_ID}', '{EXPERIMENT_NAME}', '{PROJECT_ID}', null, null, 0, 1, '{CREATED_AT}', '{UPDATED_AT}')"
def copy_operators(self, deployment_id: str,
stored_operators: schemas.OperatorList):
"""
Copies the operators to a deployment.
Creates new uuids and don't keep the experiment_id/deployment_id relationship.
Parameters
----------
stored_operators : projects.schemas.operator.OperatorsList
deployment_id : str
"""
if len(stored_operators) == 0:
raise BadRequest(
code="MissingRequiredOperatorId",
message="Necessary at least one operator.",
)
# Creates a dict to map source operator_id to its copy operator_id.
# This map will be used to build the dependencies using new operator_ids
copies_map = {}
# just a simple flag to detect the existence of a dataset operator
some_stored_operators_is_dataset = False
# default position, in case we have to create a dataset operator
leftmost_operator_position = (
stored_operators[0].position_x,
stored_operators[0].position_y,
)
for stored_operator in stored_operators:
# If we have to create a dataset operator, it is interesting that we put before the leftmost position
if stored_operator.position_x < leftmost_operator_position[0]:
leftmost_operator_position = (
stored_operator.position_x,
stored_operator.position_y,
)
if stored_operator.task.category == "DATASETS":
name = FONTE_DE_DADOS
parameters = {"type": "L", "dataset": None}
some_stored_operators_is_dataset = True
else:
name = None
parameters = stored_operator.parameters
operator_id = uuid_alpha()
operator = models.Operator(
uuid=operator_id,
name=name,
deployment_id=deployment_id,
task_id=stored_operator.task_id,
dependencies=[],
status="Setted up",
parameters=parameters,
position_x=stored_operator.position_x,
position_y=stored_operator.position_y,
)
self.session.add(operator)
self.session.flush()
copies_map[stored_operator.uuid] = {
"copy_uuid": operator_id,
"dependencies": stored_operator.dependencies,
}
# creates a DATASET type operator if doesn't exist any
if not some_stored_operators_is_dataset:
generated_dataset_operator_uuid = uuid_alpha()
operator = models.Operator(
uuid=generated_dataset_operator_uuid,
name=FONTE_DE_DADOS,
deployment_id=deployment_id,
task_id=self.task_controller.
get_or_create_dataset_task_if_not_exist(),
dependencies=[],
parameters={
"type": "L",
"dataset": None
},
position_x=leftmost_operator_position[0] -
DATASET_OPERATOR_DISTANCE,
position_y=leftmost_operator_position[1],
)
self.session.add(operator)
self.session.flush()
self.set_dependents_for_generated_dataset_operator(
copies_map, generated_dataset_operator_uuid)
self.set_dependencies_on_new_operators(copies_map)
# -*- coding: utf-8 -*-
from json import dumps
from os import remove
from unittest import TestCase
from fastapi.testclient import TestClient
from projects.api.main import app
from projects.controllers.utils import uuid_alpha
from projects.database import engine
from projects.kfp import kfp_client
from projects.object_storage import BUCKET_NAME
TEST_CLIENT = TestClient(app)
DEPLOYMENT_ID = str(uuid_alpha())
DEPLOYMENT_ID_2 = str(uuid_alpha())
NAME = "foo-bar"
PROJECT_ID = str(uuid_alpha())
CREATED_AT = "2000-01-01 00:00:00"
UPDATED_AT = "2000-01-01 00:00:00"
EXPERIMENT_ID = str(uuid_alpha())
POSITION = 0
OPERATOR_ID = str(uuid_alpha())
OPERATOR_ID_2 = str(uuid_alpha())
TASK_ID = str(uuid_alpha())
TASK_ID_2 = str(uuid_alpha())
PARAMETERS_JSON = dumps({"coef": 0.1})
DEP_EMPTY_JSON = dumps([])
IMAGE = "platiagro/platiagro-experiment-image:0.2.0"
TAGS_JSON = dumps(["PREDICTOR"])
# -*- coding: utf-8 -*-
from json import dumps
from unittest import TestCase
from fastapi.testclient import TestClient
from projects.api.main import app
from projects.controllers.utils import uuid_alpha
from projects.database import engine
from projects.object_storage import BUCKET_NAME
TEST_CLIENT = TestClient(app)
OPERATOR_ID = str(uuid_alpha())
OPERATOR_ID_2 = str(uuid_alpha())
OPERATOR_ID_3 = str(uuid_alpha())
OPERATOR_ID_4 = str(uuid_alpha())
OPERATOR_ID_5 = str(uuid_alpha())
OPERATOR_ID_6 = str(uuid_alpha())
DEPENDENCY_ID = str(uuid_alpha())
DEPENDENCY_ID_2 = str(uuid_alpha())
NAME = "foo"
NAME_2 = "bar"
DESCRIPTION = "long foo"
PROJECT_ID = str(uuid_alpha())
EXPERIMENT_ID = str(uuid_alpha())
EXPERIMENT_ID_2 = str(uuid_alpha())
TASK_ID = str(uuid_alpha())
PARAMETERS = {"coef": 0.1}
POSITION = 0
POSITION_2 = 1
# -*- coding: utf-8 -*-
from json import dumps
from unittest import TestCase
from projects.api.main import app
from projects.controllers.utils import uuid_alpha
from projects.database import engine
from projects.object_storage import BUCKET_NAME
EXPERIMENT_ID = str(uuid_alpha())
NAME = "foo"
PROJECT_ID = str(uuid_alpha())
TEMPLATE_ID = str(uuid_alpha())
COMPONENT_ID = str(uuid_alpha())
OPERATOR_ID = str(uuid_alpha())
DATASET = "iris"
TARGET = "col4"
POSITION = 0
IS_ACTIVE = True
PARAMETERS = {"coef": 0.1}
PARAMETERS_JSON = dumps(PARAMETERS)
DESCRIPTION = "long foo"
TAGS = ["PREDICTOR"]
TAGS_JSON = dumps(TAGS)
COMPONENTS_JSON = dumps([COMPONENT_ID])
TRAINING_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/components/{COMPONENT_ID}/Training.ipynb"
INFERENCE_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/components/{COMPONENT_ID}/Inference.ipynb"
CREATED_AT = "2000-01-01 00:00:00"
CREATED_AT_ISO = "2000-01-01T00:00:00"
UPDATED_AT = "2000-01-01 00:00:00"
UPDATED_AT_ISO = "2000-01-01T00:00:00"
# -*- coding: utf-8 -*-
from unittest import TestCase
from projects.api.main import app
from projects.controllers.utils import uuid_alpha
from projects.database import engine
EXPERIMENT_ID = str(uuid_alpha())
PROJECT_ID = str(uuid_alpha())
COMPARE_RESULT_ID = str(uuid_alpha())
NAME = "foo"
POSITION = 0
CREATED_AT = "2000-01-01 00:00:00"
CREATED_AT_ISO = "2000-01-01T00:00:00"
UPDATED_AT = "2000-01-01 00:00:00"
UPDATED_AT_ISO = "2000-01-01T00:00:00"
class TestCompareResults(TestCase):
def setUp(self):
self.maxDiff = None
conn = engine.connect()
text = (
f"INSERT INTO projects (uuid, name, created_at, updated_at) "
f"VALUES ('{PROJECT_ID}', '{NAME}', '{CREATED_AT}', '{UPDATED_AT}')"
)
conn.execute(text)
text = (
f"INSERT INTO experiments (uuid, name, project_id, position, is_active, created_at, updated_at) "
f"VALUES ('{EXPERIMENT_ID}', '{NAME}', '{PROJECT_ID}', '{POSITION}', 1, '{CREATED_AT}', '{UPDATED_AT}')"
