def create_operator(project_id,
deployment_id,
task_id=None,
parameters=None,
dependencies=None,
position_x=None,
position_y=None):
"""Creates a new operator in our database.
Args:
project_id (str): the project uuid.
deployment_id (str): the deployment uuid.
task_id (str): the task uuid.
parameters (dict): the parameters dict.
dependencies (list): the dependencies array.
position_x (float): position x.
position_y (float): position y.
Returns:
The operator info.
"""
if not isinstance(task_id, str):
raise BadRequest("taskId is required")
try:
raise_if_task_does_not_exist(task_id)
except NotFound as e:
raise BadRequest(e.description)
if parameters is None:
parameters = {}
raise_if_parameters_are_invalid(parameters)
if dependencies is None:
dependencies = []
operator = Operator(uuid=uuid_alpha(),
deployment_id=deployment_id,
task_id=task_id,
dependencies=dependencies,
parameters=parameters,
position_x=position_x,
position_y=position_y)
db_session.add(operator)
return operator.as_dict()
# -*- coding: utf-8 -*-
from io import BytesIO
from json import dumps
from unittest import TestCase
from minio.error import BucketAlreadyOwnedByYou
from platiagro import CATEGORICAL, DATETIME, NUMERICAL
from pipelines.api.main import app
from pipelines.database import engine
from pipelines.object_storage import BUCKET_NAME, MINIO_CLIENT
from pipelines.utils import uuid_alpha
PROJECT_ID = str(uuid_alpha())
TASK_ID = str(uuid_alpha())
RUN_ID = str(uuid_alpha())
EXP_ID_1 = str(uuid_alpha())
OP_ID_1_1 = str(uuid_alpha())
OP_ID_1_2 = str(uuid_alpha())
EXP_ID_2 = str(uuid_alpha())
OP_ID_2_1 = str(uuid_alpha())
OP_ID_2_2 = str(uuid_alpha())
EXP_ID_3 = str(uuid_alpha())
OP_ID_3_1 = str(uuid_alpha())
NAME = "foo"
DATASET = "mock.csv"
DATASET_RUN_ID_NONE = "teste.csv"
IMAGE = "platiagro/platiagro-notebook-image-test:0.2.0"
PARAMETERS = {"dataset": DATASET}
PARAMETERS_JSON = dumps(PARAMETERS)
EXPERIMENT_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/tasks/{TASK_ID}/Experiment.ipynb"
# -*- coding: utf-8 -*-
from json import dumps
from unittest import TestCase
from pipelines.api.main import app
from pipelines.utils import uuid_alpha
from pipelines.database import engine
from pipelines.object_storage import BUCKET_NAME
from pipelines.controllers.utils import init_pipeline_client
MOCKED_TRAINING_ID = "b281185b-6104-4c8c-8185-31eb53bef8de"
PROJECT_ID = str(uuid_alpha())
TASK_ID = str(uuid_alpha())
ARGUMENTS_JSON = dumps(["ARG"])
COMMANDS_JSON = dumps(["CMD"])
CREATED_AT = "2000-01-01 00:00:00"
DEPLOY_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/tasks/{TASK_ID}/Deployment.ipynb"
EX_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/tasks/{TASK_ID}/Experiment.ipynb"
IMAGE = "platiagro/platiagro-notebook-image:0.2.0"
PARAMETERS_JSON = dumps({"coef": 0.1})
TAGS_JSON = dumps(["PREDICTOR"])
UPDATED_AT = "2000-01-01 00:00:00"
DEP_EMPTY_JSON = dumps([])
EX_ID_1 = str(uuid_alpha())
OP_ID_1_1 = str(uuid_alpha())
OP_ID_1_2 = str(uuid_alpha())
DEP_OP_ID_1_1 = [OP_ID_1_1]
DEP_OP_ID_1_1_JSON = dumps(DEP_OP_ID_1_1)
EX_ID_2 = str(uuid_alpha())
# -*- coding: utf-8 -*-
from json import dumps
from unittest import TestCase
from pipelines.api.main import app
from pipelines.database import engine
from pipelines.object_storage import BUCKET_NAME
from pipelines.utils import uuid_alpha
DEPLOYMENT_ID = str(uuid_alpha())
DEPLOYMENT_ID_2 = str(uuid_alpha())
EXPERIMENT_ID = str(uuid_alpha())
PROJECT_ID = str(uuid_alpha())
OPERATOR_ID = str(uuid_alpha())
TASK_ID = str(uuid_alpha())
NAME = "foo"
NAME_2 = "foo 2"
POSITION = 0
IS_ACTIVE = True
PARAMETERS = {"coef": 0.1}
PARAMETERS_JSON = dumps(PARAMETERS)
DESCRIPTION = "long foo"
IMAGE = "platiagro/platiagro-notebook-image-test:0.2.0"
COMMANDS = ["CMD"]
COMMANDS_JSON = dumps(COMMANDS)
ARGUMENTS = ["ARG"]
ARGUMENTS_JSON = dumps(ARGUMENTS)
TAGS = ["PREDICTOR"]
TAGS_JSON = dumps(TAGS)
TASKS_JSON = dumps([TASK_ID])
EXPERIMENT_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/tasks/{TASK_ID}/Experiment.ipynb"
# -*- coding: utf-8 -*-
from unittest import TestCase
import platiagro
from pipelines.api.main import app
from pipelines.object_storage import BUCKET_NAME, MINIO_CLIENT
from pipelines.utils import uuid_alpha
EXPERIMENT_ID = str(uuid_alpha())
OPERATOR_ID = str(uuid_alpha())
RUN_ID = str(uuid_alpha())
class TestMetrics(TestCase):
def setUp(self):
self.maxDiff = None
platiagro.save_metrics(experiment_id=EXPERIMENT_ID,
operator_id=OPERATOR_ID,
run_id=RUN_ID,
accuracy=1.0)
def tearDown(self):
prefix = f"experiments/{EXPERIMENT_ID}"
for obj in MINIO_CLIENT.list_objects(BUCKET_NAME,
prefix=prefix,
recursive=True):
MINIO_CLIENT.remove_object(BUCKET_NAME, obj.object_name)
def test_list_metrics(self):
with app.test_client() as c:
rv = c.get(
# -*- coding: utf-8 -*-
from json import dumps
from unittest import TestCase
from pipelines.api.main import app
from pipelines.controllers.utils import init_pipeline_client
from pipelines.database import engine
from pipelines.object_storage import BUCKET_NAME
from pipelines.utils import uuid_alpha
MOCKED_DEPLOYMENT_ID = "aa23c286-1524-4ae9-ae44-6c3e63eb9861"
PROJECT_ID = str(uuid_alpha())
TASK_ID = str(uuid_alpha())
ARGUMENTS_JSON = dumps(["ARG"])
COMMANDS_JSON = dumps(["CMD"])
CREATED_AT = "2000-01-01 00:00:00"
DEPLOY_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/tasks/{TASK_ID}/Deployment.ipynb"
EX_NOTEBOOK_PATH = f"minio://{BUCKET_NAME}/tasks/{TASK_ID}/Experiment.ipynb"
IMAGE = "platiagro/platiagro-notebook-image:0.2.0"
PARAMETERS_JSON = dumps({"coef": 0.1})
TAGS_JSON = dumps(["PREDICTOR"])
UPDATED_AT = "2000-01-01 00:00:00"
DEP_EMPTY_JSON = dumps([])
EX_ID_1 = str(uuid_alpha())
OP_ID_1_1 = str(uuid_alpha())
OP_ID_1_2 = str(uuid_alpha())
DEP_OP_ID_1_1 = [OP_ID_1_1]
DEP_OP_ID_1_1_JSON = dumps(DEP_OP_ID_1_1)
EX_ID_2 = str(uuid_alpha())
def create_deployment(experiment_id=None,
is_active=None,
name=None,
operators=None,
position=None,
project_id=None,
status=None):
"""Creates a new deployment in our database and adjusts the position of others.
Args:
experiment_id (str): the experiment uuid.
is_active (bool): if deployment is active.
name (str): the deployment name.
operators (list): the deployment operators.
position (int): the deployment position.
project_id (str): the project uuid.
status (str): the deployment status.
Returns:
The deployment info.
"""
raise_if_project_does_not_exist(project_id)
raise_if_experiment_does_not_exist(experiment_id)
if not isinstance(name, str):
raise BadRequest("name is required")
if status and status not in VALID_STATUS:
raise BadRequest("invalid status")
check_deployment_name = db_session.query(Deployment)\
.filter(Deployment.project_id == project_id)\
.filter(Deployment.name == name)\
.first()
if check_deployment_name:
raise BadRequest("a deployment with that name already exists")
deployment = Deployment(uuid=uuid_alpha(),
experiment_id=experiment_id,
is_active=is_active,
name=name,
project_id=project_id,
status=status)
db_session.add(deployment)
if operators and len(operators) > 0:
for operator in operators:
create_operator(deployment_id=deployment.uuid,
project_id=project_id,
task_id=operator.get('taskId'),
parameters=operator.get('parameters'),
dependencies=operator.get('dependencies'),
position_x=operator.get('positionX'),
position_y=operator.get('positionY'))
db_session.commit()
if position is None:
position = sys.maxsize # will add to end of list
fix_positions(project_id=project_id,
deployment_id=deployment.uuid,
new_position=position)
return deployment.as_dict()