def delete_deployment(deployment_id): """Delete Args: deployment_id (str): deployment id. """ kfp_client = init_pipeline_client() # Get all SeldonDeployment resources. load_kube_config() custom_api = client.CustomObjectsApi() ret = custom_api.list_namespaced_custom_object( "machinelearning.seldon.io", "v1alpha2", "deployments", "seldondeployments" ) deployments = ret['items'] # Delete SeldonDeployment resource. if deployments: for deployment in deployments: if deployment['metadata']['name'] == deployment_id: delete_deployment_resource(deployment) # Delete deployment run deployment_run_id = get_deployment_by_id(deployment_id)['runId'] kfp_client.runs.delete_run(deployment_run_id) return { "message": "Deployment deleted." }
def get_experiment_run_history(experiment_id): """Get experiment run history. Args: experiment_id (str): PlatIA experiment_id. Returns: Experiment run history. """ try: client = init_pipeline_client() experiment = client.get_experiment(experiment_name=experiment_id) experiment_runs = client.list_runs(page_size='100', sort_by=created_at_desc, experiment_id=experiment.id) response = [] for run in experiment_runs.runs: workflow_manifest = json.loads(run.pipeline_spec.workflow_manifest) if workflow_manifest['metadata'][ 'generateName'] == 'common-pipeline-': run_id = run.id run_details = client.get_run(run_id) formated_operators = format_run_operators(run_details) if formated_operators: resp = {} resp['runId'] = run_id resp['createdAt'] = run.created_at resp['operators'] = formated_operators response.append(resp) except Exception: return [] return response
def get_deployments(): """Get deployments list. Returns: Deployments list. """ kfp_client = init_pipeline_client() token = '' deployment_runs = [] protocol = get_protocol() ip = get_cluster_ip() while True: list_runs = kfp_client.list_runs( page_token=token, sort_by='created_at desc', page_size=100) if list_runs.runs: runs = get_deployment_details(list_runs.runs, ip, protocol) deployment_runs.extend(runs) token = list_runs.next_page_token if token is None: break else: break return deployment_runs
def __init__(self, experiment_id, name, operators): """Create a new instance of Pipeline. Args: experiment_id (str): PlatIAgro experiment's uuid. name (str): deployment name. operators (list): list of pipeline operators. """ self._roots = [] self._operators = {} self._edges = defaultdict(list) # source: [destinations] self._inverted_edges = defaultdict(list) # destination: [sources] self._experiment_id = experiment_id self._name = name self._client = init_pipeline_client() self._experiment = self._client.create_experiment(name=experiment_id) for operator in operators: self._add_operator(operator) # Verify if the given pipeline has cycles if self._is_cyclic(): raise BadRequest('The given pipeline has cycles.')
def retry_experiment_run(experiment_id): """Re-initiate a failed or terminated experiment run. Args: experiment_id (str): PlatIA experiment_id. Returns: Experiment run details. """ client = init_pipeline_client() experiment = client.get_experiment(experiment_name=experiment_id) experiment_runs = client.list_runs(page_size='1', sort_by=created_at_desc, experiment_id=experiment.id) retry = False for run in experiment_runs.runs: if 'Failed' == run.status: init_pipeline_client().runs.retry_run(run_id=run.id) retry = True if not retry: raise NotFound('There is no failed experimentation') run_details = client.get_run(run.id) return format_pipeline_run_details(run_details)
def terminate_experiment_run(experiment_id): """Terminate experiment run. Args: experiment_id (str): PlatIA experiment_id. Returns: Deleted message. """ client = init_pipeline_client() experiment = client.get_experiment(experiment_name=experiment_id) experiment_runs = client.list_runs(page_size='1', sort_by=created_at_desc, experiment_id=experiment.id) for run in experiment_runs.runs: client.runs.terminate_run(run_id=run.id) response = {"message": "Training deleted."} return response
def delete_deployment_resource(deployment_resource): """Delete deployment resource.""" kfp_client = init_pipeline_client() @dsl.pipeline(name='Undeploy') def undeploy(): dsl.ResourceOp( name='undeploy', k8s_resource=deployment_resource, action='delete' ) kfp_client.create_run_from_pipeline_func( undeploy, {}, run_name='undeploy', namespace=KF_PIPELINES_NAMESPACE, )
def get_experiment_run(experiment_id, pretty=True): """Get experiment run details. Args: experiment_id (str): PlatIA experiment_id. pretty (boolean): well formated response Returns: Run details. """ run_details = '' try: client = init_pipeline_client() experiment = client.get_experiment(experiment_name=experiment_id) # lists runs for trainings and deployments of an experiment experiment_runs = client.list_runs(page_size='100', sort_by=created_at_desc, experiment_id=experiment.id) # find the latest training run latest_training_run = None for run in experiment_runs.runs: workflow_manifest = json.loads(run.pipeline_spec.workflow_manifest) if workflow_manifest['metadata'][ 'generateName'] == 'common-pipeline-': latest_training_run = run break if latest_training_run: run_id = latest_training_run.id run_details = client.get_run(run_id) else: return {} except Exception: return {} if pretty: return format_pipeline_run_details(run_details) else: return run_details
def retry_run_deployment(deployment_id): experiment = list(filter(lambda d: d['experimentId'] == deployment_id, get_deployments()))[0] experiment = init_pipeline_client().runs.retry_run(run_id=experiment['runId']) return experiment
def setUp(self): # Run a default pipeline for tests client = init_pipeline_client() experiment = client.create_experiment(name=MOCKED_TRAINING_ID) client.run_pipeline(experiment.id, MOCKED_TRAINING_ID, "tests/resources/mocked_training.yaml") conn = engine.connect() text = ( f"INSERT INTO tasks (uuid, name, description, image, commands, arguments, tags, experiment_notebook_path, deployment_notebook_path, is_default, created_at, updated_at) " f"VALUES ('{TASK_ID}', 'name', 'desc', '{IMAGE}', '{COMMANDS_JSON}', '{ARGUMENTS_JSON}', '{TAGS_JSON}', '{EX_NOTEBOOK_PATH}', '{DEPLOY_NOTEBOOK_PATH}', 0, '{CREATED_AT}', '{UPDATED_AT}')" ) conn.execute(text) text = ( f"INSERT INTO projects (uuid, name, created_at, updated_at) " f"VALUES ('{PROJECT_ID}', 'name', '{CREATED_AT}', '{UPDATED_AT}')") conn.execute(text) # Experiment 1 text = ( f"INSERT INTO experiments (uuid, name, project_id, position, is_active, created_at, updated_at) " f"VALUES ('{EX_ID_1}', 'ex1', '{PROJECT_ID}', '0', 1, '{CREATED_AT}', '{UPDATED_AT}')" ) conn.execute(text) text = ( f"INSERT INTO operators (uuid, experiment_id, task_id, parameters, created_at, updated_at, dependencies) " f"VALUES ('{OP_ID_1_1}', '{EX_ID_1}', '{TASK_ID}', '{PARAMETERS_JSON}', '{CREATED_AT}', '{UPDATED_AT}', '{DEP_EMPTY_JSON}')" ) conn.execute(text) text = ( f"INSERT INTO operators (uuid, experiment_id, task_id, parameters, created_at, updated_at, dependencies) " f"VALUES ('{OP_ID_1_2}', '{EX_ID_1}', '{TASK_ID}', '{PARAMETERS_JSON}', '{CREATED_AT}', '{UPDATED_AT}', '{DEP_OP_ID_1_1_JSON}')" ) conn.execute(text) # Experiment 2 text = ( f"INSERT INTO experiments (uuid, name, project_id, position, is_active, created_at, updated_at) " f"VALUES ('{EX_ID_2}', 'ex2', '{PROJECT_ID}', '1', 1, '{CREATED_AT}', '{UPDATED_AT}')" ) conn.execute(text) # Experiment 3 text = ( f"INSERT INTO experiments (uuid, name, project_id, position, is_active, created_at, updated_at) " f"VALUES ('{EX_ID_3}', 'ex3', '{PROJECT_ID}', '2', 1, '{CREATED_AT}', '{UPDATED_AT}')" ) conn.execute(text) text = ( f"INSERT INTO operators (uuid, experiment_id, task_id, parameters, created_at, updated_at, dependencies) " f"VALUES ('{OP_ID_3_1}', '{EX_ID_3}', '{TASK_ID}', '{PARAMETERS_JSON}', '{CREATED_AT}', '{UPDATED_AT}', '{DEP_OP_INVALID_JSON}')" ) conn.execute(text) # Experiment 4 text = ( f"INSERT INTO experiments (uuid, name, project_id, position, is_active, created_at, updated_at) " f"VALUES ('{EX_ID_4}', 'ex4', '{PROJECT_ID}', '3', 1, '{CREATED_AT}', '{UPDATED_AT}')" ) conn.execute(text) text = ( f"INSERT INTO operators (uuid, experiment_id, task_id, parameters, created_at, updated_at, dependencies) " f"VALUES ('{OP_ID_4_1}', '{EX_ID_4}', '{TASK_ID}', '{PARAMETERS_JSON}', '{CREATED_AT}', '{UPDATED_AT}', '{DEP_OP_ID_4_2_JSON}')" ) conn.execute(text) text = ( f"INSERT INTO operators (uuid, experiment_id, task_id, parameters, created_at, updated_at, dependencies) " f"VALUES ('{OP_ID_4_2}', '{EX_ID_4}', '{TASK_ID}', '{PARAMETERS_JSON}', '{CREATED_AT}', '{UPDATED_AT}', '{DEP_OP_ID_4_1_JSON}')" ) conn.execute(text) conn.close()