def _create_json( cls, conn, workspace, path, description=None, public_within_org=None, visibility=None, ): path = arg_handler.ensure_starts_with_slash(path) ( visibility, public_within_org, ) = _visibility._Visibility._translate_public_within_org( visibility, public_within_org) data = { "path": path, "description": description, "custom_permission": { "collaborator_type": visibility._collaborator_type_str }, "visibility": "ORG_SCOPED_PUBLIC" if public_within_org else "PRIVATE", # TODO: raise if workspace is personal "resource_visibility": visibility._visibility_str, } url = "{}://{}/api/v1/deployment/workspace/{}/endpoints".format( conn.scheme, conn.socket, workspace) response = _utils.make_request("POST", url, conn, json=data) _utils.raise_for_http_error(response) return response.json()
def delete_datasets(ids, conn): request_url = "{}://{}/api/v1/modeldb/dataset/deleteDatasets".format( conn.scheme, conn.socket) response = requests.delete(request_url, json={'ids': ids}, headers=conn.auth) _utils.raise_for_http_error(response)
def create_access_token(self, token): """ Creates an access token for the endpoint. Parameters ---------- token : str Token to create. """ if not isinstance(token, six.string_types): raise TypeError("`token` must be a string.") url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages/{}/accesstokens".format( self._conn.scheme, self._conn.socket, self.workspace, self.id, self._get_or_create_stage(), ) response = _utils.make_request("POST", url, self._conn, json={"value": token}) _utils.raise_for_http_error(response)
def delete_project(id_, conn): request_url = "{}://{}/api/v1/modeldb/project/deleteProject".format( conn.scheme, conn.socket) response = requests.delete(request_url, json={'id': id_}, headers=conn.auth) _utils.raise_for_http_error(response)
def test_no_api_deploy_error(self, experiment_run, model_for_deployment): experiment_run.log_model(model_for_deployment['model'], custom_modules=[]) experiment_run.log_requirements(['scikit-learn']) # delete model API response = _utils.make_request( "DELETE", "{}://{}/api/v1/modeldb/experiment-run/deleteArtifact".format( experiment_run._conn.scheme, experiment_run._conn.socket), experiment_run._conn, json={ 'id': experiment_run.id, 'key': "model_api.json" }) _utils.raise_for_http_error(response) with pytest.raises(RuntimeError) as excinfo: experiment_run.deploy() assert str(excinfo.value).strip( ) == "unable to deploy due to missing artifact model_api.json" conn = experiment_run._conn requests.delete( "{}://{}/api/v1/deployment/models/{}".format( conn.scheme, conn.socket, experiment_run.id), headers=conn.auth, )
def delete_experiment_run(id_, conn): request_url = "{}://{}/api/v1/modeldb/experiment-run/deleteExperimentRun".format( conn.scheme, conn.socket) response = requests.delete(request_url, json={'id': id_}, headers=conn.auth) _utils.raise_for_http_error(response)
def test_no_api_deploy_error(self, experiment_run, model_for_deployment): experiment_run.log_model(model_for_deployment['model'], custom_modules=[]) experiment_run.log_environment(Python(['scikit-learn'])) # delete model API response = _utils.make_request( "DELETE", "{}://{}/api/v1/modeldb/experiment-run/deleteArtifact".format( experiment_run._conn.scheme, experiment_run._conn.socket), experiment_run._conn, json={ 'id': experiment_run.id, 'key': _artifact_utils.MODEL_API_KEY }) _utils.raise_for_http_error(response) with pytest.raises(RuntimeError) as excinfo: experiment_run.deploy() assert _artifact_utils.MODEL_API_KEY in str(excinfo.value) conn = experiment_run._conn requests.delete( "{}://{}/api/v1/deployment/models/{}".format( conn.scheme, conn.socket, experiment_run.id), headers=conn.auth, )
def _create_build(self, model_reference): url = "{}://{}/api/v1/deployment/workspace/{}/builds".format( self._conn.scheme, self._conn.socket, self.workspace, ) if isinstance(model_reference, RegisteredModelVersion): response = _utils.make_request( "POST", url, self._conn, json={"model_version_id": model_reference.id}) elif isinstance(model_reference, ExperimentRun): response = _utils.make_request("POST", url, self._conn, json={"run_id": model_reference.id}) else: raise TypeError( "`model_reference` must be an ExperimentRun or RegisteredModelVersion" ) _utils.raise_for_http_error(response) return response.json()["id"]
def _get_endpoints(cls, conn, workspace): url = "{}://{}/api/v1/deployment/workspace/{}/endpoints".format( conn.scheme, conn.socket, workspace) response = _utils.make_request("GET", url, conn) _utils.raise_for_http_error(response) data_response = response.json() return data_response["endpoints"]
def log_training_data(self, train_features, train_targets, overwrite=False): """ Associate training data with this model reference. .. versionchanged:: 0.14.4 Instead of uploading the data itself as a CSV artifact ``'train_data'``, this method now generates a histogram for internal use by our deployment data monitoring system. .. deprecated:: 0.18.0 This method is no longer supported. Please see our documentation for information about our platform's data monitoring features. Parameters ---------- train_features : pd.DataFrame pandas DataFrame representing features of the training data. train_targets : pd.DataFrame or pd.Series pandas DataFrame representing targets of the training data. overwrite : bool, default False Whether to allow overwriting existing training data. """ warnings.warn( "This method is no longer supported. Please see our documentation" " for information about our platform's data monitoring features", category=FutureWarning, ) if train_features.__class__.__name__ != "DataFrame": raise TypeError( "`train_features` must be a pandas DataFrame, not {}".format( type(train_features))) if train_targets.__class__.__name__ == "Series": train_targets = train_targets.to_frame() elif train_targets.__class__.__name__ != "DataFrame": raise TypeError( "`train_targets` must be a pandas DataFrame or Series, not {}". format(type(train_targets))) # check for overlapping column names common_column_names = set(train_features.columns) & set( train_targets.columns) if common_column_names: raise ValueError( "`train_features` and `train_targets` combined have overlapping column names;" " please ensure column names are unique") train_df = train_features.join(train_targets) histograms = _histogram_utils.calculate_histograms(train_df) response = _utils.make_request("PUT", self._histogram_endpoint, self._conn, json=histograms) _utils.raise_for_http_error(response)
def test_integration(self, experiment_run): np = pytest.importorskip("numpy") pd = pytest.importorskip("pandas") binary_col_name = 'binary col' discrete_col_name = 'discrete col' float_col_name = 'float col' df = pd.concat( objs=[ pd.Series([True] * 10 + [False] * 20, name=binary_col_name), pd.Series([0] * 5 + [1] * 10 + [2] * 15, name=discrete_col_name), pd.Series(range(30), name=float_col_name), ], axis='columns', ) histograms = _histogram_utils.calculate_histograms(df) experiment_run.log_training_data( df[[binary_col_name, discrete_col_name]], df[float_col_name]) endpoint = "{}://{}/api/v1/monitoring/data/references/{}".format( experiment_run._conn.scheme, experiment_run._conn.socket, experiment_run.id, ) response = _utils.make_request("GET", endpoint, experiment_run._conn) _utils.raise_for_http_error(response) retrieved_histograms = response.json() # features match features = histograms['features'] retrieved_features = retrieved_histograms['features'] assert set(features.keys()) == set(retrieved_features.keys()) # binary matches binary_hist = histograms['features'][binary_col_name]['histogram'][ 'binary'] retrieved_binary_hist = retrieved_histograms['features'][ binary_col_name]['histogram']['binary'] assert binary_hist['count'] == retrieved_binary_hist['count'] # discrete matches discrete_hist = histograms['features'][discrete_col_name]['histogram'][ 'discrete'] retrieved_discrete_hist = retrieved_histograms['features'][ discrete_col_name]['histogram']['discrete'] assert discrete_hist['bucket_values'] == retrieved_discrete_hist[ 'bucket_values'] assert discrete_hist['count'] == retrieved_discrete_hist['count'] # float matches float_hist = histograms['features'][float_col_name]['histogram'][ 'float'] retrieved_float_hist = retrieved_histograms['features'][ float_col_name]['histogram']['float'] assert all( np.isclose(float_hist['bucket_limits'], retrieved_float_hist['bucket_limits'])) assert float_hist['count'] == retrieved_float_hist['count']
def _get_build_status(self, build_id): url = "{}://{}/api/v1/deployment/workspace/{}/builds/{}".format( self._conn.scheme, self._conn.socket, self.workspace, build_id) response = _utils.make_request("GET", url, self._conn) _utils.raise_for_http_error(response) return response.json()
def _create_stage(self, name="production"): url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages".format( self._conn.scheme, self._conn.socket, self.workspace, self.id) response = _utils.make_request("POST", url, self._conn, json={"name": name}) _utils.raise_for_http_error(response) return response.json()["id"]
def delete(self): """ Deletes this registered model. """ request_url = "{}://{}/api/v1/registry/registered_models/{}".format( self._conn.scheme, self._conn.socket, self.id) response = _utils.make_request("DELETE", request_url, self._conn) _utils.raise_for_http_error(response)
def delete(self): """ Deletes this endpoint. """ request_url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}".format( self._conn.scheme, self._conn.socket, self.workspace, self.id) response = _utils.make_request("DELETE", request_url, self._conn) _utils.raise_for_http_error(response)
def test_tags_is_list_of_str(self, client, tags): proj = client.set_project(tags=tags) endpoint = "{}://{}/api/v1/modeldb/project/getProjectTags".format( client._conn.scheme, client._conn.socket, ) response = _utils.make_request("GET", endpoint, client._conn, params={'id': proj.id}) _utils.raise_for_http_error(response) assert response.json().get('tags', []) == [TAG]
def _get_artifact(self, key, artifact_type=0): # check to see if key exists self._get_artifact_msg(key) # download artifact from artifact store url = self._get_url_for_artifact(key, "GET", artifact_type).url response = _utils.make_request("GET", url, self._conn) _utils.raise_for_http_error(response) return response.content
def _get_stages(self): url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages".format( self._conn.scheme, self._conn.socket, self.workspace, self.id) response = _utils.make_request("GET", url, self._conn, params={}) _utils.raise_for_http_error(response) response_json = response.json() if "stages" in response_json: return response_json["stages"] else: return []
def _update_from_build(self, update_body, wait=False): # Update stages with new build url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages/{}/update".format( self._conn.scheme, self._conn.socket, self.workspace, self.id, self._get_or_create_stage(), ) response = _utils.make_request("PUT", url, self._conn, json=update_body) _utils.raise_for_http_error(response) if wait: print("waiting for update...", end="") sys.stdout.flush() build_status = self._get_build_status(update_body["build_id"]) # have to check using build status, otherwise might never terminate while build_status["status"] not in ("finished", "error"): print(".", end="") sys.stdout.flush() time.sleep(5) build_status = self._get_build_status(update_body["build_id"]) if build_status["status"] == "error": print() failure_msg = build_status.get("message", "no error message available") raise RuntimeError( "endpoint update failed;\n{}".format(failure_msg)) # still need to wait because status might be "creating" even though build status is "finished" status_dict = self.get_status() while status_dict["status"] not in ("active", "error") or ( status_dict["status"] == "active" and len(status_dict["components"]) > 1): print(".", end="") sys.stdout.flush() time.sleep(5) status_dict = self.get_status() print() if status_dict["status"] == "error": failure_msg = status_dict['components'][-1].get( 'message', "no error message available") # NOTE: we might consider truncating the length of the logs here, # e.g. first and last 25 lines, if too unwieldy raise RuntimeError( "endpoint update failed;\n{}".format(failure_msg)) return self.get_status()
def _get_url_for_artifact(self, dataset_component_path, method, part_num=0): """ Obtains a URL to use for accessing stored artifacts. Parameters ---------- dataset_component_path : str Filepath in dataset component blob. method : {'GET', 'PUT'} HTTP method to request for the generated URL. part_num : int, optional If using Multipart Upload, number of part to be uploaded. Returns ------- response_msg : `_DatasetVersionService.GetUrlForDatasetBlobVersioned.Response` Backend response. """ if method.upper() not in ("GET", "PUT"): raise ValueError("`method` must be one of {'GET', 'PUT'}") Message = _DatasetVersionService.GetUrlForDatasetBlobVersioned msg = Message( path_dataset_component_blob_path=dataset_component_path, method=method, part_number=part_num, ) data = _utils.proto_to_json(msg) endpoint = "{}://{}/api/v1/modeldb/dataset-version/dataset/{}/datasetVersion/{}/getUrlForDatasetBlobVersioned".format( self._conn.scheme, self._conn.socket, self.dataset_id, self.id, ) response = _utils.make_request("POST", endpoint, self._conn, json=data) _utils.raise_for_http_error(response) response_msg = _utils.json_to_proto(response.json(), Message.Response) url = response_msg.url # accommodate port-forwarded NFS store if 'https://localhost' in url[:20]: url = 'http' + url[5:] if 'localhost%3a' in url[:20]: url = url.replace('localhost%3a', 'localhost:') if 'localhost%3A' in url[:20]: url = url.replace('localhost%3A', 'localhost:') response_msg.url = url return response_msg
def delete(self): """ Deletes this experiment. """ request_url = "{}://{}/api/v1/modeldb/experiment/deleteExperiment".format( self._conn.scheme, self._conn.socket) response = _utils.make_request("DELETE", request_url, self._conn, json={'id': self.id}) _utils.raise_for_http_error(response)
def test_tags_is_list_of_str(self, client, tags): client.set_project() client.set_experiment() run = client.set_experiment_run(tags=tags) endpoint = "{}://{}/api/v1/modeldb/experiment-run/getExperimentRunTags".format( client._conn.scheme, client._conn.socket, ) response = _utils.make_request("GET", endpoint, client._conn, params={'id': run.id}) _utils.raise_for_http_error(response) assert response.json().get('tags', []) == [TAG]
def test_get_experiments_in_project(self, client): expt_ids = [] proj = client.set_project() for _ in range(3): expt_ids.append(client.set_experiment().id) response = requests.get( "http://{}/api/v1/modeldb/experiment/getExperimentsInProject". format(client._conn.socket), params={'project_id': proj.id}, headers=client._conn.auth) _utils.raise_for_http_error(response) assert set(expt_ids) == set( experiment['id'] for experiment in response.json()['experiments'])
def download_docker_context(self, download_to_path, self_contained=False): """ Downloads this Model Version's Docker context ``tgz``. Parameters ---------- download_to_path : str Path to download Docker context to. self_contained : bool, default False Whether the downloaded Docker context should be self-contained. Returns ------- downloaded_to_path : str Absolute path where Docker context was downloaded to. Matches `download_to_path`. """ self._refresh_cache() endpoint = "{}://{}/api/v1/deployment/builds/dockercontext".format( self._conn.scheme, self._conn.socket, ) body = { "model_version_id": self.id, "self_contained": self_contained, } with _utils.make_request("POST", endpoint, self._conn, json=body, stream=True) as response: try: _utils.raise_for_http_error(response) except requests.HTTPError as e: # propagate error caused by missing artifact error_text = e.response.text.strip() if error_text.startswith("missing artifact"): new_e = RuntimeError( "unable to obtain Docker context due to " + error_text) six.raise_from(new_e, None) else: raise e downloaded_to_path = _request_utils.download_file( response, download_to_path, overwrite_ok=True) return os.path.abspath(downloaded_to_path)
def _del_kafka_settings(self, stage_id): if not isinstance(stage_id, six.integer_types): raise TypeError("`stage_id` must be int, not {}".format( type(stage_id))) url = "{}://{}/api/v1/deployment/stages/{}/kafka".format( self._conn.scheme, self._conn.socket, stage_id, ) response = _utils.make_request( "PUT", url, self._conn, json={"disabled": True}, ) _utils.raise_for_http_error(response)
def _get_or_create_stage(self, name="production"): if name == "production": # Check if a stage exists: url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages".format( self._conn.scheme, self._conn.socket, self.workspace, self.id) response = _utils.make_request("GET", url, self._conn, params={}) _utils.raise_for_http_error(response) response_json = response.json() if response_json["stages"]: return response_json["stages"][0]["id"] # no stage found: return self._create_stage("production") else: raise NotImplementedError("currently not supported other stages")
def _get_url_for_artifact(self, key, method, artifact_type=0, part_num=0): if method.upper() not in ("GET", "PUT"): raise ValueError("`method` must be one of {'GET', 'PUT'}") Message = _RegistryService.GetUrlForArtifact msg = Message( model_version_id=self.id, key=key, method=method, artifact_type=artifact_type, part_number=part_num, ) data = _utils.proto_to_json(msg) endpoint = "{}://{}/api/v1/registry/model_versions/{}/getUrlForArtifact".format( self._conn.scheme, self._conn.socket, self.id) response = _utils.make_request("POST", endpoint, self._conn, json=data) _utils.raise_for_http_error(response) return _utils.json_to_proto(response.json(), Message.Response)
def log_ground_truth(self, id, labels, colname): conn = self._conn url = "{}://{}/api/v1/monitoring/ingest/decorate/batch/endpoint/{}".format( conn.scheme, conn.socket, self.id) if len(labels) == 0: # TODO: raise an error pass data = [{ "id": "{}[{}]".format(id, i), "attributes": { "ground_truth.output." + colname: val, "source": "ground_truth", } } for i, val in enumerate(labels)] response = _utils.make_request("POST", url, conn, json=data) _utils.raise_for_http_error(response)
def get_update_status(self): """ Gets update status on the endpoint. Returns ------- update_status : dict of str to {None, bool, float, int, str, list, dict} """ url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages/{}/status".format( self._conn.scheme, self._conn.socket, self.workspace, self.id, self._get_or_create_stage(), ) response = _utils.make_request("GET", url, self._conn) _utils.raise_for_http_error(response) return response.json()
def _set_kafka_settings(self, stage_id, kafka_settings): if not isinstance(stage_id, six.integer_types): raise TypeError("`stage_id` must be int, not {}".format( type(stage_id))) if not isinstance(kafka_settings, KafkaSettings): raise TypeError( "`kafka_settings` must be of type verta.endpoint.KafkaSettings," " not {}".format(type(kafka_settings))) url = "{}://{}/api/v1/deployment/stages/{}/kafka".format( self._conn.scheme, self._conn.socket, stage_id, ) response = _utils.make_request("PUT", url, self._conn, json=kafka_settings._as_dict()) _utils.raise_for_http_error(response)