def _create_json(
cls,
conn,
workspace,
path,
description=None,
public_within_org=None,
visibility=None,
):
path = arg_handler.ensure_starts_with_slash(path)
(
visibility,
public_within_org,
) = _visibility._Visibility._translate_public_within_org(
visibility, public_within_org)
data = {
"path": path,
"description": description,
"custom_permission": {
"collaborator_type": visibility._collaborator_type_str
},
"visibility": "ORG_SCOPED_PUBLIC" if public_within_org else
"PRIVATE", # TODO: raise if workspace is personal
"resource_visibility": visibility._visibility_str,
}
url = "{}://{}/api/v1/deployment/workspace/{}/endpoints".format(
conn.scheme, conn.socket, workspace)
response = _utils.make_request("POST", url, conn, json=data)
_utils.raise_for_http_error(response)
return response.json()
def delete_datasets(ids, conn):
request_url = "{}://{}/api/v1/modeldb/dataset/deleteDatasets".format(
conn.scheme, conn.socket)
response = requests.delete(request_url,
json={'ids': ids},
headers=conn.auth)
_utils.raise_for_http_error(response)
def create_access_token(self, token):
"""
Creates an access token for the endpoint.
Parameters
----------
token : str
Token to create.
"""
if not isinstance(token, six.string_types):
raise TypeError("`token` must be a string.")
url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages/{}/accesstokens".format(
self._conn.scheme,
self._conn.socket,
self.workspace,
self.id,
self._get_or_create_stage(),
)
response = _utils.make_request("POST",
url,
self._conn,
json={"value": token})
_utils.raise_for_http_error(response)
def delete_project(id_, conn):
request_url = "{}://{}/api/v1/modeldb/project/deleteProject".format(
conn.scheme, conn.socket)
response = requests.delete(request_url,
json={'id': id_},
headers=conn.auth)
_utils.raise_for_http_error(response)
def test_no_api_deploy_error(self, experiment_run, model_for_deployment):
experiment_run.log_model(model_for_deployment['model'],
custom_modules=[])
experiment_run.log_requirements(['scikit-learn'])
# delete model API
response = _utils.make_request(
"DELETE",
"{}://{}/api/v1/modeldb/experiment-run/deleteArtifact".format(
experiment_run._conn.scheme, experiment_run._conn.socket),
experiment_run._conn,
json={
'id': experiment_run.id,
'key': "model_api.json"
})
_utils.raise_for_http_error(response)
with pytest.raises(RuntimeError) as excinfo:
experiment_run.deploy()
assert str(excinfo.value).strip(
) == "unable to deploy due to missing artifact model_api.json"
conn = experiment_run._conn
requests.delete(
"{}://{}/api/v1/deployment/models/{}".format(
conn.scheme, conn.socket, experiment_run.id),
headers=conn.auth,
)
def delete_experiment_run(id_, conn):
request_url = "{}://{}/api/v1/modeldb/experiment-run/deleteExperimentRun".format(
conn.scheme, conn.socket)
response = requests.delete(request_url,
json={'id': id_},
headers=conn.auth)
_utils.raise_for_http_error(response)
def test_no_api_deploy_error(self, experiment_run, model_for_deployment):
experiment_run.log_model(model_for_deployment['model'],
custom_modules=[])
experiment_run.log_environment(Python(['scikit-learn']))
# delete model API
response = _utils.make_request(
"DELETE",
"{}://{}/api/v1/modeldb/experiment-run/deleteArtifact".format(
experiment_run._conn.scheme, experiment_run._conn.socket),
experiment_run._conn,
json={
'id': experiment_run.id,
'key': _artifact_utils.MODEL_API_KEY
})
_utils.raise_for_http_error(response)
with pytest.raises(RuntimeError) as excinfo:
experiment_run.deploy()
assert _artifact_utils.MODEL_API_KEY in str(excinfo.value)
conn = experiment_run._conn
requests.delete(
"{}://{}/api/v1/deployment/models/{}".format(
conn.scheme, conn.socket, experiment_run.id),
headers=conn.auth,
)
def _create_build(self, model_reference):
url = "{}://{}/api/v1/deployment/workspace/{}/builds".format(
self._conn.scheme,
self._conn.socket,
self.workspace,
)
if isinstance(model_reference, RegisteredModelVersion):
response = _utils.make_request(
"POST",
url,
self._conn,
json={"model_version_id": model_reference.id})
elif isinstance(model_reference, ExperimentRun):
response = _utils.make_request("POST",
url,
self._conn,
json={"run_id": model_reference.id})
else:
raise TypeError(
"`model_reference` must be an ExperimentRun or RegisteredModelVersion"
)
_utils.raise_for_http_error(response)
return response.json()["id"]
def _get_endpoints(cls, conn, workspace):
url = "{}://{}/api/v1/deployment/workspace/{}/endpoints".format(
conn.scheme, conn.socket, workspace)
response = _utils.make_request("GET", url, conn)
_utils.raise_for_http_error(response)
data_response = response.json()
return data_response["endpoints"]
def log_training_data(self,
train_features,
train_targets,
overwrite=False):
"""
Associate training data with this model reference.
.. versionchanged:: 0.14.4
Instead of uploading the data itself as a CSV artifact ``'train_data'``, this method now
generates a histogram for internal use by our deployment data monitoring system.
.. deprecated:: 0.18.0
This method is no longer supported. Please see our documentation
for information about our platform's data monitoring features.
Parameters
----------
train_features : pd.DataFrame
pandas DataFrame representing features of the training data.
train_targets : pd.DataFrame or pd.Series
pandas DataFrame representing targets of the training data.
overwrite : bool, default False
Whether to allow overwriting existing training data.
"""
warnings.warn(
"This method is no longer supported. Please see our documentation"
" for information about our platform's data monitoring features",
category=FutureWarning,
)
if train_features.__class__.__name__ != "DataFrame":
raise TypeError(
"`train_features` must be a pandas DataFrame, not {}".format(
type(train_features)))
if train_targets.__class__.__name__ == "Series":
train_targets = train_targets.to_frame()
elif train_targets.__class__.__name__ != "DataFrame":
raise TypeError(
"`train_targets` must be a pandas DataFrame or Series, not {}".
format(type(train_targets)))
# check for overlapping column names
common_column_names = set(train_features.columns) & set(
train_targets.columns)
if common_column_names:
raise ValueError(
"`train_features` and `train_targets` combined have overlapping column names;"
" please ensure column names are unique")
train_df = train_features.join(train_targets)
histograms = _histogram_utils.calculate_histograms(train_df)
response = _utils.make_request("PUT",
self._histogram_endpoint,
self._conn,
json=histograms)
_utils.raise_for_http_error(response)
def test_integration(self, experiment_run):
np = pytest.importorskip("numpy")
pd = pytest.importorskip("pandas")
binary_col_name = 'binary col'
discrete_col_name = 'discrete col'
float_col_name = 'float col'
df = pd.concat(
objs=[
pd.Series([True] * 10 + [False] * 20, name=binary_col_name),
pd.Series([0] * 5 + [1] * 10 + [2] * 15,
name=discrete_col_name),
pd.Series(range(30), name=float_col_name),
],
axis='columns',
)
histograms = _histogram_utils.calculate_histograms(df)
experiment_run.log_training_data(
df[[binary_col_name, discrete_col_name]], df[float_col_name])
endpoint = "{}://{}/api/v1/monitoring/data/references/{}".format(
experiment_run._conn.scheme,
experiment_run._conn.socket,
experiment_run.id,
)
response = _utils.make_request("GET", endpoint, experiment_run._conn)
_utils.raise_for_http_error(response)
retrieved_histograms = response.json()
# features match
features = histograms['features']
retrieved_features = retrieved_histograms['features']
assert set(features.keys()) == set(retrieved_features.keys())
# binary matches
binary_hist = histograms['features'][binary_col_name]['histogram'][
'binary']
retrieved_binary_hist = retrieved_histograms['features'][
binary_col_name]['histogram']['binary']
assert binary_hist['count'] == retrieved_binary_hist['count']
# discrete matches
discrete_hist = histograms['features'][discrete_col_name]['histogram'][
'discrete']
retrieved_discrete_hist = retrieved_histograms['features'][
discrete_col_name]['histogram']['discrete']
assert discrete_hist['bucket_values'] == retrieved_discrete_hist[
'bucket_values']
assert discrete_hist['count'] == retrieved_discrete_hist['count']
# float matches
float_hist = histograms['features'][float_col_name]['histogram'][
'float']
retrieved_float_hist = retrieved_histograms['features'][
float_col_name]['histogram']['float']
assert all(
np.isclose(float_hist['bucket_limits'],
retrieved_float_hist['bucket_limits']))
assert float_hist['count'] == retrieved_float_hist['count']
def _get_build_status(self, build_id):
url = "{}://{}/api/v1/deployment/workspace/{}/builds/{}".format(
self._conn.scheme, self._conn.socket, self.workspace, build_id)
response = _utils.make_request("GET", url, self._conn)
_utils.raise_for_http_error(response)
return response.json()
def _create_stage(self, name="production"):
url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages".format(
self._conn.scheme, self._conn.socket, self.workspace, self.id)
response = _utils.make_request("POST",
url,
self._conn,
json={"name": name})
_utils.raise_for_http_error(response)
return response.json()["id"]
def delete(self):
"""
Deletes this registered model.
"""
request_url = "{}://{}/api/v1/registry/registered_models/{}".format(
self._conn.scheme, self._conn.socket, self.id)
response = _utils.make_request("DELETE", request_url, self._conn)
_utils.raise_for_http_error(response)
def delete(self):
"""
Deletes this endpoint.
"""
request_url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}".format(
self._conn.scheme, self._conn.socket, self.workspace, self.id)
response = _utils.make_request("DELETE", request_url, self._conn)
_utils.raise_for_http_error(response)
def test_tags_is_list_of_str(self, client, tags):
proj = client.set_project(tags=tags)
endpoint = "{}://{}/api/v1/modeldb/project/getProjectTags".format(
client._conn.scheme,
client._conn.socket,
)
response = _utils.make_request("GET", endpoint, client._conn, params={'id': proj.id})
_utils.raise_for_http_error(response)
assert response.json().get('tags', []) == [TAG]
def _get_artifact(self, key, artifact_type=0):
# check to see if key exists
self._get_artifact_msg(key)
# download artifact from artifact store
url = self._get_url_for_artifact(key, "GET", artifact_type).url
response = _utils.make_request("GET", url, self._conn)
_utils.raise_for_http_error(response)
return response.content
def _get_stages(self):
url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages".format(
self._conn.scheme, self._conn.socket, self.workspace, self.id)
response = _utils.make_request("GET", url, self._conn, params={})
_utils.raise_for_http_error(response)
response_json = response.json()
if "stages" in response_json:
return response_json["stages"]
else:
return []
def _update_from_build(self, update_body, wait=False):
# Update stages with new build
url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages/{}/update".format(
self._conn.scheme,
self._conn.socket,
self.workspace,
self.id,
self._get_or_create_stage(),
)
response = _utils.make_request("PUT",
url,
self._conn,
json=update_body)
_utils.raise_for_http_error(response)
if wait:
print("waiting for update...", end="")
sys.stdout.flush()
build_status = self._get_build_status(update_body["build_id"])
# have to check using build status, otherwise might never terminate
while build_status["status"] not in ("finished", "error"):
print(".", end="")
sys.stdout.flush()
time.sleep(5)
build_status = self._get_build_status(update_body["build_id"])
if build_status["status"] == "error":
print()
failure_msg = build_status.get("message",
"no error message available")
raise RuntimeError(
"endpoint update failed;\n{}".format(failure_msg))
# still need to wait because status might be "creating" even though build status is "finished"
status_dict = self.get_status()
while status_dict["status"] not in ("active", "error") or (
status_dict["status"] == "active"
and len(status_dict["components"]) > 1):
print(".", end="")
sys.stdout.flush()
time.sleep(5)
status_dict = self.get_status()
print()
if status_dict["status"] == "error":
failure_msg = status_dict['components'][-1].get(
'message', "no error message available")
# NOTE: we might consider truncating the length of the logs here,
# e.g. first and last 25 lines, if too unwieldy
raise RuntimeError(
"endpoint update failed;\n{}".format(failure_msg))
return self.get_status()
def _get_url_for_artifact(self,
dataset_component_path,
method,
part_num=0):
"""
Obtains a URL to use for accessing stored artifacts.
Parameters
----------
dataset_component_path : str
Filepath in dataset component blob.
method : {'GET', 'PUT'}
HTTP method to request for the generated URL.
part_num : int, optional
If using Multipart Upload, number of part to be uploaded.
Returns
-------
response_msg : `_DatasetVersionService.GetUrlForDatasetBlobVersioned.Response`
Backend response.
"""
if method.upper() not in ("GET", "PUT"):
raise ValueError("`method` must be one of {'GET', 'PUT'}")
Message = _DatasetVersionService.GetUrlForDatasetBlobVersioned
msg = Message(
path_dataset_component_blob_path=dataset_component_path,
method=method,
part_number=part_num,
)
data = _utils.proto_to_json(msg)
endpoint = "{}://{}/api/v1/modeldb/dataset-version/dataset/{}/datasetVersion/{}/getUrlForDatasetBlobVersioned".format(
self._conn.scheme,
self._conn.socket,
self.dataset_id,
self.id,
)
response = _utils.make_request("POST", endpoint, self._conn, json=data)
_utils.raise_for_http_error(response)
response_msg = _utils.json_to_proto(response.json(), Message.Response)
url = response_msg.url
# accommodate port-forwarded NFS store
if 'https://localhost' in url[:20]:
url = 'http' + url[5:]
if 'localhost%3a' in url[:20]:
url = url.replace('localhost%3a', 'localhost:')
if 'localhost%3A' in url[:20]:
url = url.replace('localhost%3A', 'localhost:')
response_msg.url = url
return response_msg
def delete(self):
"""
Deletes this experiment.
"""
request_url = "{}://{}/api/v1/modeldb/experiment/deleteExperiment".format(
self._conn.scheme, self._conn.socket)
response = _utils.make_request("DELETE",
request_url,
self._conn,
json={'id': self.id})
_utils.raise_for_http_error(response)
def test_tags_is_list_of_str(self, client, tags):
client.set_project()
client.set_experiment()
run = client.set_experiment_run(tags=tags)
endpoint = "{}://{}/api/v1/modeldb/experiment-run/getExperimentRunTags".format(
client._conn.scheme,
client._conn.socket,
)
response = _utils.make_request("GET", endpoint, client._conn, params={'id': run.id})
_utils.raise_for_http_error(response)
assert response.json().get('tags', []) == [TAG]
def test_get_experiments_in_project(self, client):
expt_ids = []
proj = client.set_project()
for _ in range(3):
expt_ids.append(client.set_experiment().id)
response = requests.get(
"http://{}/api/v1/modeldb/experiment/getExperimentsInProject".
format(client._conn.socket),
params={'project_id': proj.id},
headers=client._conn.auth)
_utils.raise_for_http_error(response)
assert set(expt_ids) == set(
experiment['id'] for experiment in response.json()['experiments'])
def download_docker_context(self, download_to_path, self_contained=False):
"""
Downloads this Model Version's Docker context ``tgz``.
Parameters
----------
download_to_path : str
Path to download Docker context to.
self_contained : bool, default False
Whether the downloaded Docker context should be self-contained.
Returns
-------
downloaded_to_path : str
Absolute path where Docker context was downloaded to. Matches `download_to_path`.
"""
self._refresh_cache()
endpoint = "{}://{}/api/v1/deployment/builds/dockercontext".format(
self._conn.scheme,
self._conn.socket,
)
body = {
"model_version_id": self.id,
"self_contained": self_contained,
}
with _utils.make_request("POST",
endpoint,
self._conn,
json=body,
stream=True) as response:
try:
_utils.raise_for_http_error(response)
except requests.HTTPError as e:
# propagate error caused by missing artifact
error_text = e.response.text.strip()
if error_text.startswith("missing artifact"):
new_e = RuntimeError(
"unable to obtain Docker context due to " + error_text)
six.raise_from(new_e, None)
else:
raise e
downloaded_to_path = _request_utils.download_file(
response, download_to_path, overwrite_ok=True)
return os.path.abspath(downloaded_to_path)
def _del_kafka_settings(self, stage_id):
if not isinstance(stage_id, six.integer_types):
raise TypeError("`stage_id` must be int, not {}".format(
type(stage_id)))
url = "{}://{}/api/v1/deployment/stages/{}/kafka".format(
self._conn.scheme,
self._conn.socket,
stage_id,
)
response = _utils.make_request(
"PUT",
url,
self._conn,
json={"disabled": True},
)
_utils.raise_for_http_error(response)
def _get_or_create_stage(self, name="production"):
if name == "production":
# Check if a stage exists:
url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages".format(
self._conn.scheme, self._conn.socket, self.workspace, self.id)
response = _utils.make_request("GET", url, self._conn, params={})
_utils.raise_for_http_error(response)
response_json = response.json()
if response_json["stages"]:
return response_json["stages"][0]["id"]
# no stage found:
return self._create_stage("production")
else:
raise NotImplementedError("currently not supported other stages")
def _get_url_for_artifact(self, key, method, artifact_type=0, part_num=0):
if method.upper() not in ("GET", "PUT"):
raise ValueError("`method` must be one of {'GET', 'PUT'}")
Message = _RegistryService.GetUrlForArtifact
msg = Message(
model_version_id=self.id,
key=key,
method=method,
artifact_type=artifact_type,
part_number=part_num,
)
data = _utils.proto_to_json(msg)
endpoint = "{}://{}/api/v1/registry/model_versions/{}/getUrlForArtifact".format(
self._conn.scheme, self._conn.socket, self.id)
response = _utils.make_request("POST", endpoint, self._conn, json=data)
_utils.raise_for_http_error(response)
return _utils.json_to_proto(response.json(), Message.Response)
def log_ground_truth(self, id, labels, colname):
conn = self._conn
url = "{}://{}/api/v1/monitoring/ingest/decorate/batch/endpoint/{}".format(
conn.scheme, conn.socket, self.id)
if len(labels) == 0:
# TODO: raise an error
pass
data = [{
"id": "{}[{}]".format(id, i),
"attributes": {
"ground_truth.output." + colname: val,
"source": "ground_truth",
}
} for i, val in enumerate(labels)]
response = _utils.make_request("POST", url, conn, json=data)
_utils.raise_for_http_error(response)
def get_update_status(self):
"""
Gets update status on the endpoint.
Returns
-------
update_status : dict of str to {None, bool, float, int, str, list, dict}
"""
url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages/{}/status".format(
self._conn.scheme,
self._conn.socket,
self.workspace,
self.id,
self._get_or_create_stage(),
)
response = _utils.make_request("GET", url, self._conn)
_utils.raise_for_http_error(response)
return response.json()
def _set_kafka_settings(self, stage_id, kafka_settings):
if not isinstance(stage_id, six.integer_types):
raise TypeError("`stage_id` must be int, not {}".format(
type(stage_id)))
if not isinstance(kafka_settings, KafkaSettings):
raise TypeError(
"`kafka_settings` must be of type verta.endpoint.KafkaSettings,"
" not {}".format(type(kafka_settings)))
url = "{}://{}/api/v1/deployment/stages/{}/kafka".format(
self._conn.scheme,
self._conn.socket,
stage_id,
)
response = _utils.make_request("PUT",
url,
self._conn,
json=kafka_settings._as_dict())
_utils.raise_for_http_error(response)
