def _create_build(self, model_reference):
url = "{}://{}/api/v1/deployment/workspace/{}/builds".format(
self._conn.scheme,
self._conn.socket,
self.workspace,
)
if isinstance(model_reference, RegisteredModelVersion):
response = _utils.make_request(
"POST",
url,
self._conn,
json={"model_version_id": model_reference.id})
elif isinstance(model_reference, ExperimentRun):
response = _utils.make_request("POST",
url,
self._conn,
json={"run_id": model_reference.id})
else:
raise TypeError(
"`model_reference` must be an ExperimentRun or RegisteredModelVersion"
)
_utils.raise_for_http_error(response)
return response.json()["id"]
def test_no_api_deploy_error(self, experiment_run, model_for_deployment):
experiment_run.log_model(model_for_deployment['model'],
custom_modules=[])
experiment_run.log_requirements(['scikit-learn'])
# delete model API
_utils.make_request(
"DELETE",
"{}://{}/api/v1/modeldb/experiment-run/deleteArtifact".format(
experiment_run._conn.scheme, experiment_run._conn.socket),
experiment_run._conn,
json={
'id': experiment_run.id,
'key': "model_api.json"
}).raise_for_status()
with pytest.raises(RuntimeError) as excinfo:
experiment_run.deploy()
assert str(excinfo.value).strip(
) == "unable to deploy due to missing artifact model_api.json"
conn = experiment_run._conn
requests.delete(
"{}://{}/api/v1/deployment/models/{}".format(
conn.scheme, conn.socket, experiment_run.id),
headers=conn.auth,
)
def test_ca_bundle_env_var(self, https_client):
"""Verify make_request() honors REQUESTS_CA_BUNDLE env var."""
REQUESTS_CA_BUNDLE_ENV_VAR = "REQUESTS_CA_BUNDLE"
good_ca_bundle_path = os.environ.get(REQUESTS_CA_BUNDLE_ENV_VAR)
conn = https_client._conn
url = "{}://{}/".format(conn.scheme, conn.socket)
# as a control, make sure request works
response = _utils.make_request("GET", url, conn)
conn.must_response(response)
bad_ca_bundle_path = "foo"
msg_match = ("^Could not find a suitable TLS CA certificate bundle,"
" invalid path: {}$".format(bad_ca_bundle_path))
error_type = IOError if six.PY2 else OSError
try:
os.environ[REQUESTS_CA_BUNDLE_ENV_VAR] = bad_ca_bundle_path
with pytest.raises(error_type, match=msg_match):
_utils.make_request("GET", url, https_client._conn)
finally:
if good_ca_bundle_path:
os.environ[REQUESTS_CA_BUNDLE_ENV_VAR] = good_ca_bundle_path
else:
del os.environ[REQUESTS_CA_BUNDLE_ENV_VAR]
def test_302_stop(self, client):
with pytest.raises(RuntimeError) as excinfo:
_utils.make_request(
"GET",
"http://httpbin.org/redirect-to",
client._conn,
params={
'url': "http://httpbin.org/get",
'status_code': 302,
},
)
assert str(excinfo.value).strip().startswith("received status 302")
def get_logs(self):
"""Get runtime logs of this endpoint.
Returns
-------
list of str
Lines of this endpoint's runtime logs.
"""
url = (
"{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages/{}/logs"
.format(
self._conn.scheme,
self._conn.socket,
self.workspace,
self.id,
self._get_or_create_stage(),
))
response = _utils.make_request("GET", url, self._conn)
self._conn.must_response(response)
logs = response.json().get("entries", [])
# remove duplicate lines
logs = {log.get("id", ""): log.get("message", "") for log in logs}
# sort by line number
logs = sorted(logs.items(), key=lambda item: item[0])
return [item[1] for item in logs]
def create_access_token(self, token):
"""
Creates an access token for the endpoint.
Parameters
----------
token : str
Token to create.
"""
if not isinstance(token, six.string_types):
raise TypeError("`token` must be a string.")
url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages/{}/accesstokens".format(
self._conn.scheme,
self._conn.socket,
self.workspace,
self.id,
self._get_or_create_stage(),
)
response = _utils.make_request("POST",
url,
self._conn,
json={"value": token})
_utils.raise_for_http_error(response)
def _update(self, msg, method="PATCH", update_mask=None):
self._refresh_cache(
) # to have `self._msg.registered_model_id` for URL
if update_mask:
url = "{}://{}/api/v1/registry/registered_models/{}/model_versions/{}/full_body".format(
self._conn.scheme,
self._conn.socket,
self._msg.registered_model_id,
self.id,
)
# proto converter for update_mask is missing
data = {
"model_version": _utils.proto_to_json(msg, False),
"update_mask": update_mask,
}
response = _utils.make_request(method, url, self._conn, json=data)
else:
response = self._conn.make_proto_request(
method,
"/api/v1/registry/registered_models/{}/model_versions/{}".
format(self._msg.registered_model_id, self.id),
body=msg,
include_default=False,
)
self._conn.must_proto_response(
response, _RegistryService.SetModelVersion.Response)
self._clear_cache()
def test_no_model_deploy_error(self, experiment_run, model_for_deployment):
experiment_run.log_model(model_for_deployment['model'],
custom_modules=[])
experiment_run.log_requirements(['scikit-learn'])
# delete model
response = _utils.make_request(
"DELETE",
"{}://{}/api/v1/modeldb/experiment-run/deleteArtifact".format(
experiment_run._conn.scheme, experiment_run._conn.socket),
experiment_run._conn,
json={
'id': experiment_run.id,
'key': "model.pkl"
})
_utils.raise_for_http_error(response)
with pytest.raises(RuntimeError) as excinfo:
experiment_run.deploy()
assert "model.pkl" in str(excinfo.value)
conn = experiment_run._conn
requests.delete(
"{}://{}/api/v1/deployment/models/{}".format(
conn.scheme, conn.socket, experiment_run.id),
headers=conn.auth,
)
def _get_endpoints(cls, conn, workspace):
url = "{}://{}/api/v1/deployment/workspace/{}/endpoints".format(
conn.scheme, conn.socket, workspace)
response = _utils.make_request("GET", url, conn)
_utils.raise_for_http_error(response)
data_response = response.json()
return data_response["endpoints"]
def _create_json(
cls,
conn,
workspace,
path,
description=None,
public_within_org=None,
visibility=None,
):
path = arg_handler.ensure_starts_with_slash(path)
(
visibility,
public_within_org,
) = _visibility._Visibility._translate_public_within_org(
visibility, public_within_org)
data = {
"path": path,
"description": description,
"custom_permission": {
"collaborator_type": visibility._collaborator_type_str
},
"visibility": "ORG_SCOPED_PUBLIC" if public_within_org else
"PRIVATE", # TODO: raise if workspace is personal
"resource_visibility": visibility._visibility_str,
}
url = "{}://{}/api/v1/deployment/workspace/{}/endpoints".format(
conn.scheme, conn.socket, workspace)
response = _utils.make_request("POST", url, conn, json=data)
_utils.raise_for_http_error(response)
return response.json()
def log_training_data(self,
train_features,
train_targets,
overwrite=False):
"""
Associate training data with this model reference.
.. versionchanged:: 0.14.4
Instead of uploading the data itself as a CSV artifact ``'train_data'``, this method now
generates a histogram for internal use by our deployment data monitoring system.
.. deprecated:: 0.18.0
This method is no longer supported. Please see our documentation
for information about our platform's data monitoring features.
Parameters
----------
train_features : pd.DataFrame
pandas DataFrame representing features of the training data.
train_targets : pd.DataFrame or pd.Series
pandas DataFrame representing targets of the training data.
overwrite : bool, default False
Whether to allow overwriting existing training data.
"""
warnings.warn(
"This method is no longer supported. Please see our documentation"
" for information about our platform's data monitoring features",
category=FutureWarning,
)
if train_features.__class__.__name__ != "DataFrame":
raise TypeError(
"`train_features` must be a pandas DataFrame, not {}".format(
type(train_features)))
if train_targets.__class__.__name__ == "Series":
train_targets = train_targets.to_frame()
elif train_targets.__class__.__name__ != "DataFrame":
raise TypeError(
"`train_targets` must be a pandas DataFrame or Series, not {}".
format(type(train_targets)))
# check for overlapping column names
common_column_names = set(train_features.columns) & set(
train_targets.columns)
if common_column_names:
raise ValueError(
"`train_features` and `train_targets` combined have overlapping column names;"
" please ensure column names are unique")
train_df = train_features.join(train_targets)
histograms = _histogram_utils.calculate_histograms(train_df)
response = _utils.make_request("PUT",
self._histogram_endpoint,
self._conn,
json=histograms)
_utils.raise_for_http_error(response)
def test_integration(self, experiment_run):
np = pytest.importorskip("numpy")
pd = pytest.importorskip("pandas")
binary_col_name = 'binary col'
discrete_col_name = 'discrete col'
float_col_name = 'float col'
df = pd.concat(
objs=[
pd.Series([True] * 10 + [False] * 20, name=binary_col_name),
pd.Series([0] * 5 + [1] * 10 + [2] * 15,
name=discrete_col_name),
pd.Series(range(30), name=float_col_name),
],
axis='columns',
)
histograms = _histogram_utils.calculate_histograms(df)
experiment_run.log_training_data(
df[[binary_col_name, discrete_col_name]], df[float_col_name])
endpoint = "{}://{}/api/v1/monitoring/data/references/{}".format(
experiment_run._conn.scheme,
experiment_run._conn.socket,
experiment_run.id,
)
response = _utils.make_request("GET", endpoint, experiment_run._conn)
_utils.raise_for_http_error(response)
retrieved_histograms = response.json()
# features match
features = histograms['features']
retrieved_features = retrieved_histograms['features']
assert set(features.keys()) == set(retrieved_features.keys())
# binary matches
binary_hist = histograms['features'][binary_col_name]['histogram'][
'binary']
retrieved_binary_hist = retrieved_histograms['features'][
binary_col_name]['histogram']['binary']
assert binary_hist['count'] == retrieved_binary_hist['count']
# discrete matches
discrete_hist = histograms['features'][discrete_col_name]['histogram'][
'discrete']
retrieved_discrete_hist = retrieved_histograms['features'][
discrete_col_name]['histogram']['discrete']
assert discrete_hist['bucket_values'] == retrieved_discrete_hist[
'bucket_values']
assert discrete_hist['count'] == retrieved_discrete_hist['count']
# float matches
float_hist = histograms['features'][float_col_name]['histogram'][
'float']
retrieved_float_hist = retrieved_histograms['features'][
float_col_name]['histogram']['float']
assert all(
np.isclose(float_hist['bucket_limits'],
retrieved_float_hist['bucket_limits']))
assert float_hist['count'] == retrieved_float_hist['count']
def _get_build_status(self, build_id):
url = "{}://{}/api/v1/deployment/workspace/{}/builds/{}".format(
self._conn.scheme, self._conn.socket, self.workspace, build_id)
response = _utils.make_request("GET", url, self._conn)
_utils.raise_for_http_error(response)
return response.json()
def test_json_response(self, client):
response = _utils.make_request(
"GET",
"http://httpbin.org/json",
client._conn,
)
assert isinstance(_utils.body_to_json(response), dict)
def delete(self):
"""
Deletes this registered model.
"""
request_url = "{}://{}/api/v1/registry/registered_models/{}".format(
self._conn.scheme, self._conn.socket, self.id)
response = _utils.make_request("DELETE", request_url, self._conn)
_utils.raise_for_http_error(response)
def delete(self):
"""
Deletes this endpoint.
"""
request_url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}".format(
self._conn.scheme, self._conn.socket, self.workspace, self.id)
response = _utils.make_request("DELETE", request_url, self._conn)
_utils.raise_for_http_error(response)
def _create_stage(self, name="production"):
url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages".format(
self._conn.scheme, self._conn.socket, self.workspace, self.id)
response = _utils.make_request("POST",
url,
self._conn,
json={"name": name})
_utils.raise_for_http_error(response)
return response.json()["id"]
def test_extra_auth_headers_in_request(self, strs):
headers = dict(zip(strs[:len(strs) // 2], strs[len(strs) // 2:]))
client = verta.Client(extra_auth_headers=headers)
url = "http://httpbin.org/anything"
response = _utils.make_request("GET", url, client._conn)
for key, val in headers.items():
assert key in response.request.headers
assert val == response.request.headers[key]
def test_tags_is_list_of_str(self, client, tags):
proj = client.set_project(tags=tags)
endpoint = "{}://{}/api/v1/modeldb/project/getProjectTags".format(
client._conn.scheme,
client._conn.socket,
)
response = _utils.make_request("GET", endpoint, client._conn, params={'id': proj.id})
_utils.raise_for_http_error(response)
assert response.json().get('tags', []) == [TAG]
def _get_url_for_artifact(self,
dataset_component_path,
method,
part_num=0):
"""
Obtains a URL to use for accessing stored artifacts.
Parameters
----------
dataset_component_path : str
Filepath in dataset component blob.
method : {'GET', 'PUT'}
HTTP method to request for the generated URL.
part_num : int, optional
If using Multipart Upload, number of part to be uploaded.
Returns
-------
response_msg : `_DatasetVersionService.GetUrlForDatasetBlobVersioned.Response`
Backend response.
"""
if method.upper() not in ("GET", "PUT"):
raise ValueError("`method` must be one of {'GET', 'PUT'}")
Message = _DatasetVersionService.GetUrlForDatasetBlobVersioned
msg = Message(
path_dataset_component_blob_path=dataset_component_path,
method=method,
part_number=part_num,
)
data = _utils.proto_to_json(msg)
endpoint = "{}://{}/api/v1/modeldb/dataset-version/dataset/{}/datasetVersion/{}/getUrlForDatasetBlobVersioned".format(
self._conn.scheme,
self._conn.socket,
self.dataset_id,
self.id,
)
response = _utils.make_request("POST", endpoint, self._conn, json=data)
_utils.raise_for_http_error(response)
response_msg = _utils.json_to_proto(response.json(), Message.Response)
url = response_msg.url
# accommodate port-forwarded NFS store
if 'https://localhost' in url[:20]:
url = 'http' + url[5:]
if 'localhost%3a' in url[:20]:
url = url.replace('localhost%3a', 'localhost:')
if 'localhost%3A' in url[:20]:
url = url.replace('localhost%3A', 'localhost:')
response_msg.url = url
return response_msg
def _get_stages(self):
url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages".format(
self._conn.scheme, self._conn.socket, self.workspace, self.id)
response = _utils.make_request("GET", url, self._conn, params={})
_utils.raise_for_http_error(response)
response_json = response.json()
if "stages" in response_json:
return response_json["stages"]
else:
return []
def _get_artifact(self, key, artifact_type=0):
# check to see if key exists
self._get_artifact_msg(key)
# download artifact from artifact store
url = self._get_url_for_artifact(key, "GET", artifact_type).url
response = _utils.make_request("GET", url, self._conn)
_utils.raise_for_http_error(response)
return response.content
def _update_from_build(self, update_body, wait=False):
# Update stages with new build
url = "{}://{}/api/v1/deployment/workspace/{}/endpoints/{}/stages/{}/update".format(
self._conn.scheme,
self._conn.socket,
self.workspace,
self.id,
self._get_or_create_stage(),
)
response = _utils.make_request("PUT",
url,
self._conn,
json=update_body)
_utils.raise_for_http_error(response)
if wait:
print("waiting for update...", end="")
sys.stdout.flush()
build_status = self._get_build_status(update_body["build_id"])
# have to check using build status, otherwise might never terminate
while build_status["status"] not in ("finished", "error"):
print(".", end="")
sys.stdout.flush()
time.sleep(5)
build_status = self._get_build_status(update_body["build_id"])
if build_status["status"] == "error":
print()
failure_msg = build_status.get("message",
"no error message available")
raise RuntimeError(
"endpoint update failed;\n{}".format(failure_msg))
# still need to wait because status might be "creating" even though build status is "finished"
status_dict = self.get_status()
while status_dict["status"] not in ("active", "error") or (
status_dict["status"] == "active"
and len(status_dict["components"]) > 1):
print(".", end="")
sys.stdout.flush()
time.sleep(5)
status_dict = self.get_status()
print()
if status_dict["status"] == "error":
failure_msg = status_dict['components'][-1].get(
'message', "no error message available")
# NOTE: we might consider truncating the length of the logs here,
# e.g. first and last 25 lines, if too unwieldy
raise RuntimeError(
"endpoint update failed;\n{}".format(failure_msg))
return self.get_status()
def test_empty_response_error(self, client):
response = _utils.make_request(
"GET",
"http://httpbin.org/status/200",
client._conn,
)
with pytest.raises(ValueError) as excinfo:
_utils.body_to_json(response)
msg = str(excinfo.value).strip()
assert msg.startswith("expected JSON response")
assert "<empty response>" in msg
def delete(self):
"""
Deletes this experiment.
"""
request_url = "{}://{}/api/v1/modeldb/experiment/deleteExperiment".format(
self._conn.scheme, self._conn.socket)
response = _utils.make_request("DELETE",
request_url,
self._conn,
json={'id': self.id})
_utils.raise_for_http_error(response)
def test_tags_is_list_of_str(self, client, tags):
client.set_project()
client.set_experiment()
run = client.set_experiment_run(tags=tags)
endpoint = "{}://{}/api/v1/modeldb/experiment-run/getExperimentRunTags".format(
client._conn.scheme,
client._conn.socket,
)
response = _utils.make_request("GET", endpoint, client._conn, params={'id': run.id})
_utils.raise_for_http_error(response)
assert response.json().get('tags', []) == [TAG]
def test_301_continue(self, client):
response = _utils.make_request(
"GET",
"http://httpbin.org/redirect-to",
client._conn,
params={
'url': "http://httpbin.org/get",
'status_code': 301,
},
)
assert response.status_code == 200
assert len(response.history) == 1
assert response.history[0].status_code == 301
def _get_workspace_name_by_id(self, workspace_id):
# getting workspace
response = _utils.make_request(
"GET",
"{}://{}/api/v1/uac-proxy/workspace/getWorkspaceById".format(
self._conn.scheme, self._conn.socket),
self._conn,
params={'id': workspace_id},
)
_utils.raise_for_http_error(response)
response_body = _utils.body_to_json(response)
if 'user_id' in response_body:
user_id = response_body['user_id']
# try getting user
response = _utils.make_request(
"GET",
"{}://{}/api/v1/uac-proxy/uac/getUser".format(
self._conn.scheme, self._conn.socket),
self._conn,
params={'user_id': user_id},
)
_utils.raise_for_http_error(response)
# workspace is user
return _utils.body_to_json(response)['verta_info']['username']
else:
org_id = response_body['org_id']
# try getting organization
response = _utils.make_request(
"GET",
"{}://{}/api/v1/uac-proxy/organization/getOrganizationById".
format(self._conn.scheme, self._conn.socket),
self._conn,
params={'org_id': org_id},
)
# workspace is organization
return _utils.body_to_json(response)['organization']['name']
def test_ignore_conn_err(self, client, status_code):
previous_setting = client.ignore_conn_err
client.ignore_conn_err = True
try:
response = _utils.make_request(
"GET",
"http://httpbin.org/status/{}".format(status_code),
client._conn,
)
assert response.status_code == 200
assert response.json() == {}
finally:
client.ignore_conn_err = previous_setting
def test_200_no_history(self, client):
"""
The util manually tracks and assigns the response history after resolving redirects.
If no redirects occurred, the history should be empty.
"""
response = _utils.make_request(
"GET",
"http://httpbin.org/status/200",
client._conn,
)
assert response.status_code == 200
assert not response.history