def test_from_run(self, experiment_run, model_for_deployment,
registered_model):
np = pytest.importorskip("numpy")
experiment_run.log_model(model_for_deployment['model'],
custom_modules=[])
experiment_run.log_environment(Python(['scikit-learn']))
artifact = np.random.random((36, 12))
experiment_run.log_artifact("some-artifact", artifact)
model_version = registered_model.create_version_from_run(
run_id=experiment_run.id,
name="From Run {}".format(experiment_run.id),
)
env_str = str(model_version.get_environment())
assert 'scikit-learn' in env_str
assert 'Python' in env_str
assert model_for_deployment['model'].get_params(
) == model_version.get_model().get_params()
assert np.array_equal(model_version.get_artifact("some-artifact"),
artifact)
def test_update_from_version_diff_workspace_no_access_error(
self, client_2, model_version, created_endpoints):
np = pytest.importorskip("numpy")
sklearn = pytest.importorskip("sklearn")
from sklearn.linear_model import LogisticRegression
classifier = LogisticRegression()
classifier.fit(np.random.random((36, 12)),
np.random.random(36).round())
model_version.log_model(classifier)
env = Python(requirements=["scikit-learn"])
model_version.log_environment(env)
path = verta._internal_utils._utils.generate_default_name()
endpoint = client_2.create_endpoint(path)
created_endpoints.append(endpoint)
with pytest.raises(requests.HTTPError) as excinfo:
endpoint.update(model_version, DirectUpdateStrategy(), wait=True)
excinfo_value = str(excinfo.value).strip()
assert "403" in excinfo_value
assert "Access Denied" in excinfo_value
def test_not_xgboost(self, registered_model, model):
with pytest.raises(TypeError):
registered_model.create_standard_model_from_xgboost(
model,
Python(["scikit-learn", "xgboost"]),
)
def test_not_verta(self, registered_model, model):
with pytest.raises(TypeError):
registered_model.create_standard_model(
model,
Python([]),
)
def test_not_torch(self, registered_model, model):
with pytest.raises(TypeError):
registered_model.create_standard_model_from_torch(
model,
Python(["torch"]),
)
def test_torch_no_suffix(self):
# NOTE: this test takes too long for Hypothesis
requirement = "torch==1.8.1+cu102"
env_ver = Python([requirement])
assert requirement not in env_ver.requirements
assert requirement.split("+")[0] in env_ver.requirements
def test_env_vars(self):
env_vars = os.environ.keys()
env = Python(requirements=[], env_vars=env_vars)
assert env._msg.environment_variables
def test_constraints_from_file_no_versions_error(
self, requirements_file_without_versions):
reqs = Python.read_pip_file(requirements_file_without_versions.name)
with pytest.raises(ValueError):
Python(requirements=[], constraints=reqs)
def test_inject_verta_cloudpickle(self):
env = Python(requirements=[])
requirements = {req.library for req in env._msg.python.requirements}
assert 'verta' in requirements
assert 'cloudpickle' in requirements
def test_reqs_without_versions(self, requirements_file_without_versions):
reqs = Python.read_pip_file(requirements_file_without_versions.name)
env = Python(requirements=reqs)
assert env._msg.python.requirements
def test_constraints_from_file(self, requirements_file):
reqs = Python.read_pip_file(requirements_file.name)
env = Python(requirements=[], constraints=reqs)
assert env._msg.python.constraints
def test_reqs(self, requirements_file):
reqs = Python.read_pip_file(requirements_file.name)
env = Python(requirements=reqs)
assert env._msg.python.requirements
def test_reqs_from_env(self):
reqs = Python.read_pip_environment()
env = Python(requirements=reqs)
assert env._msg.python.requirements
def test_commit(self, commit):
env = Python(requirements=[])
commit.update('env', env)
commit.save(message="banana")
assert commit.get('env')
def test_skip_spacy_models(self):
pattern = SPACY_MODEL_PATTERN + "=="
requirements = Python.read_pip_environment()
assert not any(re.match(pattern, req) for req in requirements)
def default_environment(cls):
return Python(requirements=["numpy", "scipy", "pandas"])
def test_py_ver(self):
env = Python(requirements=[])
assert env._msg.python.version.major == sys.version_info.major
assert env._msg.python.version.minor == sys.version_info.minor
assert env._msg.python.version.patch == sys.version_info.micro
def test_update_from_json_config_with_params(self, client, in_tempdir,
created_entities,
experiment_run,
model_for_deployment):
yaml = pytest.importorskip("yaml")
experiment_run.log_model(model_for_deployment['model'],
custom_modules=[])
experiment_run.log_environment(Python(['scikit-learn']))
path = verta._internal_utils._utils.generate_default_name()
endpoint = client.set_endpoint(path)
created_entities.append(endpoint)
original_status = endpoint.get_status()
original_build_ids = get_build_ids(original_status)
# Creating config dict:
config_dict = {
"run_id": experiment_run.id,
"strategy": "direct",
"autoscaling": {
"quantities": {
"min_replicas": 1,
"max_replicas": 4,
"min_scale": 0.5,
"max_scale": 2.0
},
"metrics": [{
"metric": "cpu_utilization",
"parameters": [{
"name": "target",
"value": "0.5"
}]
}, {
"metric": "memory_utilization",
"parameters": [{
"name": "target",
"value": "0.7"
}]
}]
},
"env_vars": {
"VERTA_HOST": "app.verta.ai"
},
"resources": {
"cpu": 0.25,
"memory": "100M"
}
}
filepath = "config.json"
with open(filepath, 'w') as f:
json.dump(config_dict, f)
endpoint.update_from_config(filepath)
update_status = endpoint.get_update_status()
# Check autoscaling:
autoscaling_parameters = update_status["update_request"]["autoscaling"]
autoscaling_quantities = autoscaling_parameters["quantities"]
assert autoscaling_quantities == config_dict["autoscaling"][
"quantities"]
autoscaling_metrics = autoscaling_parameters["metrics"]
assert len(autoscaling_metrics) == 2
for metric in autoscaling_metrics:
assert metric["metric_id"] in [1001, 1002, 1003]
if metric["metric_id"] == 1001:
assert metric["parameters"][0]["value"] == "0.5"
else:
assert metric["parameters"][0]["value"] == "0.7"
# Check env_vars:
assert update_status["update_request"]["env"][0][
"name"] == "VERTA_HOST"
assert update_status["update_request"]["env"][0][
"value"] == "app.verta.ai"
# Check resources:
resources_dict = Resources._from_dict(
config_dict["resources"])._as_dict(
) # config is `cpu`, wire is `cpu_millis`
assert endpoint.get_update_status(
)['update_request']['resources'] == resources_dict
def test_inject_verta_cloudpickle(self):
env = Python(requirements=["pytest"])
requirements = {req.library for req in env._msg.python.requirements}
assert "verta" in requirements
assert "cloudpickle" in requirements
def test_no_overwrite(self, experiment_run):
python_env = Python([])
experiment_run.log_environment(python_env)
assert experiment_run.has_environment
with pytest.raises(ValueError):
experiment_run.log_environment(python_env, overwrite=False)
