def test_from_run(self, experiment_run, model_for_deployment, registered_model): np = pytest.importorskip("numpy") experiment_run.log_model(model_for_deployment['model'], custom_modules=[]) experiment_run.log_environment(Python(['scikit-learn'])) artifact = np.random.random((36, 12)) experiment_run.log_artifact("some-artifact", artifact) model_version = registered_model.create_version_from_run( run_id=experiment_run.id, name="From Run {}".format(experiment_run.id), ) env_str = str(model_version.get_environment()) assert 'scikit-learn' in env_str assert 'Python' in env_str assert model_for_deployment['model'].get_params( ) == model_version.get_model().get_params() assert np.array_equal(model_version.get_artifact("some-artifact"), artifact)
def test_update_from_version_diff_workspace_no_access_error( self, client_2, model_version, created_endpoints): np = pytest.importorskip("numpy") sklearn = pytest.importorskip("sklearn") from sklearn.linear_model import LogisticRegression classifier = LogisticRegression() classifier.fit(np.random.random((36, 12)), np.random.random(36).round()) model_version.log_model(classifier) env = Python(requirements=["scikit-learn"]) model_version.log_environment(env) path = verta._internal_utils._utils.generate_default_name() endpoint = client_2.create_endpoint(path) created_endpoints.append(endpoint) with pytest.raises(requests.HTTPError) as excinfo: endpoint.update(model_version, DirectUpdateStrategy(), wait=True) excinfo_value = str(excinfo.value).strip() assert "403" in excinfo_value assert "Access Denied" in excinfo_value
def test_not_xgboost(self, registered_model, model): with pytest.raises(TypeError): registered_model.create_standard_model_from_xgboost( model, Python(["scikit-learn", "xgboost"]), )
def test_not_verta(self, registered_model, model): with pytest.raises(TypeError): registered_model.create_standard_model( model, Python([]), )
def test_not_torch(self, registered_model, model): with pytest.raises(TypeError): registered_model.create_standard_model_from_torch( model, Python(["torch"]), )
def test_torch_no_suffix(self): # NOTE: this test takes too long for Hypothesis requirement = "torch==1.8.1+cu102" env_ver = Python([requirement]) assert requirement not in env_ver.requirements assert requirement.split("+")[0] in env_ver.requirements
def test_env_vars(self): env_vars = os.environ.keys() env = Python(requirements=[], env_vars=env_vars) assert env._msg.environment_variables
def test_constraints_from_file_no_versions_error( self, requirements_file_without_versions): reqs = Python.read_pip_file(requirements_file_without_versions.name) with pytest.raises(ValueError): Python(requirements=[], constraints=reqs)
def test_inject_verta_cloudpickle(self): env = Python(requirements=[]) requirements = {req.library for req in env._msg.python.requirements} assert 'verta' in requirements assert 'cloudpickle' in requirements
def test_reqs_without_versions(self, requirements_file_without_versions): reqs = Python.read_pip_file(requirements_file_without_versions.name) env = Python(requirements=reqs) assert env._msg.python.requirements
def test_constraints_from_file(self, requirements_file): reqs = Python.read_pip_file(requirements_file.name) env = Python(requirements=[], constraints=reqs) assert env._msg.python.constraints
def test_reqs(self, requirements_file): reqs = Python.read_pip_file(requirements_file.name) env = Python(requirements=reqs) assert env._msg.python.requirements
def test_reqs_from_env(self): reqs = Python.read_pip_environment() env = Python(requirements=reqs) assert env._msg.python.requirements
def test_commit(self, commit): env = Python(requirements=[]) commit.update('env', env) commit.save(message="banana") assert commit.get('env')
def test_skip_spacy_models(self): pattern = SPACY_MODEL_PATTERN + "==" requirements = Python.read_pip_environment() assert not any(re.match(pattern, req) for req in requirements)
def default_environment(cls): return Python(requirements=["numpy", "scipy", "pandas"])
def test_py_ver(self): env = Python(requirements=[]) assert env._msg.python.version.major == sys.version_info.major assert env._msg.python.version.minor == sys.version_info.minor assert env._msg.python.version.patch == sys.version_info.micro
def test_update_from_json_config_with_params(self, client, in_tempdir, created_entities, experiment_run, model_for_deployment): yaml = pytest.importorskip("yaml") experiment_run.log_model(model_for_deployment['model'], custom_modules=[]) experiment_run.log_environment(Python(['scikit-learn'])) path = verta._internal_utils._utils.generate_default_name() endpoint = client.set_endpoint(path) created_entities.append(endpoint) original_status = endpoint.get_status() original_build_ids = get_build_ids(original_status) # Creating config dict: config_dict = { "run_id": experiment_run.id, "strategy": "direct", "autoscaling": { "quantities": { "min_replicas": 1, "max_replicas": 4, "min_scale": 0.5, "max_scale": 2.0 }, "metrics": [{ "metric": "cpu_utilization", "parameters": [{ "name": "target", "value": "0.5" }] }, { "metric": "memory_utilization", "parameters": [{ "name": "target", "value": "0.7" }] }] }, "env_vars": { "VERTA_HOST": "app.verta.ai" }, "resources": { "cpu": 0.25, "memory": "100M" } } filepath = "config.json" with open(filepath, 'w') as f: json.dump(config_dict, f) endpoint.update_from_config(filepath) update_status = endpoint.get_update_status() # Check autoscaling: autoscaling_parameters = update_status["update_request"]["autoscaling"] autoscaling_quantities = autoscaling_parameters["quantities"] assert autoscaling_quantities == config_dict["autoscaling"][ "quantities"] autoscaling_metrics = autoscaling_parameters["metrics"] assert len(autoscaling_metrics) == 2 for metric in autoscaling_metrics: assert metric["metric_id"] in [1001, 1002, 1003] if metric["metric_id"] == 1001: assert metric["parameters"][0]["value"] == "0.5" else: assert metric["parameters"][0]["value"] == "0.7" # Check env_vars: assert update_status["update_request"]["env"][0][ "name"] == "VERTA_HOST" assert update_status["update_request"]["env"][0][ "value"] == "app.verta.ai" # Check resources: resources_dict = Resources._from_dict( config_dict["resources"])._as_dict( ) # config is `cpu`, wire is `cpu_millis` assert endpoint.get_update_status( )['update_request']['resources'] == resources_dict
def test_inject_verta_cloudpickle(self): env = Python(requirements=["pytest"]) requirements = {req.library for req in env._msg.python.requirements} assert "verta" in requirements assert "cloudpickle" in requirements
def test_no_overwrite(self, experiment_run): python_env = Python([]) experiment_run.log_environment(python_env) assert experiment_run.has_environment with pytest.raises(ValueError): experiment_run.log_environment(python_env, overwrite=False)