def test_list_revisions_listdir_fail(caplog): """ Verify the server will not fail if listing directories above the current model collection directory it has, fails. """ def listdir_fail(*args, **kwargs): raise FileNotFoundError() expected_revision = "some-project-revision-123" with patch.object(os, "listdir", side_effect=listdir_fail) as mocked_listdir: with caplog.at_level(logging.CRITICAL): with tu.temp_env_vars(MODEL_COLLECTION_DIR=expected_revision): app = server.build_app({"ENABLE_PROMETHEUS": False}) app.testing = True client = app.test_client() resp = client.get("/gordo/v0/test-project/revisions") assert mocked_listdir.called_once() assert set( resp.json.keys()) == {"latest", "available-revisions", "revision"} assert resp.json["latest"] == expected_revision assert isinstance(resp.json["available-revisions"], list) assert resp.json["available-revisions"] == [expected_revision]
def test_list_revisions(tmpdir, revisions: List[str]): """ Verify the server is capable of returning the project revisions it's capable of serving. """ # Server gets the 'latest' directory to serve models from, but knows other # revisions should be available a step up from this directory. model_dir = os.path.join(tmpdir, revisions[0]) # Make revision directories under the tmpdir [os.mkdir(os.path.join(tmpdir, rev)) for rev in revisions] # type: ignore # Request from the server what revisions it can serve, should match with tu.temp_env_vars(MODEL_COLLECTION_DIR=model_dir): app = server.build_app({"ENABLE_PROMETHEUS": False}) app.testing = True client = app.test_client() resp = client.get("/gordo/v0/test-project/revisions") resp_with_revision = client.get( f"/gordo/v0/test-project/revisions?revision={revisions[-1]}") assert set( resp.json.keys()) == {"latest", "available-revisions", "revision"} assert resp.json["latest"] == os.path.basename(model_dir) assert resp.json["revision"] == os.path.basename(model_dir) assert isinstance(resp.json["available-revisions"], list) assert set(resp.json["available-revisions"]) == set(revisions) # And the request asking to use a specific revision gives back that revision, # but will return the expected latest available assert resp_with_revision.json["latest"] == os.path.basename(model_dir) assert resp_with_revision.json["revision"] == revisions[-1]
def test_model_list_view_non_existant_proj(): with tu.temp_env_vars(MODEL_COLLECTION_DIR=os.path.join("does", "not", "exist")): app = server.build_app() app.testing = True client = app.test_client() resp = client.get("/gordo/v0/test-project/models") assert resp.status_code == 200 assert resp.json["models"] == []
def gordo_ml_server_client(request, model_collection_directory, trained_model_directory): with tu.temp_env_vars(MODEL_COLLECTION_DIR=model_collection_directory): app = server.build_app() app.testing = True yield app.test_client()
def test_request_specific_revision(trained_model_directory, tmpdir, revisions): model_name = "test-model" current_revision = revisions[0] collection_dir = os.path.join(tmpdir, current_revision) # Copy trained model into revision model folders for revision in revisions: model_dir = os.path.join(tmpdir, revision, model_name) shutil.copytree(trained_model_directory, model_dir) # Now overwrite the metadata.json file to ensure the server actually reads # the metadata for this specific revision metadata_file = os.path.join(model_dir, "metadata.json") assert os.path.isfile(metadata_file) with open(metadata_file, "w") as fp: json.dump({"revision": revision, "model": model_name}, fp) with tu.temp_env_vars(MODEL_COLLECTION_DIR=collection_dir): app = server.build_app({"ENABLE_PROMETHEUS": False}) app.testing = True client = app.test_client() for revision in revisions: resp = client.get( f"/gordo/v0/test-project/{model_name}/metadata?revision={revision}" ) assert resp.status_code == 200 assert resp.json["revision"] == revision # Verify the server read the metadata.json file we had overwritten assert resp.json["metadata"] == { "revision": revision, "model": model_name } # Asking for a revision which doesn't exist gives a 410 Gone. resp = client.get( f"/gordo/v0/test-project/{model_name}/metadata?revision=does-not-exist" ) assert resp.status_code == 410 assert resp.json == { "error": "Revision 'does-not-exist' not found.", "revision": "does-not-exist", } # Again but by setting header, to ensure we also check the header resp = client.get( f"/gordo/v0/test-project/{model_name}/metadata", headers={"revision": "does-not-exist"}, ) assert resp.status_code == 410 assert resp.json == { "error": "Revision 'does-not-exist' not found.", "revision": "does-not-exist", }
def gordo_ml_server_client( request, model_collection_directory, trained_model_directory ): with tu.temp_env_vars(MODEL_COLLECTION_DIR=model_collection_directory): app = server.build_app() app.testing = True # always return a valid asset for any tag name with patch.object(sensor_tag, "_asset_from_tag_name", return_value="default"): yield app.test_client()
def test_server_version_route(model_collection_directory, gordo_revision): """ Simple route which returns the current version """ with tu.temp_env_vars(MODEL_COLLECTION_DIR=model_collection_directory): app = server.build_app() app.testing = True client = app.test_client() resp = client.get("/server-version") assert resp.status_code == 200 assert resp.json == {"revision": gordo_revision, "version": __version__}
def test_non_existant_model_metadata(tmpdir, gordo_project, api_version): """ Simple route which returns the current version """ with tu.temp_env_vars(MODEL_COLLECTION_DIR=str(tmpdir)): app = server.build_app({"ENABLE_PROMETHEUS": False}) app.testing = True client = app.test_client() resp = client.get( f"/gordo/{api_version}/{gordo_project}/model-does-not-exist/metadata" ) assert resp.status_code == 404
def test_expected_models_route(tmpdir): """ Route that gives back the expected models names, which are just read from the 'EXPECTED_MODELS' env var. """ with tu.temp_env_vars( MODEL_COLLECTION_DIR=str(tmpdir), EXPECTED_MODELS=json.dumps(["model-a", "model-b"]), ): app = server.build_app({"ENABLE_PROMETHEUS": False}) app.testing = True client = app.test_client() resp = client.get("/gordo/v0/test-project/expected-models") assert resp.json["expected-models"] == ["model-a", "model-b"]
def test_models_by_revision_list_view(caplog, tmpdir, revision_to_models): """ Server returns expected models it can serve under specific revisions. revision_to_models: Dict[str, Tuple[str, ...]] Map revision codes to models belonging to that revision. Simulate serving some revision, but having access to other revisions and its models. """ # Current collection dir for the server, order isn't important. if revision_to_models: collection_dir = os.path.join(tmpdir, list(revision_to_models.keys())[0]) else: # This will cause a failure to look up a certain revision collection_dir = str(tmpdir) # Make all the revision and model subfolders for revision in revision_to_models.keys(): os.mkdir(os.path.join(tmpdir, revision)) for model in revision_to_models[revision]: os.makedirs(os.path.join(tmpdir, revision, model), exist_ok=True) with tu.temp_env_vars(MODEL_COLLECTION_DIR=collection_dir): app = server.build_app({"ENABLE_PROMETHEUS": False}) app.testing = True client = app.test_client() for revision in revision_to_models: resp = client.get( f"/gordo/v0/test-project/models?revision={revision}") assert resp.status_code == 200 assert "models" in resp.json assert sorted(resp.json["models"]) == sorted( revision_to_models[revision]) else: # revision_to_models is empty, so there is nothing on the server. # Test that asking for some arbitrary revision will give a 404 and error message resp = client.get( f"/gordo/v0/test-project/models?revision=revision-does-not-exist" ) assert resp.status_code == 410 assert resp.json == { "error": "Revision 'revision-does-not-exist' not found.", "revision": "revision-does-not-exist", }
def test_with_prometheus(): prometheus_registry = CollectorRegistry() app = server.build_app({"ENABLE_PROMETHEUS": True}, prometheus_registry) app.testing = True client = app.test_client() client.get("/server-version") samples = [] for metric in prometheus_registry.collect(): for sample in metric.samples: if sample.name == "gordo_server_requests_total": samples.append(sample) assert (len(samples) != 0), "Could not found any 'gordo_server_requests_total' metrics" assert len( samples) == 1, "Found more then 1 'gordo_server_requests_total' metric"
def ml_server(model_collection_directory, trained_model_directory, gordo_host, gordo_project): """ # TODO: This is bananas, make into a proper object with context support? Mock a deployed controller deployment Parameters ---------- gordo_host: str Host controller should pretend to run on gordo_project: str Project controller should pretend to care about model_collection_directory: str Directory of the model to use in the target(s) Returns ------- None """ with tu.temp_env_vars(MODEL_COLLECTION_DIR=model_collection_directory): # Create gordo ml servers gordo_server_app = gordo_ml_server.build_app() gordo_server_app.testing = True gordo_server_app = gordo_server_app.test_client() def gordo_ml_server_callback(request): """ Redirect calls to a gordo server to reflect what the local testing app gives will call the correct path (assuminng only single level paths) on the gordo app. """ if request.method in ("GET", "POST"): kwargs = dict() if request.body: flask_request = Request.from_values( content_length=len(request.body), input_stream=io.BytesIO(request.body), content_type=request.headers["Content-Type"], method=request.method, ) if flask_request.json: kwargs["json"] = flask_request.json else: kwargs["data"] = { k: (io.BytesIO(f.read()), f.filename) for k, f in flask_request.files.items() } with TEST_SERVER_MUTEXT: resp = getattr(gordo_server_app, request.method.lower())( request.path_url, headers=dict(request.headers), **kwargs) return ( resp.status_code, resp.headers, json.dumps(resp.json) if resp.json is not None else resp.data, ) with responses.RequestsMock( assert_all_requests_are_fired=False) as rsps: rsps.add_callback( responses.GET, re.compile( rf".*{gordo_host}.*\/gordo\/v0\/{gordo_project}\/.+"), callback=gordo_ml_server_callback, content_type="application/json", ) rsps.add_callback( responses.POST, re.compile( rf".*{gordo_host}.*\/gordo\/v0\/{gordo_project}\/.*.\/.*"), callback=gordo_ml_server_callback, content_type="application/json", ) rsps.add_passthru("http+docker://") # Docker rsps.add_passthru("http://localhost:8086") # Local influx rsps.add_passthru("http://localhost:8087") # Local influx yield