def test_list_revisions_listdir_fail(caplog):
"""
Verify the server will not fail if listing directories above the current
model collection directory it has, fails.
"""
def listdir_fail(*args, **kwargs):
raise FileNotFoundError()
expected_revision = "some-project-revision-123"
with patch.object(os, "listdir",
side_effect=listdir_fail) as mocked_listdir:
with caplog.at_level(logging.CRITICAL):
with tu.temp_env_vars(MODEL_COLLECTION_DIR=expected_revision):
app = server.build_app({"ENABLE_PROMETHEUS": False})
app.testing = True
client = app.test_client()
resp = client.get("/gordo/v0/test-project/revisions")
assert mocked_listdir.called_once()
assert set(
resp.json.keys()) == {"latest", "available-revisions", "revision"}
assert resp.json["latest"] == expected_revision
assert isinstance(resp.json["available-revisions"], list)
assert resp.json["available-revisions"] == [expected_revision]
def test_list_revisions(tmpdir, revisions: List[str]):
"""
Verify the server is capable of returning the project revisions
it's capable of serving.
"""
# Server gets the 'latest' directory to serve models from, but knows other
# revisions should be available a step up from this directory.
model_dir = os.path.join(tmpdir, revisions[0])
# Make revision directories under the tmpdir
[os.mkdir(os.path.join(tmpdir, rev)) for rev in revisions] # type: ignore
# Request from the server what revisions it can serve, should match
with tu.temp_env_vars(MODEL_COLLECTION_DIR=model_dir):
app = server.build_app({"ENABLE_PROMETHEUS": False})
app.testing = True
client = app.test_client()
resp = client.get("/gordo/v0/test-project/revisions")
resp_with_revision = client.get(
f"/gordo/v0/test-project/revisions?revision={revisions[-1]}")
assert set(
resp.json.keys()) == {"latest", "available-revisions", "revision"}
assert resp.json["latest"] == os.path.basename(model_dir)
assert resp.json["revision"] == os.path.basename(model_dir)
assert isinstance(resp.json["available-revisions"], list)
assert set(resp.json["available-revisions"]) == set(revisions)
# And the request asking to use a specific revision gives back that revision,
# but will return the expected latest available
assert resp_with_revision.json["latest"] == os.path.basename(model_dir)
assert resp_with_revision.json["revision"] == revisions[-1]
def test_model_list_view_non_existant_proj():
with tu.temp_env_vars(MODEL_COLLECTION_DIR=os.path.join("does", "not", "exist")):
app = server.build_app()
app.testing = True
client = app.test_client()
resp = client.get("/gordo/v0/test-project/models")
assert resp.status_code == 200
assert resp.json["models"] == []
def gordo_ml_server_client(request, model_collection_directory,
trained_model_directory):
with tu.temp_env_vars(MODEL_COLLECTION_DIR=model_collection_directory):
app = server.build_app()
app.testing = True
yield app.test_client()
def test_request_specific_revision(trained_model_directory, tmpdir, revisions):
model_name = "test-model"
current_revision = revisions[0]
collection_dir = os.path.join(tmpdir, current_revision)
# Copy trained model into revision model folders
for revision in revisions:
model_dir = os.path.join(tmpdir, revision, model_name)
shutil.copytree(trained_model_directory, model_dir)
# Now overwrite the metadata.json file to ensure the server actually reads
# the metadata for this specific revision
metadata_file = os.path.join(model_dir, "metadata.json")
assert os.path.isfile(metadata_file)
with open(metadata_file, "w") as fp:
json.dump({"revision": revision, "model": model_name}, fp)
with tu.temp_env_vars(MODEL_COLLECTION_DIR=collection_dir):
app = server.build_app({"ENABLE_PROMETHEUS": False})
app.testing = True
client = app.test_client()
for revision in revisions:
resp = client.get(
f"/gordo/v0/test-project/{model_name}/metadata?revision={revision}"
)
assert resp.status_code == 200
assert resp.json["revision"] == revision
# Verify the server read the metadata.json file we had overwritten
assert resp.json["metadata"] == {
"revision": revision,
"model": model_name
}
# Asking for a revision which doesn't exist gives a 410 Gone.
resp = client.get(
f"/gordo/v0/test-project/{model_name}/metadata?revision=does-not-exist"
)
assert resp.status_code == 410
assert resp.json == {
"error": "Revision 'does-not-exist' not found.",
"revision": "does-not-exist",
}
# Again but by setting header, to ensure we also check the header
resp = client.get(
f"/gordo/v0/test-project/{model_name}/metadata",
headers={"revision": "does-not-exist"},
)
assert resp.status_code == 410
assert resp.json == {
"error": "Revision 'does-not-exist' not found.",
"revision": "does-not-exist",
}
def gordo_ml_server_client(
request, model_collection_directory, trained_model_directory
):
with tu.temp_env_vars(MODEL_COLLECTION_DIR=model_collection_directory):
app = server.build_app()
app.testing = True
# always return a valid asset for any tag name
with patch.object(sensor_tag, "_asset_from_tag_name", return_value="default"):
yield app.test_client()
def test_server_version_route(model_collection_directory, gordo_revision):
"""
Simple route which returns the current version
"""
with tu.temp_env_vars(MODEL_COLLECTION_DIR=model_collection_directory):
app = server.build_app()
app.testing = True
client = app.test_client()
resp = client.get("/server-version")
assert resp.status_code == 200
assert resp.json == {"revision": gordo_revision, "version": __version__}
def test_non_existant_model_metadata(tmpdir, gordo_project, api_version):
"""
Simple route which returns the current version
"""
with tu.temp_env_vars(MODEL_COLLECTION_DIR=str(tmpdir)):
app = server.build_app({"ENABLE_PROMETHEUS": False})
app.testing = True
client = app.test_client()
resp = client.get(
f"/gordo/{api_version}/{gordo_project}/model-does-not-exist/metadata"
)
assert resp.status_code == 404
def test_expected_models_route(tmpdir):
"""
Route that gives back the expected models names, which are just read from
the 'EXPECTED_MODELS' env var.
"""
with tu.temp_env_vars(
MODEL_COLLECTION_DIR=str(tmpdir),
EXPECTED_MODELS=json.dumps(["model-a", "model-b"]),
):
app = server.build_app({"ENABLE_PROMETHEUS": False})
app.testing = True
client = app.test_client()
resp = client.get("/gordo/v0/test-project/expected-models")
assert resp.json["expected-models"] == ["model-a", "model-b"]
def test_models_by_revision_list_view(caplog, tmpdir, revision_to_models):
"""
Server returns expected models it can serve under specific revisions.
revision_to_models: Dict[str, Tuple[str, ...]]
Map revision codes to models belonging to that revision.
Simulate serving some revision, but having access to other
revisions and its models.
"""
# Current collection dir for the server, order isn't important.
if revision_to_models:
collection_dir = os.path.join(tmpdir,
list(revision_to_models.keys())[0])
else:
# This will cause a failure to look up a certain revision
collection_dir = str(tmpdir)
# Make all the revision and model subfolders
for revision in revision_to_models.keys():
os.mkdir(os.path.join(tmpdir, revision))
for model in revision_to_models[revision]:
os.makedirs(os.path.join(tmpdir, revision, model), exist_ok=True)
with tu.temp_env_vars(MODEL_COLLECTION_DIR=collection_dir):
app = server.build_app({"ENABLE_PROMETHEUS": False})
app.testing = True
client = app.test_client()
for revision in revision_to_models:
resp = client.get(
f"/gordo/v0/test-project/models?revision={revision}")
assert resp.status_code == 200
assert "models" in resp.json
assert sorted(resp.json["models"]) == sorted(
revision_to_models[revision])
else:
# revision_to_models is empty, so there is nothing on the server.
# Test that asking for some arbitrary revision will give a 404 and error message
resp = client.get(
f"/gordo/v0/test-project/models?revision=revision-does-not-exist"
)
assert resp.status_code == 410
assert resp.json == {
"error": "Revision 'revision-does-not-exist' not found.",
"revision": "revision-does-not-exist",
}
def test_with_prometheus():
prometheus_registry = CollectorRegistry()
app = server.build_app({"ENABLE_PROMETHEUS": True}, prometheus_registry)
app.testing = True
client = app.test_client()
client.get("/server-version")
samples = []
for metric in prometheus_registry.collect():
for sample in metric.samples:
if sample.name == "gordo_server_requests_total":
samples.append(sample)
assert (len(samples) !=
0), "Could not found any 'gordo_server_requests_total' metrics"
assert len(
samples) == 1, "Found more then 1 'gordo_server_requests_total' metric"
def ml_server(model_collection_directory, trained_model_directory, gordo_host,
gordo_project):
"""
# TODO: This is bananas, make into a proper object with context support?
Mock a deployed controller deployment
Parameters
----------
gordo_host: str
Host controller should pretend to run on
gordo_project: str
Project controller should pretend to care about
model_collection_directory: str
Directory of the model to use in the target(s)
Returns
-------
None
"""
with tu.temp_env_vars(MODEL_COLLECTION_DIR=model_collection_directory):
# Create gordo ml servers
gordo_server_app = gordo_ml_server.build_app()
gordo_server_app.testing = True
gordo_server_app = gordo_server_app.test_client()
def gordo_ml_server_callback(request):
"""
Redirect calls to a gordo server to reflect what the local testing app gives
will call the correct path (assuminng only single level paths) on the
gordo app.
"""
if request.method in ("GET", "POST"):
kwargs = dict()
if request.body:
flask_request = Request.from_values(
content_length=len(request.body),
input_stream=io.BytesIO(request.body),
content_type=request.headers["Content-Type"],
method=request.method,
)
if flask_request.json:
kwargs["json"] = flask_request.json
else:
kwargs["data"] = {
k: (io.BytesIO(f.read()), f.filename)
for k, f in flask_request.files.items()
}
with TEST_SERVER_MUTEXT:
resp = getattr(gordo_server_app, request.method.lower())(
request.path_url,
headers=dict(request.headers),
**kwargs)
return (
resp.status_code,
resp.headers,
json.dumps(resp.json)
if resp.json is not None else resp.data,
)
with responses.RequestsMock(
assert_all_requests_are_fired=False) as rsps:
rsps.add_callback(
responses.GET,
re.compile(
rf".*{gordo_host}.*\/gordo\/v0\/{gordo_project}\/.+"),
callback=gordo_ml_server_callback,
content_type="application/json",
)
rsps.add_callback(
responses.POST,
re.compile(
rf".*{gordo_host}.*\/gordo\/v0\/{gordo_project}\/.*.\/.*"),
callback=gordo_ml_server_callback,
content_type="application/json",
)
rsps.add_passthru("http+docker://") # Docker
rsps.add_passthru("http://localhost:8086") # Local influx
rsps.add_passthru("http://localhost:8087") # Local influx
yield
