def test_alibi_explain_anchor_image_tensorflow_protocol(self, namespace):
spec = "../resources/tf_cifar_anchor_image_explainer.yaml"
name = "cifar10-default-explainer"
vs_prefix = (f"seldon/{namespace}/cifar10-explainer/default/v1/models/"
f"cifar10-classifier:explain")
retry_run(f"kubectl apply -f {spec} -n {namespace}")
wait_for_deployment(name, namespace)
# note: we add a batch dimension but it should be really one image
test_data = np.random.randn(1, 32, 32, 3)
inference_request = {"instances": test_data.tolist()}
for attempt in Retrying(
wait=wait_fixed(TENACITY_WAIT),
stop=stop_after_attempt(TENACITY_STOP_AFTER_ATTEMPT),
):
with attempt:
r = requests.post(
f"http://localhost:8004/{vs_prefix}",
json=inference_request,
)
explanation = r.json()
assert explanation["meta"]["name"] == "AnchorImage"
assert "anchor" in explanation["data"]
assert "precision" in explanation["data"]
assert "coverage" in explanation["data"]
run(f"kubectl delete -f {spec} -n {namespace}", shell=True)
def test_alibi_explain_anchor_tabular(self, namespace):
spec = "../resources/iris_anchor_tabular_explainer.yaml"
name = "iris-default-explainer"
vs_prefix = f"seldon/{namespace}/iris-explainer/default/api/v1.0/explain"
inference_request = {
"data": {
"names": ["text"],
"ndarray": [[5.964, 4.006, 2.081, 1.031]],
}
}
retry_run(f"kubectl apply -f {spec} -n {namespace}")
wait_for_deployment(name, namespace)
time.sleep(AFTER_WAIT_SLEEP)
for attempt in Retrying(
wait=wait_fixed(TENACITY_WAIT),
stop=stop_after_attempt(TENACITY_STOP_AFTER_ATTEMPT),
):
with attempt:
r = requests.post(
f"http://localhost:8004/{vs_prefix}",
json=inference_request,
)
explanation = r.json()
assert explanation["meta"]["name"] == "AnchorTabular"
assert "anchor" in explanation["data"]
assert "precision" in explanation["data"]
assert "coverage" in explanation["data"]
run(f"kubectl delete -f {spec} -n {namespace}", shell=True)
def test_alibi_detect_cifar10_rclone(self, namespace):
spec = "../resources/adserver-cifar10-od-rclone.yaml"
name = "cifar10-od-server-rclone"
vs_prefix = name
retry_run(f"kubectl apply -f {spec} -n {namespace}")
wait_for_deployment(name, namespace)
time.sleep(AFTER_WAIT_SLEEP)
with open(self.truck_json) as f:
data = json.load(f)
for attempt in Retrying(
wait=wait_fixed(TENACITY_WAIT),
stop=stop_after_attempt(TENACITY_STOP_AFTER_ATTEMPT),
):
with attempt:
r = requests.post(
f"http://localhost:8004/{vs_prefix}/",
json=data,
headers=self.HEADERS,
)
j = r.json()
assert j["data"]["is_outlier"][0] == 0
assert j["meta"]["name"] == "OutlierVAE"
assert j["meta"]["detector_type"] == "offline"
assert j["meta"]["data_type"] == "image"
with open(self.truck_json_outlier) as f:
data = json.load(f)
for attempt in Retrying(
wait=wait_fixed(TENACITY_WAIT),
stop=stop_after_attempt(TENACITY_STOP_AFTER_ATTEMPT),
):
with attempt:
r = requests.post(
f"http://localhost:8004/{vs_prefix}/",
json=data,
headers=self.HEADERS,
)
j = r.json()
assert j["data"]["is_outlier"][0] == 1
assert j["meta"]["name"] == "OutlierVAE"
assert j["meta"]["detector_type"] == "offline"
assert j["meta"]["data_type"] == "image"
run(f"kubectl delete -f {spec} -n {namespace}", shell=True)
def test_alibi_explain_anchor_tabular(self, namespace):
spec = "../resources/iris_anchor_tabular_explainer_v2.yaml"
name = "iris-default-explainer"
vs_prefix = (f"seldon/{namespace}/iris-explainer/default/v2/models/"
f"iris-default-explainer/infer")
test_data = np.array([[5.964, 4.006, 2.081, 1.031]])
inference_request = {
"parameters": {
"content_type": "np"
},
"inputs": [
{
"name": "explain",
"shape": test_data.shape,
"datatype": "FP32",
"data": test_data.tolist(),
"parameters": {
"content_type": "np"
},
},
],
}
retry_run(f"kubectl apply -f {spec} -n {namespace}")
wait_for_deployment(name, namespace)
time.sleep(AFTER_WAIT_SLEEP)
for attempt in Retrying(
wait=wait_fixed(TENACITY_WAIT),
stop=stop_after_attempt(TENACITY_STOP_AFTER_ATTEMPT),
):
with attempt:
r = requests.post(
f"http://localhost:8004/{vs_prefix}",
json=inference_request,
)
# note: explanation will come back in v2 as a nested json dictionary
explanation = json.loads(r.json()["outputs"][0]["data"])
assert explanation["meta"]["name"] == "AnchorTabular"
assert "anchor" in explanation["data"]
assert "precision" in explanation["data"]
assert "coverage" in explanation["data"]
run(f"kubectl delete -f {spec} -n {namespace}", shell=True)
