def test_retrying_rpc_exception():
first_call = [True]
def predict_mock(request, timeout):
inputs = tf.contrib.util.make_ndarray(request.inputs['images'])
assert all([x == y for x, y in zip(inputs[0], [1, 2])])
if (first_call[0]):
first_call[0] = False
raise grpc.RpcError()
return_data = np.asarray([[11, 22]])
return_tensor = tf.contrib.util.make_tensor_proto(
return_data, types_pb2.DT_FLOAT, return_data.shape)
result = mock.MagicMock()
result.outputs = {"output_alias": return_tensor}
return result
stub_mock = mock.Mock()
stub_mock.Predict = mock.MagicMock(side_effect=predict_mock)
client = PredictionClient("localhost", 50051)
client._get_grpc_stub = lambda: stub_mock
result = client.score_numpy_array(np.asarray([[1, 2]], dtype='f'))
assert all([x == y for x, y in zip(result[0], [11, 22])])
def test_remote_featurizer_create_package_and_service():
test_config = get_test_config()
deployment_client = DeploymentClient(
test_config['test_subscription_id'],
test_config['test_resource_group'],
test_config['test_model_management_account'],
service_principal_params=get_service_principal())
id = uuid.uuid4().hex[:5]
model_name = "int-test-rf-model-" + id
service_name = "int-test-rf-service-" + id
service_def_path = "/tmp/modelrf"
in_images = tf.placeholder(tf.string)
image_tensors = preprocess_array(in_images)
remote_service_name = ("int-test-featurizer-svc-" + str(uuid.uuid4()))[:30]
model = RemoteQuantizedResNet50(
test_config['test_subscription_id'],
test_config['test_resource_group'],
test_config['test_model_management_account'],
os.path.expanduser("~/models"),
remote_service_name,
service_principal_params=get_service_principal())
model.import_graph_def(include_featurizer=True, input_tensor=image_tensors)
service_def = ServiceDefinition()
service_def.pipeline.append(
TensorflowStage(tf.Session(), in_images, image_tensors))
service_def.pipeline.append(BrainWaveStage(model))
service_def.pipeline.append(
TensorflowStage(tf.Session(), model.classifier_input,
model.classifier_output))
service_def.save(service_def_path)
# create service
model_id = deployment_client.register_model(model_name, service_def_path)
service = deployment_client.create_service(service_name, model_id)
prediction_client = PredictionClient(service.ipAddress, service.port)
top_result = sorted(enumerate(
prediction_client.score_image("/tmp/share1/shark.jpg")),
key=lambda x: x[1],
reverse=True)[:1]
# 'tiger shark' is class 3
assert top_result[0][0] == 3
deployment_client.delete_service(service.id)
deployment_client.delete_model(model_id)
def worker(IP, path):
count = 0
tt = 0
#path = '/home/srpitcha/images/dog.png'
for i in range(tper):
ts = time.time()
count += 1
client = PredictionClient(IP, 80, False, '')
results = client.score_image(path)
te = time.time()
tt += te-ts
mtime = tt/count
meantime.append(mtime)
def test_retrying_rpc_exception():
first_call = [True]
channel_mock_loaded = {'value': 0}
channel_mock_closed = {'value': 0}
def unary_unary(id, request_serializer, response_deserializer):
result = mock.MagicMock()
if id == '/tensorflow.serving.PredictionService/Predict':
if (first_call[0]):
first_call[0] = False
return lambda req, timeout: (_ for _ in
()).throw(grpc.RpcError())
return_data = np.asarray([[11, 22]])
return_tensor = tf.contrib.util.make_tensor_proto(
return_data, types_pb2.DT_FLOAT, return_data.shape)
result.outputs = {"output_alias": return_tensor}
return lambda req, timeout: result
def load_channel_mock():
channel_mock_loaded['value'] += 1
return channel_mock
def close_channel_mock():
channel_mock_closed['value'] += 1
now = datetime.now()
channel_mock = mock.Mock()
channel_mock.unary_unary = mock.MagicMock(side_effect=unary_unary)
channel_mock.close = close_channel_mock
client = PredictionClient("localhost",
50051,
channel_shutdown_timeout=timedelta(minutes=1))
client._channel_func = load_channel_mock
client._get_datetime_now = lambda: now
result = client.score_numpy_array(np.asarray([[1, 2]], dtype='f'))
assert all([x == y for x, y in zip(result[0], [11, 22])])
assert channel_mock_loaded['value'] == 2
assert channel_mock_closed['value'] == 1
def _import_featurizer_graph_def(self, input_map):
service = self.__deployment_client.get_service_by_name(
self.__service_name)
if (service is None):
model_name = self.model_name + "-" + self.version + "-model"
temp_dir = tempfile.mkdtemp()
model_path = os.path.join(temp_dir, "model")
service_def = ServiceDefinition()
service_def.pipeline.append(BrainWaveStage(self))
service_def.save(model_path)
model_id = self.__deployment_client.register_model(
model_name, model_path)
service = self.__deployment_client.create_service(
self.__service_name, model_id)
self.__client = PredictionClient(service.ipAddress, service.port) #pylint: disable=E1101
return input_map['InputImage'], tf.py_func(self._remote_service_call,
[input_map['InputImage']],
tf.float32)
def worker(IP, path, trainedDs):
count = 0
tt = 0
datadir = os.path.expanduser(path)
cat_files = glob.glob(os.path.join(datadir, 'pass', '*.jpg'))
dog_files = glob.glob(os.path.join(datadir, 'fail', '*.jpg'))
pdb.set_trace()
for image in cat_files[:5]:
ts = time.time()
count += 1
client = PredictionClient(IP, 80, False, '')
results = client.score_image(image)
results = enumerate(results)
sorted_results = sorted(results, key=lambda x: x[1], reverse=True)
for top in sorted_results[:5]:
print(trainedDs[top[0]], 'confidence:', top[1])
te = time.time()
tt += te - ts
mtime = tt / count
meantime.append(mtime)
def test_create_new_channel_after_timeout_expires():
channel_mock_loaded = {'value': 0}
def unary_unary(id, request_serializer, response_deserializer):
result = mock.MagicMock()
if id == '/tensorflow.serving.PredictionService/Predict':
return_data = np.asarray([[1, 2, 3]])
return_tensor = tf.contrib.util.make_tensor_proto(
return_data, types_pb2.DT_FLOAT, return_data.shape)
result.outputs = {"output_alias": return_tensor}
return lambda req, timeout: result
def load_channel_mock():
channel_mock_loaded['value'] += 1
return channel_mock
now = datetime.now()
channel_mock = mock.Mock()
channel_mock.unary_unary = mock.MagicMock(side_effect=unary_unary)
image_file_path = os.path.join(tempfile.mkdtemp(), "img.png")
image_file = open(image_file_path, "w")
image_file.write("abc")
image_file.close()
client = PredictionClient("localhost",
50051,
channel_shutdown_timeout=timedelta(minutes=1))
client._channel_func = load_channel_mock
client._get_datetime_now = lambda: now
result = client.score_image(image_file_path)
assert all([x == y for x, y in zip(result, [1, 2, 3])])
assert channel_mock_loaded['value'] == 1
now = now + timedelta(seconds=50)
result = client.score_image(image_file_path)
assert all([x == y for x, y in zip(result, [1, 2, 3])])
assert channel_mock_loaded['value'] == 1
now = now + timedelta(seconds=20)
result = client.score_image(image_file_path)
assert all([x == y for x, y in zip(result, [1, 2, 3])])
assert channel_mock_loaded['value'] == 1
now = now + timedelta(seconds=70)
result = client.score_image(image_file_path)
assert all([x == y for x, y in zip(result, [1, 2, 3])])
assert channel_mock_loaded['value'] == 2
def worker(IP, path):
datadir = os.path.expanduser(path)
pass_files= glob.glob(os.path.join(datadir, 'pass', '*.jpg'))
fail_files = glob.glob(os.path.join(datadir, 'fail', '*.jpg'))
dest_pass = os.path.join(datadir, 'output', 'pass')
dest_fail = os.path.join(datadir, 'output', 'fail')
print("Pass files")
for image in pass_files:
client = PredictionClient(IP, 80, False, '')
results = client.score_image(image)
if (results >= 0.5):
print("PASS", image, results)
cp(image, dest_pass)
else:
print("FAIL", image, results)
cp(image, dest_fail)
print("Fail files")
for image in fail_files:
client = PredictionClient(IP, 80, False, '')
results = client.score_image(image)
if (results < 0.5):
cp(image, dest_fail)
else:
print(image, results)
cp(image, dest_pass)
def test_score_numpy_array():
def predict_mock(request, timeout):
inputs = tf.contrib.util.make_ndarray(request.inputs['images'])
assert all([x == y for x, y in zip(inputs[0], [ 1, 2, 3 ])])
assert all([x == y for x, y in zip(inputs[1], [ 4, 5, 6 ])])
return_data = np.asarray([[ 11, 22, 33 ], [ 44, 55, 66 ]])
return_tensor = tf.contrib.util.make_tensor_proto(return_data, types_pb2.DT_FLOAT, return_data.shape)
result = mock.MagicMock()
result.outputs = { "output_alias": return_tensor }
return result
stub_mock = mock.Mock()
stub_mock.Predict = mock.MagicMock(side_effect=predict_mock)
client = PredictionClient("localhost", 50051)
client.stub = stub_mock
result = client.score_numpy_array(np.asarray([[1, 2, 3], [4, 5, 6]], dtype='f'))
assert all([x == y for x, y in zip(result[0], [ 11, 22, 33 ])])
assert all([x == y for x, y in zip(result[1], [ 44, 55, 66 ])])
def test_score_image():
def predict_mock(request, timeout):
inputs = request.inputs['images'].string_val
assert inputs[0].decode('utf-8') == "abc"
return_data = np.asarray([[1, 2, 3]])
return_tensor = tf.contrib.util.make_tensor_proto(
return_data, types_pb2.DT_FLOAT, return_data.shape)
result = mock.MagicMock()
result.outputs = {"output_alias": return_tensor}
return result
stub_mock = mock.Mock()
stub_mock.Predict = mock.MagicMock(side_effect=predict_mock)
image_file_path = os.path.join(tempfile.mkdtemp(), "img.png")
image_file = open(image_file_path, "w")
image_file.write("abc")
image_file.close()
client = PredictionClient("localhost", 50051)
client._get_grpc_stub = lambda: stub_mock
result = client.score_image(image_file_path)
assert all([x == y for x, y in zip(result, [1, 2, 3])])
class RemoteQuantizedResNet50(QuantizedResNet50):
def __init__(self,
subscription_id,
resource_group,
model_management_account,
model_base_path,
remote_service_name=None):
super().__init__(model_base_path)
self.__deployment_client = DeploymentClient(subscription_id,
resource_group,
model_management_account)
self.__service_name = remote_service_name if remote_service_name is not None else "featurizer-service-" + hashlib.md5(
(self.model_name + "-" +
self.version).encode("utf-8")).hexdigest()[:6]
def _import_featurizer_graph_def(self, input_map):
service = self.__deployment_client.get_service_by_name(
self.__service_name)
if (service is None):
model_name = self.model_name + "-" + self.version + "-model"
temp_dir = tempfile.mkdtemp()
model_path = os.path.join(temp_dir, "model")
service_def = ServiceDefinition()
service_def.pipeline.append(BrainWaveStage(self))
service_def.save(model_path)
model_id = self.__deployment_client.register_model(
model_name, model_path)
service = self.__deployment_client.create_service(
self.__service_name, model_id)
self.__client = PredictionClient(service.ipAddress, service.port) #pylint: disable=E1101
return input_map['InputImage'], tf.py_func(self._remote_service_call,
[input_map['InputImage']],
tf.float32)
def _remote_service_call(self, data):
return self.__client.score_numpy_array(data)
def cleanup_remote_service(self):
service = self.__deployment_client.get_service_by_name(
self.__service_name)
if service is not None:
print("Deleting service", service.id)
self.__deployment_client.delete_service(service.id)
print("Deleted service", service.id)
print("Deleting model", service.modelId)
self.__deployment_client.delete_model(service.modelId)
print("Deleted model", service.modelId)
def worker(IP, path):
datadir = os.path.expanduser(path)
pass_files= glob.glob(os.path.join(datadir, 'pass', '*.jpg'))
fail_files = glob.glob(os.path.join(datadir, 'fail', '*.jpg'))
print("Pass files")
for image in pass_files:
client = PredictionClient(IP, 80, False, '')
results = client.score_image(image)
print(image, results)
if (results < 0.5):
pdb.set_trace()
print(image, results)
print("Fail files")
for image in fail_files:
client = PredictionClient(IP, 80, False, '')
results = client.score_image(image)
print(image, results)
if (results >= 0.5):
pdb.set_trace()
print(image, results)
def test_create_update_and_delete_service():
override_token_funcs()
test_config = get_test_config()
deployment_client = DeploymentClient(
test_config['test_subscription_id'],
test_config['test_resource_group'],
test_config['test_model_management_account'])
cleanup_old_test_services(deployment_client)
id = uuid.uuid4().hex[:5]
model_name = "int-test-model-" + id
service_name = "int-test-service-" + id
service_def_path = "/tmp/model"
in_images = tf.placeholder(tf.string)
image_tensors = preprocess_array(in_images)
model = LocalQuantizedResNet50(os.path.expanduser("~/models"))
model.import_graph_def(include_featurizer=False)
service_def = ServiceDefinition()
service_def.pipeline.append(
TensorflowStage(tf.Session(), in_images, image_tensors))
service_def.pipeline.append(BrainWaveStage(model))
service_def.pipeline.append(
TensorflowStage(tf.Session(), model.classifier_input,
model.classifier_output))
service_def.save(service_def_path)
# create service
first_model_id = deployment_client.register_model(model_name,
service_def_path)
service = deployment_client.create_service(service_name, first_model_id)
service_list = deployment_client.list_services()
assert any(x.name == service_name for x in service_list)
prediction_client = PredictionClient(service.ipAddress, service.port)
top_result = sorted(enumerate(
prediction_client.score_image("/tmp/share1/shark.jpg")),
key=lambda x: x[1],
reverse=True)[:1]
# 'tiger shark' is class 3
assert top_result[0][0] == 3
# update service, remove classifier
service_def = ServiceDefinition()
service_def.pipeline.append(
TensorflowStage(tf.Session(), in_images, image_tensors))
service_def.pipeline.append(BrainWaveStage(model))
service_def.save(service_def_path)
second_model_id = deployment_client.register_model(model_name,
service_def_path)
deployment_client.update_service(service.id, second_model_id)
result = prediction_client.score_image("/tmp/share1/shark.jpg")
assert all([x == y for x, y in zip(np.array(result).shape, [1, 1, 2048])])
# wait for timeout of Azure LB
time.sleep(4 * 60 + 10)
result = prediction_client.score_image("/tmp/share1/shark.jpg")
assert all([x == y for x, y in zip(np.array(result).shape, [1, 1, 2048])])
deployment_client.delete_service(service.id)
deployment_client.delete_model(first_model_id)
deployment_client.delete_model(second_model_id)
def test_create_client_raises_if_port_is_none():
with pytest.raises(ValueError):
PredictionClient("localhost", None)
def test_create_client_raises_if_host_is_none():
with pytest.raises(ValueError):
PredictionClient(None, 50051)
def test_create_client_with_auth():
client = PredictionClient("localhost", 50051, True, "key1")
assert client is not None
def test_create_client():
client = PredictionClient("localhost", 50051)
assert client is not None
import requests
from amlrealtimeai.client import PredictionClient
import argparse
parse = argparse.ArgumentParser(description='AML inferencing client')
parse.add_argument('IP', type=str, help='IP of the FPGA runtime node')
parse.add_argument('path',
type=str,
help='Path to image that need to inferenced')
arg = parse.parse_args()
IP = arg.IP
path = arg.path
client = PredictionClient(IP, 80, False, '')
results = client.score_image(path)
#print(results)
trained_ds = requests.get(
"https://raw.githubusercontent.com/Lasagne/Recipes/master/examples/resnet50/imagenet_classes.txt"
).text.splitlines()
#for line in trained_ds:
# print(line)
# map results [class_id] => [confidence]
results = enumerate(results)
#for line in results:
# print(line)
# sort results by confidence
sorted_results = sorted(results, key=lambda x: x[1], reverse=True)
#for line in sorted_results:
# print(line)
