def test_correct_signature(self):
runtime = TensorflowRuntime("models/tf_summator")
runtime.start(port="9090")
try:
time.sleep(1)
channel = grpc.insecure_channel('localhost:9090')
client = hs.PredictionServiceStub(channel=channel)
a = hs.TensorProto()
a.ParseFromString(
tf.contrib.util.make_tensor_proto(
3, dtype=tf.int8, shape=[]).SerializeToString())
b = hs.TensorProto()
b.ParseFromString(
tf.contrib.util.make_tensor_proto(
2, dtype=tf.int8, shape=[]).SerializeToString())
request = hs.PredictRequest(
model_spec=hs.ModelSpec(signature_name="add"),
inputs={
"a": a,
"b": b
})
result = client.Predict(request)
expected = hs.PredictResponse(
outputs={
"sum":
hs.TensorProto(dtype=hs.DT_INT8,
tensor_shape=hs.TensorShapeProto(),
int_val=[5])
})
self.assertEqual(result, expected)
finally:
runtime.stop()
def test_lstm(self):
runtime = TensorflowRuntime("models/lstm")
runtime.start(port="9090")
results = []
try:
time.sleep(1)
channel = grpc.insecure_channel('localhost:9090')
client = hs.PredictionServiceStub(channel=channel)
shape = hs.TensorShapeProto(dim=[
hs.TensorShapeProto.Dim(size=-1),
hs.TensorShapeProto.Dim(size=1),
hs.TensorShapeProto.Dim(size=24)
])
data_tensor = hs.TensorProto(
dtype=hs.DT_FLOAT,
tensor_shape=shape,
float_val=[x for x in np.random.random(24).tolist()])
for x in range(1, 5):
request = hs.PredictRequest(
model_spec=hs.ModelSpec(signature_name="infer"),
inputs={"data": data_tensor})
result = client.Predict(request)
results.append(result)
finally:
runtime.stop()
print(results)
def simulate_production_traffic(path="./data",
application_name="mnist_app",
host=None,
request_delay=1,
request_amount=10000,
imgs_file="imgs.npz",
labels_file="labels.npz",
shuffle=False):
# conn = psycopg2.connect(f"postgresql://{POSTGRES_USER}:{POSTGRES_PASS}@{POSTGRES_HOST}:{POSTGRES_PORT}/{POSTGRES_DB}")
# cur = conn.cursor()
# cur.execute('''
# CREATE TABLE IF NOT EXISTS
# requests (hex_uid varchar(256), ground_truth integer);
# ''')
# conn.commit()
if not host:
host = f"dev.k8s.hydrosphere.io"
creds = grpc.ssl_channel_credentials()
channel = grpc.secure_channel(host, creds)
# channel = grpc.insecure_channel(host)
stub = hs.PredictionServiceStub(channel)
# an application, that will be invoked
model_spec = hs.ModelSpec(name=application_name)
# basic shape for images
tensor_shape = hs.TensorShapeProto(dim=[
hs.TensorShapeProto.Dim(size=1),
hs.TensorShapeProto.Dim(size=28),
hs.TensorShapeProto.Dim(size=28),
hs.TensorShapeProto.Dim(size=1),
])
images, labels = generate_data(path,
imgs_file,
labels_file,
shuffle=shuffle)
for index, (image, label) in tqdm(enumerate(zip(images, labels)),
total=request_amount):
if index == request_amount: break
if not image.flags['C_CONTIGUOUS']:
image = np.ascontiguousarray(image)
# form a request and get a prediction
tensor = hs.TensorProto(dtype=hs.DT_FLOAT,
tensor_shape=tensor_shape,
float_val=image.flatten().tolist())
request = hs.PredictRequest(model_spec=model_spec,
inputs={"imgs": tensor})
stub.Predict(request)
# insert uid and ground_truth labels into database
# cur.execute("INSERT INTO requests VALUES (%s, %s)",
# (hashlib.sha1(image).hexdigest(), int(label)))
# conn.commit()
time.sleep(request_delay)
def predict(tokenized):
global graph
tokenized_sentence = np.array(tokenized.int64_val, ndmin=2)
with graph.as_default():
prediction = amazon_model.predict([tokenized_sentence])
confidence = prediction[0,0]
label = 1 if confidence >= 0.5 else 0
conf_tensor = hs.TensorProto(
double_val=[confidence],
dtype=hs.DT_DOUBLE,
tensor_shape=hs.TensorShapeProto())
label_tensor = hs.TensorProto(
int_val=[label],
dtype=hs.DT_INT32,
tensor_shape=hs.TensorShapeProto())
return hs.PredictResponse(outputs={'confidence': conf_tensor, 'label':label_tensor})
def postprocess(self, rpredictions, rlocalisations, rbbox_img):
rclasses, rscores, rbboxes = np_methods.ssd_bboxes_select(
rpredictions,
rlocalisations,
SSDServer.ssd_anchors,
select_threshold=SSDServer.SELECT_TRESHOLD,
img_shape=SSDServer.NET_SHAPE,
num_classes=21,
decode=True)
rbboxes = np_methods.bboxes_clip(rbbox_img, rbboxes)
rclasses, rscores, rbboxes = np_methods.bboxes_sort(
rclasses, rscores, rbboxes, top_k=SSDServer.TOP_K)
rclasses, rscores, rbboxes = np_methods.bboxes_nms(
rclasses, rscores, rbboxes, nms_threshold=SSDServer.NMS_TRESHOLD)
rbboxes = np_methods.bboxes_resize(rbbox_img, rbboxes)
rclasses = list(map(lambda c: SSDServer.VOC_MAP.get(c, "NA"),
rclasses))
class_arr = [bytes(x, "utf-8") for x in rclasses]
classes_tensor = hs.TensorProto(
dtype=hs.DT_STRING,
tensor_shape=hs.TensorShapeProto(
dim=[hs.TensorShapeProto.Dim(size=-1)]),
string_val=class_arr)
scores_tensor = hs.TensorProto(
dtype=hs.DT_DOUBLE,
tensor_shape=hs.TensorShapeProto(
dim=[hs.TensorShapeProto.Dim(size=-1)]),
double_val=rscores)
bboxes_tensor = hs.TensorProto(dtype=hs.DT_DOUBLE,
tensor_shape=hs.TensorShapeProto(dim=[
hs.TensorShapeProto.Dim(size=-1),
hs.TensorShapeProto.Dim(size=4)
]),
double_val=rbboxes.flatten())
return classes_tensor, scores_tensor, bboxes_tensor
def infer(features):
data = np.array(features.double_val) \
.reshape([dim.size for dim in features.tensor_shape.dim])
prediction = clf.predict(data)
guess_shape = hs.TensorShapeProto(
dim=[hs.TensorShapeProto.Dim(size=item) for item in prediction.shape])
guess = hs.TensorProto(dtype=hs.DT_BOOL,
bool_val=prediction,
tensor_shape=guess_shape)
return hs.PredictResponse(outputs={'is_fraud': guess})
def predict(**kwargs):
extracted = extract_value(kwargs['input'])
with graph.as_default():
probas = m.predict(extracted)
classes = np.array(probas).argmax(axis=0)
probas_proto = hs.TensorProto(
double_val=probas.flatten().tolist(),
dtype=hs.DT_DOUBLE,
tensor_shape=hs.TensorShapeProto(
dim=[hs.TensorShapeProto.Dim(size=-1), hs.TensorShapeProto.Dim(size=10)]))
classes_proto = hs.TensorProto(
int64_val=classes.flatten().tolist(),
dtype=hs.DT_INT64,
tensor_shape=hs.TensorShapeProto(
dim=[hs.TensorShapeProto.Dim(size=-1), hs.TensorShapeProto.Dim(size=1)]))
return hs.PredictResponse(outputs={"classes": classes_proto, "probabilities": probas_proto})
def predict(**kwargs):
extracted = np.array(
[extract_value(kwargs[feature]) for feature in features])
transformed = np.dstack(extracted).reshape(1, len(features))
predicted = monitoring_model.decision_function(transformed)
response = hs.TensorProto(double_val=[predicted.item()],
dtype=hs.DT_DOUBLE,
tensor_shape=hs.TensorShapeProto())
return hs.PredictResponse(outputs={"value": response})
def predict(**kwargs):
extracted = np.array(
[extract_value(kwargs[feature]) for feature in features])
transformed = np.dstack(extracted).reshape(1, len(features))
predicted = clf.predict(transformed)
response = hs.TensorProto(int64_val=[predicted.item()],
dtype=hs.DT_INT64,
tensor_shape=hs.TensorShapeProto())
return hs.PredictResponse(outputs={"classes": response})
def tokenize(text):
sentence = [str.decode() for str in text.string_val][0]
if 'www.' in sentence or 'http:' in sentence or 'https:' in sentence or '.com' in sentence:
sentence = re.sub(r"([^ ]+(?<=\.[a-z]{3}))", "<url>", sentence)
tok_sentence = tokenizer.texts_to_sequences([sentence])
pad_sentence = sequence.pad_sequences(tok_sentence, maxlen=maxlen)[0]
tok_tensor = hs.TensorProto(int64_val=pad_sentence,
dtype=hs.DT_INT64,
tensor_shape=hs.TensorShapeProto(
dim=[hs.TensorShapeProto.Dim(size=100)]))
return hs.PredictResponse(outputs={'tokenized': tok_tensor})
def predict(shaped_image):
result = model(shaped_image)
inference = result[0][0] * 1000 - 453
if (inference > 0.5):
prediction = "normal ({})".format(inference)
else:
prediction = "abnormal ({})".format(inference)
prediction_tensor_shape = hs.TensorShapeProto(dim=hs.TensorShapeProto(
size=1))
prediction_tensor_proto = hs.TensorProto\
(
dtype = hs.DT_STRING,
string_val = [prediction],
tensor_shape = prediction_tensor_shape
)
return hs.PredictResponse(outputs={"result": prediction_tensor_proto})
def infer(msg):
# TODO только для этого numpy?
msg_arr = np.array(msg.string_val)
msg_str = msg_arr[0]
tokens = infer_str(msg_str)
y = hs.TensorProto(
dtype=hs.DT_STRING,
string_val=[t.encode('utf-8', 'ignore') for t in tokens],
tensor_shape=hs.TensorShapeProto(
dim=[hs.TensorShapeProto.Dim(size=len(tokens))]))
# 3. Return the result
return hs.PredictResponse(outputs={"preprocessed_msg": y})
def simulate_production_traffic(host=None,
request_delay=2,
request_amount=10000,
file="combined.npz",
shuffle=False):
conn = psycopg2.connect(
f"postgresql://{USER}:{PASS}@{ADDRESS}:{PORT}/{DATABASE}")
cur = conn.cursor()
cur.execute('''
CREATE TABLE IF NOT EXISTS
requests (timestamp bigint, uid integer, ground_truth integer);
''')
if not host:
namespace = os.environ["NAMESPACE"]
host = f"hydro-serving-ui-{namespace}:9091"
channel = grpc.insecure_channel(host)
stub = hs.PredictionServiceStub(channel)
# an application, that will be invoked
model_spec = hs.ModelSpec(name="mnist-app")
# basic shape for images
tensor_shape = hs.TensorShapeProto(dim=[
hs.TensorShapeProto.Dim(size=1),
hs.TensorShapeProto.Dim(size=28),
hs.TensorShapeProto.Dim(size=28),
hs.TensorShapeProto.Dim(size=1),
])
images, labels = generate_data('/data', file, shuffle=shuffle)
for index, (image, label) in tqdm(enumerate(zip(images, labels)),
total=request_amount):
if index == request_amount: break
# form a request
tensor = hs.TensorProto(dtype=hs.DT_FLOAT,
tensor_shape=tensor_shape,
float_val=image.flatten().tolist())
request = hs.PredictRequest(model_spec=model_spec,
inputs={"imgs": tensor})
# get prediction
result = stub.Predict(request)
# insert trace_id and ground_truth labels into database
cur.execute("INSERT INTO requests VALUES (%s, %s, %s)",
(result.trace_data.ts, result.trace_data.uid, int(label)))
conn.commit()
time.sleep(request_delay)
def normalize(data):
print(data)
max_val = max(data.double_val)
norm_data = [float(x) / max_val for x in data.double_val]
shape = hs.TensorShapeProto(dim=[
hs.TensorShapeProto.Dim(size=1),
hs.TensorShapeProto.Dim(size=1),
hs.TensorShapeProto.Dim(size=24)
])
data = hs.TensorProto(dtype=hs.DT_DOUBLE,
double_val=norm_data,
tensor_shape=shape)
print(data)
return hs.PredictResponse(outputs={"data": data})
def increment(number): # <- keep in mind the signature
request_number = tf.make_ndarray(number)
response_number = request_number + 1
response_tensor_shape = [
hs.TensorShapeProto.Dim(size=dim.size)
for dim in number.tensor_shape.dim
]
response_tensor = hs.TensorProto(
int_val=response_number.flatten(),
dtype=hs.DT_INT32,
tensor_shape=hs.TensorShapeProto(dim=response_tensor_shape))
return hs.PredictResponse(outputs={"number": response_tensor})
def predict(sample):
output = model(sample)
if (output[0] > output[1]):
prediction = [b"normal"]
else:
prediction = [b"abnormal"]
response_tensor_shape = hs.TensorShapeProto(dim=hs.TensorShapeProto.Dim(
size=1))
return hs.PredictResponse(outputs = {"result": hs.TensorProto(dtype = DT_STRING,\
string_val = prediction, tensor_shape = response_tensor_shape)})
def infer(pclass, sex, age, fare, parch):
df = pd.DataFrame({
'Pclass': pclass.int_val,
'Sex': sex.string_val,
'Age': age.int_val,
'Fare': fare.double_val,
'Parch': parch.int_val
})
df['Sex'] = df['Sex'].map({'male': 0, 'female': 1}).to_frame()
score = gbm.predict(df.values)
tensor = hs.TensorProto(dtype=hs.DT_INT32,
int_val=score,
tensor_shape=hs.TensorShapeProto(
dim=[hs.TensorShapeProto.Dim(size=-1)]))
return hs.PredictResponse(outputs={"survived": tensor})
def convert_tf_signature():
with tf.Session() as sess:
meta_graph = tf.saved_model.loader.load(
sess, [tf.saved_model.tag_constants.SERVING],
"models/tf_summator")
signatures = []
for name, sig in meta_graph.signature_def.items():
inputs = []
for inp_name, inp in sig.inputs.items():
shape = hs.TensorShapeProto()
shape.ParseFromString(inp.tensor_shape.SerializeToString())
field = hs.ModelField(field_name=inp_name,
type=inp.dtype,
shape=shape)
inputs.append(field)
outputs = []
for out_name, out in sig.outputs.items():
shape = hs.TensorShapeProto()
shape.ParseFromString(out.tensor_shape.SerializeToString())
field = hs.ModelField(field_name=out_name,
type=out.dtype,
shape=shape)
outputs.append(field)
schema_signature = hs.ModelSignature(signature_name=name,
inputs=inputs,
outputs=outputs)
signatures.append(schema_signature)
model_def = hs.ModelContract(model_name="tf_summator",
signatures=signatures)
with open('models/tf_summator/contract.protobin', 'wb') as f:
f.write(model_def.SerializeToString())
with open('models/tf_summator/contract.prototxt', 'w') as f:
f.write(str(model_def))
with open('models/tf_summator/contract.original.prototxt',
'w') as f:
f.write(str(meta_graph))
def infer(**kwargs):
tensor = kwargs["x"]
data = np.array(tensor.double_val).reshape(1, 300, 3, 3)
result = 0
for i in range(data.shape[1]):
with graph.as_default():
vector = data[0, i, :, :].reshape(1, 9)
rec_vector = model.predict(vector)
result += np.mean(np.square(vector - rec_vector), axis=0)
y_tensor = hs.TensorProto(dtype=hs.DT_DOUBLE,
double_val=result.flatten().tolist(),
tensor_shape=hs.TensorShapeProto())
# 4. Return the result
return hs.PredictResponse(outputs={"value": y_tensor})
def tokenize(x):
sentences = np.array(x.string_val)
sentences = sentences.reshape([dim.size for dim in x.tensor_shape.dim])
tokenized = np.copy(sentences)
for index, sentence in enumerate(sentences):
tokenized[index] = " ".join(
nltk.word_tokenize(str(sentence[0], encoding="utf-8").lower()))
tokenized = hs.TensorProto(dtype=hs.DT_STRING,
string_val=tokenized.flatten(),
tensor_shape=hs.TensorShapeProto(dim=[
hs.TensorShapeProto.Dim(size=-1),
hs.TensorShapeProto.Dim(size=1)
]))
return hs.PredictResponse(outputs={"input_data": tokenized})
def infer(x):
# 1. Retrieve tensor's content and put it to numpy array
data = np.array(x.double_val)
data = data.reshape([dim.size for dim in x.tensor_shape.dim])
# 2. Make a prediction
with graph.as_default():
result = model.predict(data)
print(result)
# 3. Pack the answer
y_shape = hs.TensorShapeProto(dim=[hs.TensorShapeProto.Dim(size=5)])
y_tensor = hs.TensorProto(dtype=hs.DT_DOUBLE,
double_val=result.flatten(),
tensor_shape=y_shape)
# 4. Return the result
return hs.PredictResponse(outputs={"y": y_tensor})
def infer(**kwargs):
# use graph with initialized model in it
with graph.as_default():
features = kwargs['X']
data = np.array(features.double_val) \
.reshape([dim.size for dim in features.tensor_shape.dim])
predicted = autoencoder.predict(data)
score = np.mean(np.square(predicted - data), axis=1)
response_shape = hs.TensorShapeProto(
dim=[hs.TensorShapeProto.Dim(size=item) for item in (1, 1)])
response_tensor = hs.TensorProto(dtype=hs.DT_DOUBLE,
double_val=np.expand_dims(
score.flatten(), axis=0),
tensor_shape=response_shape)
return hs.PredictResponse(outputs={"reconstructed": response_tensor})
def infer(**kwargs):
df = pd.DataFrame({
'Pclass': kwargs['pclass'].int_val,
'Sex': kwargs['sex'].string_val,
'Age': kwargs['age'].int_val,
'Fare': kwargs['fare'].double_val,
'Parch': kwargs['parch'].int_val
})
df['Sex'] = df['Sex'].map({'male': 0, 'female': 1}).to_frame()
score = gbm.predict(df.values)
return hs.PredictResponse(
outputs={
"survived":
hs.TensorProto(dtype=hs.DT_INT32,
int_val=score,
tensor_shape=hs.TensorShapeProto(
dim=[hs.TensorShapeProto.Dim(size=-1)]))
})
def infer(faces):
img_data = np.array(faces.int_val)
imgs = img_data.reshape([dim.size for dim in faces.tensor_shape.dim])
# preprocess imgs for facenet
for i in range(len(imgs)):
imgs[i] = prewhiten(imgs[i])
with graph.as_default():
with tf.Session(graph=graph) as sess:
feed_dict = {images_placeholder: imgs, phase_train_placeholder: False}
emb_array = sess.run(embeddings, feed_dict=feed_dict)
predictions = classifier.predict_proba(emb_array)
class_indices = np.argmax(predictions, axis=1)
classes = [class_names[index] for index in class_indices]
y_shape = hs.TensorShapeProto(dim=[hs.TensorShapeProto.Dim(size=len(classes))])
class_prediciton = hs.TensorProto(
dtype=hs.DT_STRING,
string_val=[class_name.encode() for class_name in classes],
tensor_shape=y_shape)
return hs.PredictResponse(outputs={'y': class_prediciton})
def detect(x):
data = np.array(x.int_val, dtype=np.uint8)
img = data.reshape([dim.size for dim in x.tensor_shape.dim])
dets = detector(img, 1)
boxes = dlib.full_object_detections()
for i, detection in enumerate(dets):
boxes.append(sp(img, detection))
faces = []
for i in range(len(boxes)):
faces.append(dlib.get_face_chip(img, boxes[i], size=160))
faces = np.array(faces)
faces_shape = hs.TensorShapeProto(dim=[hs.TensorShapeProto.Dim(size=item) for item in faces.shape])
faces_tensor = hs.TensorProto(
dtype=hs.DT_UINT8,
int_val=faces.flatten(),
tensor_shape=faces_shape
)
return hs.PredictResponse(outputs={'faces': faces_tensor})
def tensorflow_case(self, tf_version):
docker_client = docker.from_env()
container = docker_client.containers.run(
"hydrosphere/serving-runtime-tensorflow:{}-latest".format(tf_version),
remove=True, detach=True,
ports={'9090/tcp': 9090},
volumes={os.path.abspath('models/tf_summator'): {'bind': '/model', 'mode': 'ro'}}
)
time.sleep(15)
try:
channel = grpc.insecure_channel('localhost:9090')
client = hs.PredictionServiceStub(channel=channel)
a = hs.TensorProto()
a.ParseFromString(tf.contrib.util.make_tensor_proto(3, dtype=tf.int8).SerializeToString())
b = hs.TensorProto()
b.ParseFromString(tf.contrib.util.make_tensor_proto(2, dtype=tf.int8).SerializeToString())
request = hs.PredictRequest(
model_spec=hs.ModelSpec(signature_name="add"),
inputs={
"a": a,
"b": b
}
)
result = client.Predict(request)
expected = hs.PredictResponse(
outputs={
"sum": hs.TensorProto(
dtype=hs.DT_INT8,
tensor_shape=hs.TensorShapeProto(),
int_val=[5]
)
}
)
self.assertEqual(result, expected)
finally:
print("Container logs:")
print(container.logs().decode("utf-8"))
container.stop()
time.sleep(15)
import grpc
import hydro_serving_grpc as hs # pip install hydro-serving-grpc
# connect to your ML Lamba instance
channel = grpc.insecure_channel("localhost")
stub = hs.PredictionServiceStub(channel)
# 1. define a model, that you'll use
model_spec = hs.ModelSpec(name="linear_regression", signature_name="infer")
# 2. define tensor_shape for Tensor instance
tensor_shape = hs.TensorShapeProto(
dim=[
hs.TensorShapeProto.Dim(size=-1),
hs.TensorShapeProto.Dim(size=2)
]
)
# 3. define tensor with needed data
tensor = hs.TensorProto(dtype=hs.DT_DOUBLE, tensor_shape=tensor_shape, double_val=[1,1,1,1])
# 4. create PredictRequest instance
request = hs.PredictRequest(model_spec=model_spec, inputs={"x": tensor})
# call Predict method
result = stub.Predict(request)
def pack_predict(result):
tensor = hs.TensorProto(dtype=hs.DT_DOUBLE,
double_val=[result],
tensor_shape=hs.TensorShapeProto())
return hs.PredictResponse(outputs={"value": tensor})
request_id=trace_id,
latency=0.014,
)
# 2. Create a PredictRequest message. PredictRequest is used to define the data
# passed to the model for inference.
predict_request_proto = hs.PredictRequest(
model_spec=hs.ModelSpec(
name="external-model-example",
signature_name="predict",
),
inputs={
"in": hs.TensorProto(
dtype=hs.DT_DOUBLE,
double_val=[random.random()],
tensor_shape=hs.TensorShapeProto()
),
},
)
# 3. Create a PredictResponse message. PredictResponse is used to define the
# outputs of the model inference.
predict_response_proto = hs.PredictResponse(
outputs={
"out": hs.TensorProto(
dtype=hs.DT_DOUBLE,
double_val=[random.random()],
tensor_shape=hs.TensorShapeProto()
),
},
)
