def make_request_fn():
"""Returns a request function."""
request_fn = serving_utils.make_grpc_request_fn(
servable_name="transformer",
server="172.16.11.202:9000",
timeout_secs=10)
return request_fn
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
validate_flags()
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
problem = registry.problem(FLAGS.problem)
hparams = tf.contrib.training.HParams(
data_dir=os.path.expanduser(FLAGS.data_dir))
problem.get_hparams(hparams)
if FLAGS.cloud_mlengine_model_name:
request_fn = serving_utils.make_cloud_mlengine_request_fn(
credentials=GoogleCredentials.get_application_default(),
model_name=FLAGS.cloud_mlengine_model_name,
version=FLAGS.cloud_mlengine_model_version)
else:
request_fn = serving_utils.make_grpc_request_fn(
servable_name=FLAGS.servable_name,
server=FLAGS.server,
timeout_secs=FLAGS.timeout_secs)
while True:
inputs = FLAGS.inputs_once if FLAGS.inputs_once else input(">> ")
outputs = serving_utils.predict([inputs], problem, request_fn)
print_str = """
Input:
{inputs}
Output:
{outputs}
"""
print(print_str.format(inputs=inputs, outputs=outputs[0]))
if FLAGS.inputs_once:
break
def __init__(self, translate_host, translate_port, source_lang, target_lang, model_name, problem, t2t_usr_dir, data_dir, preprocess_cmd, postprocess_cmd):
"""Initialize a TransformerTranslator object according to the given
configuration settings.
@param translate_port: the port at which the Moses translator operates
@param recase_port: the port at which the recaser operates
@param source_lang: source language (ISO-639-1 ID)
@param target_lang: target language (ISO-639-1 ID)
@param preprocess_cmd: bash command for text preprocessing
@param postprocess_cmd: bash command for text posprocessing
"""
# precompile Tensorflow server addresses
self.server = translate_host + ":" + translate_port
# initialize text processing tools (can be shared among threads)
self.tokenizer = Tokenizer({'lowercase': True,
'moses_escape': True})
self.preprocess = preprocess_cmd
self.postprocess = postprocess_cmd
usr_dir.import_usr_dir(t2t_usr_dir)
self.problem = registry.problem(problem)
hparams = tf.contrib.training.HParams(
data_dir=os.path.expanduser(data_dir))
self.problem.get_hparams(hparams)
self.request_fn = serving_utils.make_grpc_request_fn(
servable_name=model_name,
server=self.server,
timeout_secs=30)
def send_sentences_to_backend(self, sentences, src, tgt):
if self.prefix_with:
prefix = self.prefix_with.format(source=src, target=tgt)
else:
prefix = ''
outputs = []
request_fn = serving_utils.make_grpc_request_fn(
servable_name=self.model, timeout_secs=500, server=self.server)
for batch in np.array_split(
sentences,
ceil(len(sentences) / current_app.config['BATCH_SIZE'])):
try:
outputs += list(
map(
lambda sent_score: sent_score[0],
serving_utils.predict(
[prefix + sent for sent in batch.tolist()],
self.problem, request_fn)))
except:
# When tensorflow serving restarts web clients seem to "remember" the channel where
# the connection have failed. clearing up the session, seems to solve that
session.clear()
raise
return outputs
def __init__(self, translate_host, translate_port, source_lang,
target_lang, model_name, problem, t2t_usr_dir, data_dir,
preprocess_cmd, postprocess_cmd):
"""Initialize a TransformerTranslator object according to the given
configuration settings.
@param translate_port: the port at which the Moses translator operates
@param recase_port: the port at which the recaser operates
@param source_lang: source language (ISO-639-1 ID)
@param target_lang: target language (ISO-639-1 ID)
@param preprocess_cmd: bash command for text preprocessing
@param postprocess_cmd: bash command for text posprocessing
"""
# precompile Tensorflow server addresses
self.server = translate_host + ":" + translate_port
# initialize text processing tools (can be shared among threads)
self.tokenizer = Tokenizer({'lowercase': True, 'moses_escape': True})
self.preprocess = preprocess_cmd
self.postprocess = postprocess_cmd
usr_dir.import_usr_dir(t2t_usr_dir)
self.problem = registry.problem(problem)
hparams = tf.contrib.training.HParams(
data_dir=os.path.expanduser(data_dir))
self.problem.get_hparams(hparams)
self.request_fn = serving_utils.make_grpc_request_fn(
servable_name=model_name, server=self.server, timeout_secs=30)
def _do_send_request(self, text_arr):
"""
Divide the arr into batches and send the batches to the backend to be processed
:param text_arr: individual elements of arr will be grouped into batches
:return:
"""
outputs = []
request_fn = serving_utils.make_grpc_request_fn(
servable_name=self.model, timeout_secs=500, server=self.server)
for batch in np.array_split(text_arr,
ceil(len(text_arr) / self.batch_size)):
try:
models.log.debug(f"===== sending batch\n{pformat(batch)}\n")
outputs += list(
map(
lambda sent_score: sent_score[0],
serving_utils.predict(batch.tolist(), self.problem,
request_fn)))
except:
# When tensorflow serving restarts web clients seem to "remember" the channel where
# the connection have failed. clearing up the session, seems to solve that
session.clear()
raise
return outputs
def my_make_request_fn(servable_name, server):
"""Returns a request function."""
request_fn = serving_utils.make_grpc_request_fn(
servable_name=servable_name,
server=server,
timeout_secs=FLAGS.timeout_secs)
return request_fn
def make_request_fn(self):
"""Returns a request function."""
request_fn = serving_utils.make_grpc_request_fn(
servable_name=self.servable_name,
server=self.server,
timeout_secs=10)
return request_fn
def make_request_fn():
"""Returns a request function."""
if True:
request_fn = serving_utils.make_grpc_request_fn(
servable_name=servable_name,
server=server + port,
timeout_secs=100)
#print(request_fn,'<-6')
return request_fn
def test_e2e_export_and_query(self):
"""Test that we can export and query the model via tf.serving."""
FLAGS.t2t_usr_dir = _get_t2t_usr_dir()
FLAGS.problem = "github_function_docstring"
FLAGS.data_dir = "/mnt/nfs-east1-d/data"
FLAGS.tmp_dir = "/mnt/nfs-east1-d/tmp"
FLAGS.output_dir = tempfile.mkdtemp()
#FLAGS.export_dir = os.path.join(FLAGS.output_dir, "export")
FLAGS.model = "similarity_transformer_dev"
FLAGS.hparams_set = "similarity_transformer_tiny"
FLAGS.train_steps = 1
FLAGS.schedule = "train"
timeout_secs = 10
usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
t2t_trainer.main(None)
export.main(None)
# ----
# Start model server
# Will start a tf model server on an un-used port and
# kill process on exit.
_, server, _ = TensorflowModelServer().RunServer(
FLAGS.model, FLAGS.output_dir)
# ----
# Query the server
return
doc_query = [1, 2, 3] # Dummy encoded doc query
code_query = [1, 2, 3] # Dummy encoded code query
# Alternatively for query, without going through query.main()
# TODO: Is servable_name the same as model name?
request_fn = serving_utils.make_grpc_request_fn(
servable_name=FLAGS.model,
server=server,
timeout_secs=timeout_secs)
# Compute embeddings
# TODO: May need to customize how these queries are fed in, potentially
# side-stepping serving_utils.predict.
encoded_string = serving_utils.predict([doc_query], problem_object,
request_fn)
encoded_code = serving_utils.predict([code_query], problem_object,
request_fn)
def make_request_fn():
"""Returns a request function."""
if FLAGS.cloud_mlengine_model_name:
request_fn = serving_utils.make_cloud_mlengine_request_fn(
credentials=GoogleCredentials.get_application_default(),
model_name=FLAGS.cloud_mlengine_model_name,
version=FLAGS.cloud_mlengine_model_version)
else:
request_fn = serving_utils.make_grpc_request_fn(
servable_name=FLAGS.servable_name,
server=FLAGS.server,
timeout_secs=FLAGS.timeout_secs)
return request_fn
def make_request_fn():
"""Returns a request function."""
if FLAGS.cloud_mlengine_model_name:
request_fn = serving_utils.make_cloud_mlengine_request_fn(
credentials=GoogleCredentials.get_application_default(),
model_name=FLAGS.cloud_mlengine_model_name,
version=FLAGS.cloud_mlengine_model_version)
else:
request_fn = serving_utils.make_grpc_request_fn(
servable_name=FLAGS.servable_name,
server=FLAGS.server,
timeout_secs=FLAGS.timeout_secs)
return request_fn
def batch_query_server(self, queries, server, hparams, timeout_secs=5):
"""Query a served model with a batch of multiple queries."""
problem = hparams.problem
request_fn = serving_utils.make_grpc_request_fn(
servable_name=self.model_name, server=server, timeout_secs=timeout_secs)
responses = []
for query in queries:
response = serving_utils.predict(query, problem, request_fn)
responses.append(response)
return responses
def make_request_fn():
request_fn = serving_utils.make_grpc_request_fn(
servable_name=FLAGS.servable_name,
server=FLAGS.server,
timeout_secs=FLAGS.timeout_secs)
return request_fn
def predict():
data = {"status": False}
if request.args.get('question'):
data["status"] = True
question = request.args.get('question')
print("************************************************************")
print("question: " + question)
print("************************************************************")
data["question"] = question
usr_dir.import_usr_dir("/home/saplab/aivivn-vn-tones/t2t")
problem = registry.problem("translate_vndt")
hparams = hparam.HParams(data_dir=os.path.expanduser(
"/home/saplab/aivivn-vn-tones/t2t_datagen"))
problem.get_hparams(hparams)
request_fn = serving_utils.make_grpc_request_fn(
servable_name="translate_vndt",
server="localhost:9000",
timeout_secs=10)
outputs = serving_utils.predict([question], problem, request_fn)
outputs, = outputs
output, score = outputs
answer = ""
for count in range(len(question)):
answer += (output[count])
question = answer
data["question"] = question
input_data = {
'options': {
'n_best': True,
'n_best_size': 3,
'max_answer_length': 30
},
'data': [{
'paragraphs': [{
'qas': [{
'question':
question,
'id':
27527,
'answers': [{
'answer_id': 25239,
'document_id': 37254,
'question_id': 27527,
'text': "",
'answer_start': 14,
}],
'is_impossible':
False,
}],
"context":
"Vi-rút corona mới là loại vi-rút corona mới chưa từng được phát hiện trước đây. Vi-rút gây ra bệnh vi-rút corona 2019 (COVID-19), không cùng loại vi-rút corona thường lan truyền ở người và gây ra bệnh nhẹ, giống như cảm lạnh thông thường. Vào ngày 11 tháng 2, 2020 Tổ Chức Y Tế Thế Giới đã tuyên bố tên chính thức cho căn bệnh đang gây bùng phát là vi-rút corona 2019 mới, lần đầu được xác định tại Vũ Hán, Trung Quốc. Tên mới cho căn bệnh này là bệnh vi-rút corona 2019, gọi tắt là COVID-19. Trong chữ COVID-19, 'CO' viết tắt của từ 'corona, VI' viết tắt của từ 'vi-rút,' và 'D' là bệnh. Trước đó, căn bệnh này được gọi là \\\"vi-rút corona mới 2019\\\" hoặc \\\"nCoV-2019\\\". Có nhiều loại vi-rút corona ở người bao gồm một số loại thường gây ra các chứng bệnh nhẹ ở đường hô hấp trên. COVID-19 là một bệnh mới, do một loại vi-rút corona mới chưa từng thấy ở người gây ra.",
"document_id":
37255
}]
}]
}
(eval_examples, eval_features) = process_inputs(input_data=input_data)
hostport = '127.0.0.1:8500'
channel = grpc.insecure_channel(hostport)
stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
model_request = predict_pb2.PredictRequest()
model_request.model_spec.name = 'MyQA'
record_iterator = \
tf.python_io.tf_record_iterator(path='/home/saplab/albert_vi/Data/predict_file1.rtf'
)
for string_record in record_iterator:
model_request.inputs['examples'].CopyFrom(
tf.contrib.util.make_tensor_proto(string_record,
dtype=tf.string,
shape=[1]))
result_future = stub.Predict.future(model_request, 30.0)
raw_result = result_future.result().outputs
formatted_result = process_result(raw_result)
re = process_output(
formatted_result,
eval_examples,
eval_features,
input_data,
n_best=True,
n_best_size=20,
max_answer_length=30,
)
data["answer"] = re
return json.dumps(data, ensure_ascii=False)
def make_request_fn(self, name):
request_fn = serving_utils.make_grpc_request_fn(
servable_name=name,
server=server_config + ':' + str(listen_port_config),
timeout_secs=100)
return request_fn
