def make_request_fn(): """Returns a request function.""" request_fn = serving_utils.make_grpc_request_fn( servable_name="transformer", server="172.16.11.202:9000", timeout_secs=10) return request_fn
def main(_): tf.logging.set_verbosity(tf.logging.INFO) validate_flags() usr_dir.import_usr_dir(FLAGS.t2t_usr_dir) problem = registry.problem(FLAGS.problem) hparams = tf.contrib.training.HParams( data_dir=os.path.expanduser(FLAGS.data_dir)) problem.get_hparams(hparams) if FLAGS.cloud_mlengine_model_name: request_fn = serving_utils.make_cloud_mlengine_request_fn( credentials=GoogleCredentials.get_application_default(), model_name=FLAGS.cloud_mlengine_model_name, version=FLAGS.cloud_mlengine_model_version) else: request_fn = serving_utils.make_grpc_request_fn( servable_name=FLAGS.servable_name, server=FLAGS.server, timeout_secs=FLAGS.timeout_secs) while True: inputs = FLAGS.inputs_once if FLAGS.inputs_once else input(">> ") outputs = serving_utils.predict([inputs], problem, request_fn) print_str = """ Input: {inputs} Output: {outputs} """ print(print_str.format(inputs=inputs, outputs=outputs[0])) if FLAGS.inputs_once: break
def __init__(self, translate_host, translate_port, source_lang, target_lang, model_name, problem, t2t_usr_dir, data_dir, preprocess_cmd, postprocess_cmd): """Initialize a TransformerTranslator object according to the given configuration settings. @param translate_port: the port at which the Moses translator operates @param recase_port: the port at which the recaser operates @param source_lang: source language (ISO-639-1 ID) @param target_lang: target language (ISO-639-1 ID) @param preprocess_cmd: bash command for text preprocessing @param postprocess_cmd: bash command for text posprocessing """ # precompile Tensorflow server addresses self.server = translate_host + ":" + translate_port # initialize text processing tools (can be shared among threads) self.tokenizer = Tokenizer({'lowercase': True, 'moses_escape': True}) self.preprocess = preprocess_cmd self.postprocess = postprocess_cmd usr_dir.import_usr_dir(t2t_usr_dir) self.problem = registry.problem(problem) hparams = tf.contrib.training.HParams( data_dir=os.path.expanduser(data_dir)) self.problem.get_hparams(hparams) self.request_fn = serving_utils.make_grpc_request_fn( servable_name=model_name, server=self.server, timeout_secs=30)
def send_sentences_to_backend(self, sentences, src, tgt): if self.prefix_with: prefix = self.prefix_with.format(source=src, target=tgt) else: prefix = '' outputs = [] request_fn = serving_utils.make_grpc_request_fn( servable_name=self.model, timeout_secs=500, server=self.server) for batch in np.array_split( sentences, ceil(len(sentences) / current_app.config['BATCH_SIZE'])): try: outputs += list( map( lambda sent_score: sent_score[0], serving_utils.predict( [prefix + sent for sent in batch.tolist()], self.problem, request_fn))) except: # When tensorflow serving restarts web clients seem to "remember" the channel where # the connection have failed. clearing up the session, seems to solve that session.clear() raise return outputs
def __init__(self, translate_host, translate_port, source_lang, target_lang, model_name, problem, t2t_usr_dir, data_dir, preprocess_cmd, postprocess_cmd): """Initialize a TransformerTranslator object according to the given configuration settings. @param translate_port: the port at which the Moses translator operates @param recase_port: the port at which the recaser operates @param source_lang: source language (ISO-639-1 ID) @param target_lang: target language (ISO-639-1 ID) @param preprocess_cmd: bash command for text preprocessing @param postprocess_cmd: bash command for text posprocessing """ # precompile Tensorflow server addresses self.server = translate_host + ":" + translate_port # initialize text processing tools (can be shared among threads) self.tokenizer = Tokenizer({'lowercase': True, 'moses_escape': True}) self.preprocess = preprocess_cmd self.postprocess = postprocess_cmd usr_dir.import_usr_dir(t2t_usr_dir) self.problem = registry.problem(problem) hparams = tf.contrib.training.HParams( data_dir=os.path.expanduser(data_dir)) self.problem.get_hparams(hparams) self.request_fn = serving_utils.make_grpc_request_fn( servable_name=model_name, server=self.server, timeout_secs=30)
def _do_send_request(self, text_arr): """ Divide the arr into batches and send the batches to the backend to be processed :param text_arr: individual elements of arr will be grouped into batches :return: """ outputs = [] request_fn = serving_utils.make_grpc_request_fn( servable_name=self.model, timeout_secs=500, server=self.server) for batch in np.array_split(text_arr, ceil(len(text_arr) / self.batch_size)): try: models.log.debug(f"===== sending batch\n{pformat(batch)}\n") outputs += list( map( lambda sent_score: sent_score[0], serving_utils.predict(batch.tolist(), self.problem, request_fn))) except: # When tensorflow serving restarts web clients seem to "remember" the channel where # the connection have failed. clearing up the session, seems to solve that session.clear() raise return outputs
def my_make_request_fn(servable_name, server): """Returns a request function.""" request_fn = serving_utils.make_grpc_request_fn( servable_name=servable_name, server=server, timeout_secs=FLAGS.timeout_secs) return request_fn
def make_request_fn(self): """Returns a request function.""" request_fn = serving_utils.make_grpc_request_fn( servable_name=self.servable_name, server=self.server, timeout_secs=10) return request_fn
def make_request_fn(): """Returns a request function.""" if True: request_fn = serving_utils.make_grpc_request_fn( servable_name=servable_name, server=server + port, timeout_secs=100) #print(request_fn,'<-6') return request_fn
def test_e2e_export_and_query(self): """Test that we can export and query the model via tf.serving.""" FLAGS.t2t_usr_dir = _get_t2t_usr_dir() FLAGS.problem = "github_function_docstring" FLAGS.data_dir = "/mnt/nfs-east1-d/data" FLAGS.tmp_dir = "/mnt/nfs-east1-d/tmp" FLAGS.output_dir = tempfile.mkdtemp() #FLAGS.export_dir = os.path.join(FLAGS.output_dir, "export") FLAGS.model = "similarity_transformer_dev" FLAGS.hparams_set = "similarity_transformer_tiny" FLAGS.train_steps = 1 FLAGS.schedule = "train" timeout_secs = 10 usr_dir.import_usr_dir(FLAGS.t2t_usr_dir) t2t_trainer.main(None) export.main(None) # ---- # Start model server # Will start a tf model server on an un-used port and # kill process on exit. _, server, _ = TensorflowModelServer().RunServer( FLAGS.model, FLAGS.output_dir) # ---- # Query the server return doc_query = [1, 2, 3] # Dummy encoded doc query code_query = [1, 2, 3] # Dummy encoded code query # Alternatively for query, without going through query.main() # TODO: Is servable_name the same as model name? request_fn = serving_utils.make_grpc_request_fn( servable_name=FLAGS.model, server=server, timeout_secs=timeout_secs) # Compute embeddings # TODO: May need to customize how these queries are fed in, potentially # side-stepping serving_utils.predict. encoded_string = serving_utils.predict([doc_query], problem_object, request_fn) encoded_code = serving_utils.predict([code_query], problem_object, request_fn)
def make_request_fn(): """Returns a request function.""" if FLAGS.cloud_mlengine_model_name: request_fn = serving_utils.make_cloud_mlengine_request_fn( credentials=GoogleCredentials.get_application_default(), model_name=FLAGS.cloud_mlengine_model_name, version=FLAGS.cloud_mlengine_model_version) else: request_fn = serving_utils.make_grpc_request_fn( servable_name=FLAGS.servable_name, server=FLAGS.server, timeout_secs=FLAGS.timeout_secs) return request_fn
def make_request_fn(): """Returns a request function.""" if FLAGS.cloud_mlengine_model_name: request_fn = serving_utils.make_cloud_mlengine_request_fn( credentials=GoogleCredentials.get_application_default(), model_name=FLAGS.cloud_mlengine_model_name, version=FLAGS.cloud_mlengine_model_version) else: request_fn = serving_utils.make_grpc_request_fn( servable_name=FLAGS.servable_name, server=FLAGS.server, timeout_secs=FLAGS.timeout_secs) return request_fn
def batch_query_server(self, queries, server, hparams, timeout_secs=5): """Query a served model with a batch of multiple queries.""" problem = hparams.problem request_fn = serving_utils.make_grpc_request_fn( servable_name=self.model_name, server=server, timeout_secs=timeout_secs) responses = [] for query in queries: response = serving_utils.predict(query, problem, request_fn) responses.append(response) return responses
def make_request_fn(): request_fn = serving_utils.make_grpc_request_fn( servable_name=FLAGS.servable_name, server=FLAGS.server, timeout_secs=FLAGS.timeout_secs) return request_fn
def predict(): data = {"status": False} if request.args.get('question'): data["status"] = True question = request.args.get('question') print("************************************************************") print("question: " + question) print("************************************************************") data["question"] = question usr_dir.import_usr_dir("/home/saplab/aivivn-vn-tones/t2t") problem = registry.problem("translate_vndt") hparams = hparam.HParams(data_dir=os.path.expanduser( "/home/saplab/aivivn-vn-tones/t2t_datagen")) problem.get_hparams(hparams) request_fn = serving_utils.make_grpc_request_fn( servable_name="translate_vndt", server="localhost:9000", timeout_secs=10) outputs = serving_utils.predict([question], problem, request_fn) outputs, = outputs output, score = outputs answer = "" for count in range(len(question)): answer += (output[count]) question = answer data["question"] = question input_data = { 'options': { 'n_best': True, 'n_best_size': 3, 'max_answer_length': 30 }, 'data': [{ 'paragraphs': [{ 'qas': [{ 'question': question, 'id': 27527, 'answers': [{ 'answer_id': 25239, 'document_id': 37254, 'question_id': 27527, 'text': "", 'answer_start': 14, }], 'is_impossible': False, }], "context": "Vi-rút corona mới là loại vi-rút corona mới chưa từng được phát hiện trước đây. Vi-rút gây ra bệnh vi-rút corona 2019 (COVID-19), không cùng loại vi-rút corona thường lan truyền ở người và gây ra bệnh nhẹ, giống như cảm lạnh thông thường. Vào ngày 11 tháng 2, 2020 Tổ Chức Y Tế Thế Giới đã tuyên bố tên chính thức cho căn bệnh đang gây bùng phát là vi-rút corona 2019 mới, lần đầu được xác định tại Vũ Hán, Trung Quốc. Tên mới cho căn bệnh này là bệnh vi-rút corona 2019, gọi tắt là COVID-19. Trong chữ COVID-19, 'CO' viết tắt của từ 'corona, VI' viết tắt của từ 'vi-rút,' và 'D' là bệnh. Trước đó, căn bệnh này được gọi là \\\"vi-rút corona mới 2019\\\" hoặc \\\"nCoV-2019\\\". Có nhiều loại vi-rút corona ở người bao gồm một số loại thường gây ra các chứng bệnh nhẹ ở đường hô hấp trên. COVID-19 là một bệnh mới, do một loại vi-rút corona mới chưa từng thấy ở người gây ra.", "document_id": 37255 }] }] } (eval_examples, eval_features) = process_inputs(input_data=input_data) hostport = '127.0.0.1:8500' channel = grpc.insecure_channel(hostport) stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) model_request = predict_pb2.PredictRequest() model_request.model_spec.name = 'MyQA' record_iterator = \ tf.python_io.tf_record_iterator(path='/home/saplab/albert_vi/Data/predict_file1.rtf' ) for string_record in record_iterator: model_request.inputs['examples'].CopyFrom( tf.contrib.util.make_tensor_proto(string_record, dtype=tf.string, shape=[1])) result_future = stub.Predict.future(model_request, 30.0) raw_result = result_future.result().outputs formatted_result = process_result(raw_result) re = process_output( formatted_result, eval_examples, eval_features, input_data, n_best=True, n_best_size=20, max_answer_length=30, ) data["answer"] = re return json.dumps(data, ensure_ascii=False)
def make_request_fn(self, name): request_fn = serving_utils.make_grpc_request_fn( servable_name=name, server=server_config + ':' + str(listen_port_config), timeout_secs=100) return request_fn