def evaluate_line(): config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) # while True: # try: # line = input("请输入测试句子:") # result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) # print(result) # except Exception as e: # logger.info(e) # line = input("请输入测试句子:") line = "哮喘古代文献也称“鼻息”、“肩息”、“上气”等。" result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) print(line) print([(x["word"], x["type"]) for x in result["entities"]]) line = "喘病是指由于外感或内伤,导致肺失宣降,肺气上逆或气无所主,肾失摄纳,以致呼吸困难,甚则张口抬肩,鼻翼煽动,不能平卧等为主要临床特征的一种病证。 " result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) print(line) print([(x["word"], x["type"]) for x in result["entities"]])
def evaluate_file(file, target): config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, NERModel, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) with open(file, encoding='u8') as fr, open(target, mode='w', encoding='u8') as fw: for line in fr: result = model.evaluate_line( sess, input_from_line(line.strip(), char_to_id), id_to_tag) print(result) fw.write(json.dumps(result, ensure_ascii=False)) fw.write("\n") fw.flush() while True: line = input("请输入测试句子:") result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) print(result)
def evaluate_line(testfile): config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) Test = pd.read_csv(testfile) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) # while True: # line = input("请输入测试句子:") # result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) # print(result) ID = [] entities = [] for i in tqdm(range(len(Test))): id = Test.iloc[i]['id'] ID.append(id) print('id', id) #标题文本识别 sentence_title = str(Test.iloc[i]['title']) # print('sentence_title',sentence_title) if sentence_title != 'nan': #evaluate 返回的为list result_title = model.evaluate_line( sess, input_from_line(sentence_title, char_to_id), id_to_tag) # for item in result_title['entities']: # title.append(item['word'].strip().strip('"').strip()) #文本实体识别 sentence_text = str(Test.iloc[i]['text']) # print('sentence_text',sentence_text) if sentence_text != 'nan': result_text = model.evaluate_line( sess, input_from_line(sentence_text, char_to_id), id_to_tag) # for item in result_text['entities']: # text.append(item['word'].strip().strip('"').strip()) #标题+内容去重 print('result_title', result_title) print('result_text', result_text) entities_str = ';'.join(list(set(result_title + result_text))) print('entities_str', entities_str) entities.append(entities_str) output = pd.DataFrame(list(zip(ID, entities))) print(output) output.to_csv('Test_result.csv', encoding='utf8')
def evaluate_line(): config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, NERModel, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) # txt = input("请输入文件:") # with open(txt, encoding='u8') as test_file: # for line in test_file.readlines(): # line = line.split(',') # result = model.evaluate_line(sess, input_from_line(line[1], char_to_id), id_to_tag) # print(result) while True: line = input("请输入测试句子:") result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) print(result)
def evaluate_line(): config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) user_path = os.path.expanduser("~") with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) with open( user_path + '/share/deep_learning/data/knowledge_graph/entity_relation/corpus.yml' ) as f: corpus = yaml.load(f) for key in corpus: sentences = corpus[key] sentences = list(set(sentences)) for sen in sentences: sen_strip = sen.replace(' ', '') result = model.evaluate_line( sess, input_from_line(sen_strip, char_to_id), id_to_tag) extract_entity(result, key)
def evaluate_line(sents): global static_model config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory graph = tf.Graph() # tf_config = tf.ConfigProto() # tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) # with tf.Session(config=tf_config) as sess: sess = tf.InteractiveSession(graph=graph) print(" start create model") static_model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) print(" end create model") result = static_model.evaluate_line(sess, input_from_line(sents, char_to_id), id_to_tag) sess.close() return result
def evaluate_line(): config = utils.load_config(FLAGS.config_file) # 读取配置文件 log_path = os.path.join("evl_log", FLAGS.log_test) # ./log/train.log logger = utils.get_logger(log_path) # log文件名及路径 # limit GPU memory tf_config = tf.ConfigProto() # TensorFlow 会话的配置项 tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: # map_file 中存储着字与id,tag与id之间的对应关系 char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) # char_to_id 每个字对应的id, id_to_char 两者是相对应的 # print('char_to_id: ', char_to_id) # print('tag_to_id: ', tag_to_id) with tf.Session(config=tf_config) as sess: model = utils.create_model(sess, Model, FLAGS.ckpt_path, data_utils.load_word2vec, config, id_to_char, logger) while True: try: line = input("请输入测试句子:") if line == 'exit': break result = model.evaluate_line( sess, data_utils.input_from_line(line, char_to_id), id_to_tag) print(result) logger.debug(result) except Exception as e: logger.info(e)
def evaluate_line(): config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger, False) # while True: # # try: # # line = input("请输入测试句子:") # # result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) # # print(result) # # except Exception as e: # # logger.info(e) # # line = input("请输入测试句子:") # result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) # print(result) line = u"香港的房价已经到达历史巅峰,乌溪沙地铁站上盖由新鸿基地产公司开发的银湖天峰,现在的尺价已经超过一万五千港币。" line = u"这是测试语句,国务院加入测试" result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) print(result)
def evaluate_lines(user_query): config = load_config("config_file") logger = get_logger("train.log") # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open("maps.pkl", "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, "ckpt", load_word2vec, config, id_to_char, logger) try: results = [] lines = user_query.split('<SEP>') for item in lines: result = model.evaluate_line(sess, input_from_line(item, char_to_id), id_to_tag) results.append(result) print(result + '\n') except Exception as e: logger.info(e) return results
def evaluate_line(if_ask=False, another_input='', save=None): #save=FLAGS.ckpt_path config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, save, load_word2vec, config, id_to_char, logger) if if_ask == False: line = input("今天过的如何:\n") if if_ask == True: line = another_input result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) print(result) mytext, label_list = result ##三箱西瓜,苹果,身高180公分,体重90斤,酒一瓶不喝,心情很好 return mytext, label_list
def evaluate_line(): config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) while True: f1 = r'F:\PycharmProjects\dl2\deeplearning\data\nlpcc2016\nlpcc-iccpol-2016.kbqa.training.testing-data-all.txt' with open(f1 + '.crf.txt', 'w', encoding='utf-8') as o1: with codecs.open(f1, 'r', 'utf-8') as f1: for l in f1.readlines(): line = l.split('\t')[0] result = model.evaluate_line( sess, input_from_line(line, char_to_id), id_to_tag) print(result) o1.write(l.strip('\n').strip('\r') + '\t') if len(result['entities']) > 0: for k, v in dict(result['entities'][0]).items(): o1.write("%s\t%s\t" % (k, v)) o1.write('\n')
def predict_line(): config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) tf_config = tf.ConfigProto() # 读取词典 with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: # 根据保存的模型读取模型 model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) while True: # 反复输入句子进行预测 line = input("请输入测试句子:") inputs, tag = model.evaluate_line( sess, input_from_line(line, char_to_id), id_to_tag) if FLAGS.server: # 如果是命令行测试就打印json数据 result = result_to_json(inputs, tag) result = js.dumps(result, ensure_ascii=False, indent=4, separators=(',', ': ')) with open('./result/result.json', 'w', encoding='utf-8') as f: f.write(result) print("预测结果为:{}".format(result))
def evaluate_line(): config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) while True: # try: # line = input("请输入测试句子:") # result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) # print(result) # except Exception as e: # logger.info(e) line = unicode('在万达集团的老总王健林的著名采访之后,深圳出现了一家公司叫做赚它一个亿网络科技有限公司', 'utf-8') # line = input("请输入测试句子:") result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) print_result_p27(result) print(line) break
def evaluate_line(): config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) #while True: # try: # line = input("请输入测试句子:") # result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) # print(result) # except Exception as e: # logger.info(e) start = time.clock() fr=codecs.open("data/test_for_output1.txt","r","utf-8") fw = codecs.open("result/output1.txt", "w", "utf-8") result=[] for line in fr: result.append(model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag)) for res in result: #print(res) fw.write(' '.join(res)+'\n') end = time.clock() print('Running time: %s Seconds' % (end - start))
def get_text_input(): # http://127.0.0.1:5002/?inputStr="最开心" text = request.args.get('inputStr') # print(text) if len(text.strip()) > 0: aa = model.evaluate_line(sess, input_from_line(text, char_to_id), id_to_tag) return jsonify(aa)
def evaluate_line(): config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) f = codecs.open(os.path.join(FLAGS.test_filepath, "127_9.txt"), "r", "utf-8") s = f.read() line = [] sent = '' for i in range(len(s)): if s[i] != '。': sent += s[i] else: sent += s[i] line.append(sent) sent = '' line = input("请输入测试句子:") for info in line: print(info) result = model.evaluate_line(sess, input_from_line(info, char_to_id), id_to_tag) for info1 in result['entities']: print(info1)
def evaluate_predictsamples(): config = load_config(FLAGS.config_file) logger = get_logger('log/test.log') print_config(config, logger) # limit GPU memory logger.info("start predict") tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) index = 1 with open('data/predict.txt', 'w', encoding='utf-8') as f_write: read_file = 'data/test.txt' for line in open(read_file, 'r', encoding='utf-8-sig').readlines(): result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) print(result) content = str(index) + ',' if result is not None: entities = result['entities'] for entity in entities: if entity: print(entity['word'] + '\t' + entity['type']) # content += entity['word'] + '\t' + str(entity['start']) + '\t' + str(entity['end']) + '\t' + entity['type']+ ';' content += entity['word'] + '\t' + entity[ 'type'] + ';' f_write.write(content + '\n') index += 1
def predict_from_pb(self, document): row = {'content': document} df = pandas.DataFrame().append(row, ignore_index=True) filename = "data/{}.csv".format(time.time()) df.to_csv(filename, index=False, escapechar="\\", columns=['content']) with tf.Graph().as_default(): output_graph_def = tf.GraphDef() with open(os.path.join(args.model_folder, 'money_model.pb'), "rb") as f: output_graph_def.ParseFromString(f.read()) tf.import_graph_def(output_graph_def, name="") with tf.Session() as sess: # config=config sess.run(tf.global_variables_initializer()) # Get the input placeholders from the graph by name char_input = sess.graph.get_tensor_by_name('CharInputs:0') seg_input = sess.graph.get_tensor_by_name('SegInputs:0') drop_keep_prob = sess.graph.get_tensor_by_name('Dropout:0') # Tensors we want to evaluate,outputs lengths = sess.graph.get_tensor_by_name('lengths:0') logits = sess.graph.get_tensor_by_name("project/logits_outputs:0") # predictions = sess.graph.get_tensor_by_name("Accuracy/predictions:0") trans = sess.graph.get_tensor_by_name("crf_loss/transitions:0") # fo = open(test_data_path, "r", encoding='utf8') # all_data = fo.readlines() # fo.close() # for line in all_data: # 一行行遍历 lines = document.split(r"\n") lines = [line for line in lines if len(line)>0] list_original = [] list_amounts = [] for line in lines: input_batch = input_from_line(line, self.char_to_id) # 处理测试数据格式 feed_dict = create_feed_dict(input_batch, char_input, seg_input, drop_keep_prob) # 创建输入的feed_dict seq_len, scores = sess.run([lengths, logits], feed_dict) print('---') transition_matrix = trans.eval() batch_paths = decode(scores, seq_len, transition_matrix) tags = [self.id_to_tag[str(idx)] for idx in batch_paths[0]] #print(tags) result = result_to_json(input_batch[0][0], tags) original = str(result['string']) entities = result['entities'] if len(entities) != 0: list_original.extend([original] * len(entities)) for entity in entities: #是数字金额需要增强逻辑 if digit_regex.match(entity['word']) and len(entity['word']) >= 1: aug_word = augment(entity['word'], line) list_amounts.append(aug_word) else: list_amounts.append(entity['word']) #print(entity['word']) return {"answer":list_amounts,"line":list_original}
def address_segment(self, address): # print(address, type(address)) # <class 'str'> model, sess, char_to_id, id_to_tag = get_model() address_segment_result = model.evaluate_line( sess, input_from_line(address, char_to_id), id_to_tag) # print(address_segment_result, type(address_segment_result)) # <class 'dict'> return address_segment_result
def evaluate_file(config): config = load_config(config["config_file"]) logger = get_logger(config["log_file"]) with open(config["map_file"], "rb") as f: word_to_id, id_to_word, tag_to_id, id_to_tag = pickle.load(f) fout = open("data/result_entities.json", "w+") tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with tf.Session(config=tf_config) as sess: model = Model(config) logger.info("读取现有模型...") ckpt = tf.train.get_checkpoint_state(config["ckpt_path"]) # 从模型路径获取ckpt model.saver.restore(sess, ckpt.model_checkpoint_path) with open("data/result.json") as f: for index, line in enumerate(f): if index % 1000 == 0: print(index) data = json.loads(line.strip()) answer = data["answers"][0] if not answer: answer += " " result = model.evaluate_line(sess, input_from_line(answer, word_to_id), id_to_tag) entities = list(set([item["word"] for item in result["entities"]])) data["entity_answers"] = [entities] fout.write(json.dumps(data, ensure_ascii=False) + '\n') fout.close()
def evaluate_ht(): submit_path_ht = 'submit_sample/hetong.csv' submit_path_file = open(submit_path_ht, 'a+', encoding='gbk') submit_path_file.write('公告id,甲方,乙方,项目名称,合同名称,合同金额上限,合同金额下限,联合体成员\n') config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) rootdir = '/home/utopia/corpus/FDDC_part2_data/FDDC_announcements_round1_test_a_20180605/重大合同/html/' list = os.listdir(rootdir) # 列出文件夹下所有的目录与文件 for i in range(0, len(list)): htmlpath = os.path.join(rootdir, list[i]) if os.path.isfile(htmlpath): print(htmlpath) s_arr = levelText_withtable(htmlpath) candidates = [] for j in range(len(s_arr)): sen = s_arr[j] result = model.evaluate_line( sess, input_from_line(sen, char_to_id), id_to_tag) entities = result.get('entities') if len(entities) > 0: for en in entities: en['sid'] = j en['pid'] = list[i] candidates.append(en) org_ht(candidates, submit_path_file) print('-------------------------------------------------')
def evaluate_line(): # 写死 FLAGS.config_file = 'forum_config/config_file' FLAGS.log_file = 'forum_config/log/train.log' FLAGS.ckpt_path = 'forum_ckpt/' # FLAGS.ckpt_path = 'ckpt/' FLAGS.map_file = 'forum_config/maps.pkl' config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) while True: # try: # line = input("请输入测试句子:") # result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) # print(result) # except Exception as e: # logger.info(e) line = input("请输入测试句子:") result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) print(result)
def content_ner(): config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) for i in range(302): newslist = [] m = i + 1 f = open('newsLists/%d.txt' % m, 'r') for line in f.readlines(): newslist.append(int(line)) fout = codecs.open('content_ner/%d.json' % m, 'w', encoding='utf-8') for i in range(len(newslist)): day = newslist[i] f = codecs.open('D:/PycharmProjects/news_data/%d.json' % day, encoding='utf-8') f_d = json.load(f) content = f_d["content"] content = strip_tags(content) result = model.evaluate_line( sess, input_from_line(content, char_to_id), id_to_tag) dicObj = json.dumps(result) fout.write(dicObj) fout.write("\n") fout.close()
def evaluate_line(): config = load_config(FLAGS.config_file) #从文件config_file 中读取配置数据 #{'model_type': 'idcnn', 'num_chars': 3538, 'char_dim': 100, 'num_tags': 51, 'seg_dim': 20, 'lstm_dim': 100, 'batch_size': 20, 'emb_file': 'E:\\pythonWork3.6.2\\NERuselocal\\NERuselocal\\data\\vec.txt', 'clip': 5, 'dropout_keep': 0.5, 'optimizer': 'adam', 'lr': 0.001, 'tag_schema': 'iobes', 'pre_emb': True, 'zeros': True, 'lower': False} logger = get_logger(FLAGS.log_file) #写日志文件名字为 train.log # limit GPU memory tf_config = tf.ConfigProto( ) #实例化一个设置GPU的对象 函数用在创建session的时候,用来对session进行参数配置 tf_config.gpu_options.allow_growth = True #1动态申请显存需要多少使用多少 2限制GPU使用率 config.gpu_options.per_process_gpu_memory_fraction = 0.4 with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) while True: # try: # line = input("请输入测试句子:") # result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) # print(result) # except Exception as e: # logger.info(e) line = input("请输入测试句子:") result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) print(result)
def evaluate_line(): config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True # map_file文件需要改 with open(FLAGS.map_file, "rb+") as f: # pkllf=pickle.load(f) char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) while True: # try: # line = input("请输入测试句子:") # result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) # print(result) # except Exception as e: # logger.info(e) # 改成file in file out line = input("请输入测试句子:") result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) print(result)
def get_slot_dl(text): """ 获得一个句子的slot_table :param text: 用户输入的句子 # :param tf_sess: tensorflow 的sessiong :return: """ tf.reset_default_graph() # FIXME: 为了可以保证重复多次输入(不过会很慢)整合代码时,可以尝试将NLU和DM分离,使用统一的文件管理。 FLAGS.config_file = '/forum_config/config_file' FLAGS.log_file = '/forum_config/log/train.log' FLAGS.ckpt_path = '/forum_ckpt/' FLAGS.map_file = '/forum_config/maps.pkl' file_path = os.path.dirname(__file__) config = load_config(file_path+FLAGS.config_file) logger = get_logger(file_path+FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(file_path+FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) result = model.evaluate_line(sess, input_from_line(text, char_to_id), id_to_tag) return result
def get_text_input(): text = request.args.get('inputStr') if text: aa = model.evaluate_line(sess, input_from_line(text, char_to_id), id_to_tag) return jsonify(aa)
def nlp(): #print(request.headers) line = request.form.get('text') result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) #print(result) answer = '' for i in range(len(result['entities'])): answer =answer + result['entities'][i]['word'] + '/n' return answer
def get_text_input(): # logging.info("connect successfully") # tf_config = tf.ConfigProto() # tf_config.gpu_options.allow_growth = True # http://127.0.0.1:5002/?inputStr="..." text = request.args.get('inputStr') if text: aa = model.evaluate_line(sess, input_from_line(text, char_to_id), id_to_tag) return jsonify(aa)
def evaluate2(line_lis,sess): all_dict = [] print('buuuuuuuuuuuuuuuug1',line_lis) for i in line_lis: result = model.evaluate_line(sess, input_from_line(i, char_to_id), id_to_tag) mytext, label_list = result ##三箱西瓜,苹果,身高180公分,体重90斤,酒一瓶不喝,心情很好 dii = dictionary(label_list=label_list, mytext=mytext) all_dict.append(dii) print('debugggggggggggg1') return all_dict
def evaluate_line(): config = load_config(FLAGS.config_file) logger = get_logger(FLAGS.log_file) # limit GPU memory tf_config = tf.ConfigProto() tf_config.gpu_options.allow_growth = True with open(FLAGS.map_file, "rb") as f: char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f) with tf.Session(config=tf_config) as sess: model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger) while True: # try: # line = input("请输入测试句子:") # result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) # print(result) # except Exception as e: # logger.info(e) line = input("请输入测试句子:") result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag) print(result)