def run(self): time_all = [] bc = BertClient(port=PORT, show_server_config=False) for _ in range(self.num_repeat): start_t = time.perf_counter() bc.encode(self.batch) time_all.append(time.perf_counter() - start_t) print(time_all) self.avg_time = mean(time_all)
class BenchmarkClient(threading.Thread): def __init__(self, args): super().__init__() self.batch = [ ''.join( random.choices(string.ascii_uppercase + string.digits, k=args.client_seq_len)) for _ in args.client_batch_size ] self.bc = BertClient() self.num_repeat = args.num_repeat self.avg_time = 0 def run(self): time_all = [] for _ in range(self.num_repeat): start_t = time.clock() self.bc.encode(self.batch) time_all.append(time.clock() - start_t) self.avg_time = mean(time_all)
def test(): print("Loading test data...") start_time = time.time() x_test = process_file_nolabel(test_dir, word_to_id, config.seq_length) bc = BertClient() x_test = bc.encode(x_test) # (test 2364, 80) session = tf.Session() session.run(tf.global_variables_initializer()) saver = tf.train.Saver() saver.restore(sess=session, save_path=save_path) # 读取保存的模型 # print('Testing...') # loss_test, acc_test = evaluate(session, x_test, y_test) # msg = 'Test Loss: {0:>6.2}, Test Acc: {1:>7.2%}' # print(msg.format(loss_test, acc_test)) batch_size = 32 data_len = len(x_test) num_batch = int((data_len - 1) / batch_size) + 1 # y_test_cls = np.argmax(y_test, 1) y_pred_cls = np.zeros(shape=len(x_test), dtype=np.int32) # 保存预测结果 for i in range(num_batch): # 逐批次处理 start_id = i * batch_size end_id = min((i + 1) * batch_size, data_len) feed_dict = { model.input_x: x_test[start_id:end_id], model.keep_prob: 1.0 } y_pred_cls[start_id:end_id] = session.run(model.y_pred_cls, feed_dict=feed_dict) # 评估 print("Precision, Recall and F1-Score...") # print(metrics.classification_report(y_test_cls, y_pred_cls, target_names=categories)) # 混淆矩阵 print("Confusion Matrix...") # cm = metrics.confusion_matrix(y_test_cls, y_pred_cls) # print(cm) time_dif = get_time_dif(start_time) print("Time usage:", time_dif) return y_pred_cls
def main(): # 汽车观点提取,分过词的数据 # df_train = pd.read_csv('./data/ccf_car_train.csv') # df_test = pd.read_csv('./data/ccf_car_test.csv') # train = df_train['word_seg'] # test_x = df_test['content'] # target = df_train['class'] # train = np.array(train) # target = np.array(target) # list = [] # for i in target: # list.append(i) bc = BertClient() train = np.load('./data/train_x.npy') train = list(train) train = bc.encode(train) target = np.load('./data/train_y.npy') test_x = np.load('./data/test_x.npy') test_y = np.load('./data/test_y.npy') test_x = list(test_x) test_x = bc.encode(test_x) # 分为训练集和测试的时候加上 # train, test_x, target, test_y = train_test_split(train, target, test_size = 0.15, random_state = 0) # np.savetxt('./data/train_x', train_X, fmt='%s') # np.savetxt('./data/train_y', train_y, fmt='%s') # np.savetxt('./data/test_X', test_X, fmt='%s') # np.savetxt('./data/test_y', test_y, fmt='%s') # df_test = pd.read_csv(test_file) # df_train = df_train.drop(['article'], axis=1) # df_test = df_test.drop(['article'], axis=1) # ngram_range:tuple (min_n, max_n) 要提取的不同n-gram的n值范围的下边界和上边界。 将使用n的所有值,使得min_n <= n <= max_n。 # max_features: 构建一个词汇表,该词汇表仅考虑语料库中按术语频率排序的最高max_features # if feature == 'Count': # vectoriser = CountVectorizer(ngram_range=(1, 2), min_df = 3) # elif feature == 'Tfidf': # vectoriser = TfidfVectorizer(ngram_range=(1, 5), min_df = 3, max_df = 0.7) # # 构建特征,先训练 # vectoriser.fit(train) # 训练完进行归一化 总共有315503个词,过滤小于3的,剩下23546, max_df 貌似没用 # (7957, 2082), (1990, 2082) type:crs_matrix # train_X = vectoriser.transform(train) # test_X = vectoriser.transform(test_x) # y_train = df_train['class'] - 1 # train_X = np.array(train_X.data).reshape(train_X.shape) # 开始构建分类器 if classfier == 'LR': ''' c:正则化系数λ的倒数,float类型,默认为1.0。必须是正浮点型数。像SVM一样,越小的数值表示越强的正则化 ''' rg = LogisticRegression(C=1) rg.fit(train, target) y_pred = rg.predict(test_x) # elif classfier == 'NB': # # 使用默认的配置对分类器进行初始化。 # mnb_count = MultinomialNB(alpha=0.2) # # 使用朴素贝叶斯分类器,对CountVectorizer(不去除停用词)后的训练样本进行参数学习。 # mnb_count.fit(train_X, target) # y_pred = mnb_count.predict(test_X) # # elif classfier =='tree': # DT = tree.DecisionTreeClassifier() # DT.fit(train_X, target) # y_pred = DT.predict(test_X) ''' kernel='linear'时,为线性核,C越大分类效果越好,但有可能会过拟合(defaul C=1)。 kernel='rbf'时(default),为高斯核,gamma值越小,分类界面越连续;gamma值越大,分类界面越“散”,分类效果越好,但有可能会过拟合。 ''' # C=0.1 准确率高召回率低 C = 0.8 # elif classfier =='RT': # sv = RandomForestClassifier(n_estimators=400) # sv.fit(train_X, target) # # y_hat = sv.predict(train_X) # y_pred = sv.predict(test_X) # scores = cross_val_score(sv, train_X, target, cv=5) # print(scores) # 从sklearn.metrics 导入 classification_report。 # 输出更加详细的其他评价分类性能的指标。 print('classifier is : ' + classfier + '\tFeature is : ' + feature) print(classification_report(test_y, y_pred))
# using BertClient in multicast way import sys import threading from service.client import BertClient def client_clone(id, idx): bc = BertClient(port=int(sys.argv[1]), port_out=int(sys.argv[2]), identity=id) for j in bc.listen(): print('clone-client-%d: received %d x %d' % (idx, j.shape[0], j.shape[1])) if __name__ == '__main__': bc = BertClient(port=int(sys.argv[1]), port_out=int(sys.argv[2])) # start two cloned clients sharing the same identity as bc for j in range(2): t = threading.Thread(target=client_clone, args=(bc.identity, j)) t.start() with open('README.md') as fp: data = [v for v in fp if v.strip()] for _ in range(3): vec = bc.encode(data) print('bc received %d x %d' % (vec.shape[0], vec.shape[1]))
import sys import time from service.client import BertClient if __name__ == '__main__': bc = BertClient(ip='localhost', port=int(sys.argv[1])) # encode a list of strings with open('README.md', encoding='utf8') as fp: data = [v for v in fp if v.strip()] for j in range(1, 200, 10): start_t = time.time() tmp = data * j bc.encode(tmp) time_t = time.time() - start_t print('encoding %d strs in %.2fs, speed: %d/s' % (len(tmp), time_t, int(len(tmp) / time_t))) # bad example: encode a string # print(ec.encode('abc'))
from service.client import BertClient bc = BertClient('ip = localhost') vec = bc.encode(['hello world', 'good day'])
def RL_tuning_model(config): emd_file = config.nemd_dir, config.cemd_dir train_data, test_data, eval_data, vocab, embd = load_data(config.data_dir, emd_file, "BA") ##train_Id = [SOS] + clean_Id, ground_truth = clean_Id + [EOS], clean_len = get_length(ground_truth) train_noisy_Id, train_noisy_len, train_char_noisy_Id, train_char_noisy_len, train_nemd, train_target_Id, train_input_Id, train_clean_Id, train_clean_len, train_answer_Id, train_answer_len, max_char, max_word = train_data # noisy_Id, noisy_len, char_noisy_Id, char_noisy_len, ground_truth, train_Id, clean_Id, clean_len, answer_Id, answer_len, max_char, max_word = train_data test_noisy_Id, test_noisy_len, test_char_noisy_Id, test_char_noisy_len,test_nemd, test_ground_truth, test_train_Id, test_clean_Id, test_clean_len, test_answer_Id, test_answer_len = test_data # eval_noisy_Id, eval_noisy_len, eval_char_noisy_Id, eval_char_noisy_len,eval_target_Id, eval_input_Id, eval_clean_Id, eval_clean_len, eval_answer_Id, eval_answer_len =eval_data # config.max_target_length = max_word L = max_word embd = np.array(embd) batch_size =config.batch_size discount_factor = config.discount_factor sen_reward_rate = config.sen_reward_rate lam = config.lam gamma = config.gamma beam_width = config.beam_width update_N = config.update_N # config = tf.ConfigProto() # config.gpu_options.allow_growth = True model_name = "BS_" values = {} checkpoint_path = config.seq2seq_ckp_dir BS_ckp_dir = config.BS_ckp_dir reader = pywrap_tensorflow.NewCheckpointReader(checkpoint_path) var_to_shape_map = reader.get_variable_to_shape_map() for key in var_to_shape_map: if 'loss_fun' not in key: values[model_name + key + ':0'] = reader.get_tensor(key) vocab_size =len(vocab) G_BeamSearch = tf.Graph() with G_BeamSearch.as_default(): BeamSearch_seq2seq = Network.BeamSearch_Seq2seq(config, embd, vocab_size, model_name) seq2seq_len = L with tf.Session(graph=G_BeamSearch) as sess_beamsearch: sess_beamsearch.run(tf.global_variables_initializer()) for v in tf.trainable_variables(): if v.name in values.keys(): v.load(values[v.name], sess_beamsearch) for epo in range(config.epoch): print(" epoch: {}".format(epo)) idx = np.arange(0, len(train_noisy_Id)) idx = list(np.random.permutation(idx)) Bleu_score1 = [] Bleu_score2 = [] Bleu_score3 = [] Bleu_score4 = [] Rouge_score = [] Meteor_score = [] Cider_score = [] for batch in range(len(train_noisy_Id) / config.batch_size): source_shuffle, source_len, source_nemd, char_Id, char_len, train_shuffle, target_shuffle, target_len, answer_shuffle, answer_len = next_batch( train_noisy_Id, train_noisy_len, train_nemd, train_char_noisy_Id, train_char_noisy_len, train_input_Id, train_target_Id, train_clean_len, train_answer_Id, train_answer_len, batch_size, batch, idx) source_shuffle = tf.keras.preprocessing.sequence.pad_sequences(source_shuffle, maxlen=max_word, padding='post', truncating="post", value=EOS) source_emd = [] for n in source_nemd: source_emd.append(n[0:source_shuffle.shape[1]]) source_emd = np.asarray(source_emd) target_shuffle_in = tf.keras.preprocessing.sequence.pad_sequences(target_shuffle, maxlen=seq2seq_len, padding='post', truncating="post", value=EOS) # target_emd = [] # for c in train_cemd: # target_emd.append(c[0:train_shuffle.shape[1]]) # target_emd = np.asarray(target_emd) train_shuffle = tf.keras.preprocessing.sequence.pad_sequences(train_shuffle, maxlen=seq2seq_len, padding='post', truncating="post", value=EOS) whole_noisy_char_Id = [] for sen_char in char_Id: sen_len = len(sen_char) for i in range(len(source_shuffle[0]) - sen_len): sen_char.append([0] * max_char) ## fix the char with the length of max_word whole_noisy_char_Id.append(sen_char) whole_noisy_char_Id = np.asarray(whole_noisy_char_Id) whole_char_len = [] for char_List in char_len: whole_char_len.append(char_List[:len(source_shuffle[0])]) # initial_input_in = [target_shuffle_in[i][-1] for i in range(config.batch_size)] target_len = np.tile(seq2seq_len, config.batch_size) # print "the shape of source_shuffle ", np.array(source_shuffle).shape # print "the shape of source_len ", np.array(source_len).shape # print "the shape of source_emd ", np.array(source_emd).shape # print "the shape of whole_noisy_char_Id ", np.array(whole_noisy_char_Id).shape # print "the shape of whole_char_len ", np.array(whole_char_len).shape # print "the shape of target_len ", np.array(target_len).shape # print "the shape of train_shuffle ", np.array(train_shuffle).shape # print "the shape of target_shuffle_in ", np.array(target_shuffle_in).shape fd = {BeamSearch_seq2seq.encoder_inputs: source_shuffle, BeamSearch_seq2seq.encoder_inputs_length: source_len, BeamSearch_seq2seq.encoder_emb: source_emd, BeamSearch_seq2seq.encoder_char_ids: whole_noisy_char_Id, BeamSearch_seq2seq.encoder_char_len: whole_char_len, BeamSearch_seq2seq.decoder_length: target_len, BeamSearch_seq2seq.decoder_inputs: train_shuffle, # BeamSearch_seq2seq.target_emb: target_emd, BeamSearch_seq2seq.decoder_targets: target_shuffle_in, } if batch <= 100: ids, _, loss_seq2seq = sess_beamsearch.run( [BeamSearch_seq2seq.seq_ids, BeamSearch_seq2seq.train_op,BeamSearch_seq2seq.loss_seq2seq], fd) if batch % 100 == 0: print ('Batch {} Seq2seq_loss: '.format(batch),loss_seq2seq) # noisy_sen = [] # for pre_id in source_shuffle[0]: # if pre_id != EOS and pre_id != PAD and pre_id != SOS: # noisy_sen.append(vocab[pre_id]) # # clean_sen = [] # for clean_id in target_shuffle[0]: # if clean_id != EOS and clean_id != PAD and clean_id != SOS: # clean_sen.append(vocab[clean_id]) # # gene_sen = [] # for gen_id in ids[0]: # if gen_id != EOS and gen_id != PAD and gen_id != SOS: # gene_sen.append(vocab[gen_id]) # # print("question" + str(1) + "\n"); # print("noisy question: " + " ".join(noisy_sen) + "\n") # print("clean question: " + " ".join(clean_sen) + "\n") # print("generated question: " + " ".join(gene_sen) + "\n") elif batch > 100: RL_logits, policy, values = sess_beamsearch.run([BeamSearch_seq2seq.predicting_logits, BeamSearch_seq2seq.predicting_scores, BeamSearch_seq2seq.value], fd) # generated_que_input = np.insert(generated_que, 0, SOS, axis=1)[:, 0:-1] # target_shuffle = tf.keras.preprocessing.sequence.pad_sequences(target_shuffle, maxlen=None, # padding='post',truncating="post", value=EOS) generated_target_len = RL_logits.shape[1] bc = BertClient(port=5555, port_out=5556) generated_sen_beam = [] for beam in range(beam_width): for b in range(batch_size): sen = [] for l in range(RL_logits.shape[1]): sen.append(vocab[int(RL_logits[b][l][beam])]) generated_sen_beam.append(" ".join(sen)) bert_emd_beam = bc.encode(generated_sen_beam) # bert_emd = # bert_emd_beam.append(bert_emd[:,:generated_target_len,:]) bert_emd_beam = bert_emd_beam[:,1:generated_target_len+1,:] bert_emd_beam = bert_emd_beam.reshape(batch_size, beam_width, bert_emd_beam.shape[1], bert_emd_beam.shape[2]) sen_len_batch_beam = generated_target_len * np.ones((batch_size,beam_width)) # print "the shape of bert emd: ", bert_emd_beam.shape # # # source_shuffle_beam = np.tile(source_shuffle, (beam_width, 1)) # source_len_beam = np.tile(source_len, (beam_width)) # source_emd_beam = np.tile(source_emd, (beam_width,1,1)) # print "the shape of source_emd_beam", source_emd_beam.shape # whole_noisy_char_Id_beam = np.tile(whole_noisy_char_Id,(beam_width,1,1)) # whole_char_len_beam = np.tile(whole_char_len,(beam_width,1)) # print "the char_len: ", whole_char_len_beam.shape # RL_logits_beam = np.transpose(RL_logits,(0,2,1)).reshape(batch_size*beam_width, generated_target_len) # # answer_shuffle_beam = np.tile(answer_shuffle,(beam_width,1)) # answer_len_beam = np.tile(answer_len, (beam_width)) # # print "the shape of source_shuffle: ", source_shuffle_beam.shape # print "the input of target: ", RL_logits_beam.shape # # # for beam in range(beam_width): # # sen_len_batch_beam = [] # # for b in range(batch_size): # # sen_len_batch =[] # # l=0 # # while (l < generated_target_len and RL_logits[b][l][beam] != 1): # # l = l + 1 # # sen_len_batch.append(l) # # sen_len_batch_beam.append(sen_len_batch) # # Character_data= source_shuffle_beam,source_len_beam,source_emd_beam, whole_noisy_char_Id_beam, whole_char_len_beam, RL_logits_beam, sen_len_batch_beam, generated_target_len, bert_emd_beam # QA_data = answer_shuffle_beam, answer_len_beam, source_shuffle_beam, source_len_beam, source_emd_beam, RL_logits_beam, sen_len_batch_beam, bert_emd_beam, vocab Character_data = source_shuffle, source_len, source_emd, whole_noisy_char_Id, whole_char_len, RL_logits, sen_len_batch_beam, generated_target_len, bert_emd_beam QA_data = answer_shuffle, answer_len, source_shuffle, source_len, source_emd, RL_logits, sen_len_batch_beam, bert_emd_beam, vocab RL_rewards = Reward(config, Character_data, QA_data, vocab, embd) values_t = np.transpose(values,(2,0,1)) values_t1 = np.transpose(values[:,1:,:],(2,0,1)) values_t1 = np.insert(values_t1, values_t1.shape[2], values=np.zeros(batch_size), axis=2) Returns_beam =[] Deltas_beam = [] for beam in range(len(RL_rewards)): Returns_batch =[] Deltas_batch = [] for batch_len in range(len(RL_rewards[1])): returns, deltas = get_training_data(RL_rewards[beam][batch_len], values_t[beam][batch_len], values_t1[beam][batch_len], lam, gamma) Returns_batch.append(returns) Deltas_batch.append(deltas) Returns_beam.append(Returns_batch) Deltas_beam.append(Deltas_batch) Returns_beam = np.transpose(Returns_beam,(1,2,0)) Deltas_beam = np.transpose(Deltas_beam,(1,2,0)) old_policy = policy for N in range(update_N): fd = {BeamSearch_seq2seq.returns_ph: Returns_beam, BeamSearch_seq2seq.advantages_ph: Deltas_beam, BeamSearch_seq2seq.old_log_probs_ph: old_policy, BeamSearch_seq2seq.encoder_inputs: source_shuffle, BeamSearch_seq2seq.encoder_inputs_length: source_len, BeamSearch_seq2seq.encoder_emb: source_emd, BeamSearch_seq2seq.encoder_char_ids: whole_noisy_char_Id, BeamSearch_seq2seq.encoder_char_len: whole_char_len, BeamSearch_seq2seq.decoder_length: target_len, BeamSearch_seq2seq.decoder_inputs: train_shuffle, BeamSearch_seq2seq.decoder_targets: target_shuffle_in} policy_l, entr_l, loss_ppo, ratio, _, policy= sess_beamsearch.run( [BeamSearch_seq2seq.policy_loss, BeamSearch_seq2seq.entropy, BeamSearch_seq2seq.loss, BeamSearch_seq2seq.ratio, BeamSearch_seq2seq.optimize_expr, BeamSearch_seq2seq.predicting_scores], fd) print ('Batch {} PPO_loss: '.format(batch),loss_ppo) # print (" The ratio is :", ratio) # for t in range(batch_size): # ref = [] # hyp = [] # for tar_id in target_shuffle[t]: # if tar_id != EOS and tar_id != PAD: # ref.append(vocab[tar_id]) # # print vocab[tar_id] # for pre_id in RL_logits[t]: # if pre_id != EOS and pre_id != PAD: # hyp.append(vocab[pre_id]) # # hyp_sen = u" ".join(hyp).encode('utf-8') # ref_sen = u" ".join(ref).encode('utf-8') # dic_hyp = {} # dic_hyp[0] = [hyp_sen] # dic_ref = {} # dic_ref[0] = [ref_sen] # sen_bleu, _ = Bleu_obj.compute_score(dic_ref, dic_hyp) # sen_rouge = Rouge_obj.compute_score(dic_ref, dic_hyp) # sen_meteor, _ = Meteor_obj.compute_score(dic_ref, dic_hyp) # sen_cider, _ = cIDER_obj.compute_score(dic_ref, dic_hyp) # Bleu_score1.append(sen_bleu[0]) # Bleu_score2.append(sen_bleu[1]) # Bleu_score3.append(sen_bleu[2]) # Bleu_score4.append(sen_bleu[3]) # Rouge_score.append(sen_rouge[0]) # Meteor_score.append(sen_meteor) # Cider_score.append(sen_cider) # # if batch == 0 or batch % 100 == 0: # print(' batch {}'.format(batch)) # print(' minibatch reward of training: {}'.format(- loss_ppo)) # #print the result # for t in range(5): ## five sentences # print('Question {}'.format(t)) # print("noisy question:") # print(" ".join(map(lambda i: vocab[i], list(source_shuffle[t]))).strip().replace("<EOS>"," ")) # print("clean question:") # print(" ".join(map(lambda i: vocab[i], list(target_shuffle[t]))).strip().replace("<EOS>"," ")) # print("the generated first question:") # pre_sen = [] # for log_id in RL_logits[t, :, 0]: # if log_id != -1 and log_id!=2 and log_id!=0: # pre_sen.append(log_id) # print(" ".join(map(lambda i: vocab[i], pre_sen)).strip()) # print("\n") # print("\n Bleu_score1, Bleu_score2, Bleu_score3, Bleu_score4, Rouge, Cider\n") # print(" {:.4f}, {:.4f}, {:.4f}, {:.4f}, {:.4f}, {:.4f}, {:.4f}".format( # sum(Bleu_score1) / float(len(Bleu_score1)), # sum(Bleu_score2) / float(len(Bleu_score2)), # sum(Bleu_score3) / float(len(Bleu_score3)), # sum(Bleu_score4) / float(len(Bleu_score4)), # sum(Rouge_score) / float(len(Rouge_score)), # sum(Meteor_score) / float(len(Meteor_score)), # sum(Cider_score) / float(len(Cider_score)) # )) gc.collect() if (batch % 100 == 0 and batch < 100) or (batch % 20 == 0 and batch >100): generated_test_sen = [] test_answer_sen = [] test_noisy_sen = [] test_clean_sen = [] # saver.save(sess_beamsearch, BS_ckp_dir) ##testing set result: for batch_test in range(len(test_noisy_Id) / batch_size): idx_t = np.arange(0, len(test_noisy_Id)) idx_t = list(np.random.permutation(idx_t)) test_source_shuffle, test_source_len, test_source_nemd, test_char_Id, test_char_len_in, test_train_shuffle, test_target_shuffle, test_target_len, test_answer_shuffle, test_ans_len = next_batch( test_noisy_Id, test_noisy_len, test_nemd, test_char_noisy_Id, test_char_noisy_len, test_train_Id, test_ground_truth, test_clean_len, test_answer_Id, test_answer_len, batch_size, batch_test, idx_t) for an in test_answer_shuffle: test_answer_sen.append(vocab[anum] for anum in an) for no in test_source_shuffle: test_noisy_sen.append(vocab[si] for si in no) for cl in test_target_shuffle: test_clean_sen.append(vocab[ci] for ci in cl) test_source_shuffle_in = tf.keras.preprocessing.sequence.pad_sequences(test_source_shuffle, maxlen = max_word, padding='post', truncating="post",value=EOS) test_source_emd = [] for n in test_source_nemd: test_source_emd.append(n[0 : test_source_shuffle_in.shape[1]]) test_source_emd = np.asarray(test_source_emd) val_target_shuffle_in=[] for val in test_target_shuffle: val_target_shuffle_in.append([val[0]]) val_train_shuffle_in = [] for tra in test_train_shuffle: val_train_shuffle_in.append([tra[0]]) initial_input_in = [SOS for i in range(batch_size)] # [SOS for i in range(batch_size)] val_target_len_in = np.tile(1, batch_size) test_batch_noisy_char_Id = [] for sen_char in test_char_Id: sen_len = len(sen_char) for i in range(len(test_source_shuffle_in[0]) - sen_len): sen_char.append([0] * max_char) ## fix the char with the length of max_word test_batch_noisy_char_Id.append(sen_char) test_batch_noisy_char_Id = np.asarray(test_batch_noisy_char_Id) test_char_len = [] for char_List in test_char_len_in: test_char_len.append(char_List[:len(test_source_shuffle_in[0])]) # print "the shape of test_noisy_char_Id ", np.array(test_batch_noisy_char_Id).shape # if np.array(test_batch_noisy_char_Id).shape == (batch_size,): # continue # print "the shape of test_char_len ", np.array(test_char_len).shape # # print "the shape of source_shuffle ", np.array(test_source_shuffle_in).shape # # print "the shape of source_len ", np.array(test_source_len).shape # # print "the shape of source_emd ", np.array(test_source_emd).shape # # print "the shape of whole_noisy_char_Id ", np.array(test_batch_noisy_char_Id).shape # # print "the shape of whole_char_len ", np.array(test_char_len).shape fd = {BeamSearch_seq2seq.encoder_inputs: test_source_shuffle_in, BeamSearch_seq2seq.encoder_inputs_length: test_source_len, BeamSearch_seq2seq.encoder_emb: test_source_emd, BeamSearch_seq2seq.encoder_char_ids: test_batch_noisy_char_Id, BeamSearch_seq2seq.encoder_char_len: test_char_len} val_id = sess_beamsearch.run([BeamSearch_seq2seq.ids], fd) for t in range(batch_size): ref = [] hyp = [] for tar_id in test_target_shuffle[t]: if tar_id != EOS and tar_id != PAD and tar_id != SOS: ref.append(vocab[tar_id]) # print vocab[tar_id] for pre_id in val_id[0][t]: if pre_id != EOS and pre_id != PAD and pre_id != SOS: hyp.append(vocab[pre_id]) generated_test_sen.append(hyp) hyp_sen = u" ".join(hyp).encode('utf-8') ref_sen = u" ".join(ref).encode('utf-8') dic_hyp = {} dic_hyp[0] = [hyp_sen] dic_ref = {} dic_ref[0] = [ref_sen] sen_bleu, _ = Bleu_obj.compute_score(dic_ref, dic_hyp) sen_rouge = Rouge_obj.compute_score(dic_ref, dic_hyp) sen_meteor, _ = Meteor_obj.compute_score(dic_ref, dic_hyp) sen_cider, _ = cIDER_obj.compute_score(dic_ref, dic_hyp) Bleu_score1.append(sen_bleu[0]) Bleu_score2.append(sen_bleu[1]) Bleu_score3.append(sen_bleu[2]) Bleu_score4.append(sen_bleu[3]) Rouge_score.append(sen_rouge[0]) Meteor_score.append(sen_meteor) Cider_score.append(sen_cider) print("\n Bleu_score1, Bleu_score2, Bleu_score3, Bleu_score4, Rouge, Cider\n") print(" {:.4f}, {:.4f}, {:.4f}, {:.4f}, {:.4f}, {:.4f}, {:.4f}".format( sum(Bleu_score1) / float(len(Bleu_score1)), sum(Bleu_score2) / float(len(Bleu_score2)), sum(Bleu_score3) / float(len(Bleu_score3)), sum(Bleu_score4) / float(len(Bleu_score4)), sum(Rouge_score) / float(len(Rouge_score)), sum(Meteor_score) / float(len(Meteor_score)), sum(Cider_score) / float(len(Cider_score)) )) bleu_score = sum(Bleu_score1) / float(len(Bleu_score1)) for i in range(len(generated_test_sen[0:2])): print("question"+str(i) + "\n") print("answer: "+" ".join(test_answer_sen[i]) + "\n") print("noisy question: " + " ".join(test_noisy_sen[i]) + "\n") print("clean question: " + " ".join(test_clean_sen[i]) + "\n") print("generated question: " + " ".join(generated_test_sen[i]) + "\n") fname = BS_ckp_dir +str(bleu_score) f=open(fname, "wb") f.write("\n Bleu_score1, Bleu_score2, Bleu_score3, Bleu_score4, Rouge, Meteor Cider\n") f.write(" {:.4f}, {:.4f}, {:.4f}, {:.4f}, {:.4f}, {:.4f}, {:.4f} \n".format( sum(Bleu_score1) / float(len(Bleu_score1)), sum(Bleu_score2) / float(len(Bleu_score2)), sum(Bleu_score3) / float(len(Bleu_score3)), sum(Bleu_score4) / float(len(Bleu_score4)), sum(Rouge_score) / float(len(Rouge_score)), sum(Meteor_score) / float(len(Meteor_score)), sum(Cider_score) / float(len(Cider_score)) )) for i in range(len(generated_test_sen)): f.write("question"+str(i) + "\n") f.write("answer: "+" ".join(test_answer_sen[i]) + "\n") f.write("noisy question: " + " ".join(test_noisy_sen[i]) + "\n") f.write("clean question: " + " ".join(test_clean_sen[i]) + "\n") f.write("generated question: " + " ".join(generated_test_sen[i]) + "\n") network_vars = tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, config.PPoscope) saver = tf.train.Saver(sharded=False) saver.save(sess_beamsearch, config.BS_ckp_dir)
b = nn.Linear(76, 76) w = list(b.parameters()) print(b(a) == a @ w[0].transpose(-2, -1) + w[1]) # %% Cosine distance for Various Vectors. from scipy import spatial from service.client import BertClient bc = BertClient(ip='xxx', port=3004, port_out=3005) sentences = [ '나 는 너 를 사랑 하다 여', '나 는 너 를 사랑 하다 였 다', '사랑 누 가 말하다 였 나', ] (q_length, q_tokens, q_embedding, q_ids) = bc.encode(sentences) love_1 = q_embedding[0][5] love_2 = q_embedding[1][5] love_3 = q_embedding[2][1] print(spatial.distance.cdist([love_2, love_3], [love_1], metric='cosine')) print(spatial.distance.cdist([love_2 + love_3], [love_1], metric='cosine')) print(spatial.distance.cdist([love_2 * 10 + love_3], [love_1], metric='cosine')) print(spatial.distance.cdist([love_2 + love_3 * 10], [love_1], metric='cosine')) print( spatial.distance.cdist([(love_2 + love_3) * 10], [love_1], metric='cosine'))
import os import GPUtil import tensorflow as tf from service.client import BertClient os.environ['CUDA_VISIBLE_DEVICES'] = str(GPUtil.getFirstAvailable()[0]) tf.logging.set_verbosity(tf.logging.INFO) with open('README.md') as fp: data = [v for v in fp if v.strip()] bc = BertClient() list_vec = bc.encode(data) list_label = [0 for _ in data] # a dummy list of all-zero labels # write tfrecords with tf.python_io.TFRecordWriter('tmp.tfrecord') as writer: def create_float_feature(values): return tf.train.Feature(float_list=tf.train.FloatList(value=values)) def create_int_feature(values): return tf.train.Feature(int64_list=tf.train.Int64List( value=list(values))) for (vec, label) in zip(list_vec, list_label): features = { 'features': create_float_feature(vec), 'labels': create_int_feature([label])
def Reward(config, Character_data, QA_data, vocab, embd): batch_size = config.batch_size vocab_size = len(vocab) num_units = config.num_units beam_width = config.beam_width Bidirection = config.Bidirection Embd_train = config.Embd_train Attention = config.Attention discount_factor = config.discount_factor dropout = config.dropout char_num = config.char_num char_dim = config.char_dim char_hidden_units = config.char_hidden_units Seq2Seq_pretain_ckp = config.seq2seq_ckp_dir QA_pretain_ckp = config.QA_ckp_dir model_cond = "training" G_Seq2Seq = tf.Graph() sess_word_rw = tf.Session(graph=G_Seq2Seq) with G_Seq2Seq.as_default(): Seq2Seq_model = CharacterBA.Char_Seq2Seq(batch_size, vocab_size, num_units, embd, model_cond, Bidirection, Embd_train, char_hidden_units, Attention, char_num, char_dim) saver_word_rw = tf.train.Saver() saver_word_rw.restore(sess_word_rw, Seq2Seq_pretain_ckp) model_type = "testing" G_QA_similiarity = tf.Graph() sess_QA_rw = tf.Session(graph=G_QA_similiarity) with G_QA_similiarity.as_default(): QA_simi_model = QA_similiarity.QA_similiarity(batch_size, num_units, embd, model_type) saver_sen_rw = tf.train.Saver() saver_sen_rw.restore(sess_QA_rw, QA_pretain_ckp) source_shuffle, source_len, source_emd, whole_noisy_char_Id, char_len, generated_sen, sen_len_batch_beam, generated_target_len, bert_emd = Character_data logits_reward_beam = [] # print "the generated_sen shape: ", generated_sen.shape # print "sen_len_batch_beam: ", sen_len_batch_beam.shape # print "bert_emd: ", bert_emd.shape for i in range(beam_width): fd_seq = { Seq2Seq_model.encoder_inputs: source_shuffle, Seq2Seq_model.encoder_inputs_length: source_len, Seq2Seq_model.encoder_emb: source_emd, Seq2Seq_model.encoder_char_ids: whole_noisy_char_Id, Seq2Seq_model.encoder_char_len: char_len, Seq2Seq_model.decoder_inputs: generated_sen[:, :, i], Seq2Seq_model.decoder_length: sen_len_batch_beam[:, i], # Seq2Seq_model.target_emb: bert_emd, Seq2Seq_model.dropout_rate: dropout } logits_reward_beam.append( sess_word_rw.run(Seq2Seq_model.softmax_logits, fd_seq)) answer_shuffle, answer_len, source_shuffle, source_len, source_emd, generated_sen, sen_len_batch_beam, bert_emd, vocab = QA_data EOS = 1 answer_shuffle = tf.keras.preprocessing.sequence.pad_sequences( answer_shuffle, maxlen=None, padding='post', value=EOS) QA_reward_beam = [] for i in range(beam_width): fd_qa = { QA_simi_model.answer_inputs: answer_shuffle, QA_simi_model.answer_inputs_length: answer_len, QA_simi_model.question1_inputs: source_shuffle, QA_simi_model.question1_inputs_length: source_len, QA_simi_model.question2_inputs: generated_sen[:, :, i], QA_simi_model.question2_inputs_length: sen_len_batch_beam[:, i], QA_simi_model.question1_emd: source_emd, QA_simi_model.question2_emd: bert_emd[:, i, :, :] } #[] QA_reward_beam.append( sess_QA_rw.run([QA_simi_model.two_distance], fd_qa)) # print "the shape logits_reward_beam", np.array(logits_reward_beam).shape logits_reward_beam = np.array(logits_reward_beam).reshape( batch_size * beam_width, generated_target_len, vocab_size) QA_reward_beam = np.array(QA_reward_beam).reshape(batch_size * beam_width) generated_sen = np.array(generated_sen).reshape(batch_size * beam_width, generated_target_len) # print "the shape of generated_sen ", generated_sen.shape logits_batch_beam = [] for b in range(batch_size * beam_width): logits_batch = [] for l in range(generated_target_len): id = generated_sen[b][l] logits_v = logits_reward_beam[b][l][id] logits_batch.append(logits_v) logits_batch_beam.append(logits_batch) bc = BertClient(port=4445, port_out=4446) gen_sen = [] for idx in range(batch_size * beam_width): sen = [] for l in range(generated_target_len): sen.append(vocab[int(generated_sen[idx][l])]) gen_sen.append(" ".join(sen)) bert_pro_beam = bc.encode(gen_sen) bert_pro_beam = bert_pro_beam[:, 1:generated_target_len + 1, :] bert_weight_path = config.bert_weight_path if os.path.isfile(bert_weight_path): f = open(bert_weight_path, "rb") data = pickle.load(f) output_weights = data["bert_W"] output_bias = data["bert_b"] output_weights = np.transpose(output_weights) logits = np.matmul(bert_pro_beam, output_weights) logits = np.add(logits, output_bias) log_probs = np.divide( np.exp(logits), np.tile(np.sum(np.exp(logits), axis=2), output_bias.shape[0]).reshape(logits.shape[0], logits.shape[1], logits.shape[2])) # print "the size of log_probs", log_probs.shape # tf.nn.log_softmax(logits, axis=-1) bert_Id, bert_vocab_size = bert_vocab_map_Id(generated_sen, config, vocab) bert_reward_beam = [] for i in range(batch_size * beam_width): bert_reward = [] for j in range(generated_target_len): bert_reward.append(log_probs[i][j][bert_Id[i][j]]) bert_reward_beam.append(bert_reward) # bert_reward_beam =[] # for i in range(beam_width): # gen_sen =[] # for b in range(batch_size): # sen = [] # for l in range(generated_sen.shape[1]): # sen.append(vocab[int(generated_sen[b][l][i])]) # gen_sen.append(" ".join(sen)) # bert_pro = bc.encode(gen_sen) # bert_pro = bert_pro[:,:generated_target_len,:] # # bert_weight_path = config.bert_weight_path # if os.path.isfile(bert_weight_path): # f=open(bert_weight_path,"rb") # data = pickle.load(f) # output_weights = data["bert_W"] # output_bias = data["bert_b"] # output_weights = np.transpose(output_weights) # logits = np.matmul(bert_pro, output_weights) # logits = np.add(logits, output_bias) # log_probs = np.divide(np.exp(logits),np.tile(np.sum(np.exp(logits), axis=2),output_bias.shape[0]).reshape(logits.shape[0],logits.shape[1],logits.shape[2])) # # tf.nn.log_softmax(logits, axis=-1) # # bert_Id = bert_vocab_map_Id(generated_sen, config, vocab) # # bert_reward_beam = [] # for i in range(batch_size * beam_width): # bert_reward = [] # for j in range(generated_target_len): # bert_reward.append(log_probs[i][j][bert_Id[i][j]]) # bert_reward_beam.append(bert_reward) RL_reward_beam_dis = [] for beam in range(beam_width * batch_size): RL_reward = [] for idx in range(generated_target_len): if idx != len(logits_batch_beam[beam]) - 1: RL_reward.append(vocab_size * logits_batch_beam[beam][idx] + bert_vocab_size * bert_reward_beam[beam][idx]) elif idx == len(logits_batch_beam[beam]) - 1: RL_reward.append(vocab_size * logits_batch_beam[beam][idx] + bert_vocab_size * bert_reward_beam[beam][idx] + 10 * QA_reward_beam[beam]) RL_reward_batch_dis = discounted_rewards_cal(RL_reward, discount_factor) RL_reward_beam_dis.append(RL_reward_batch_dis) RL_reward_beam_dis = np.array(RL_reward_beam_dis).reshape( batch_size, beam_width, generated_target_len) RL_reward_beam_dis_mm = RL_reward_beam_dis - np.tile( np.mean(RL_reward_beam_dis, axis=1), (beam_width, 1)).reshape(batch_size, beam_width, generated_target_len) # print "the RL_reward_dis: ", np.mean(RL_reward_beam_dis_mm) RL_reward_beam_dis_mm = np.transpose(RL_reward_beam_dis_mm, (1, 0, 2)) return RL_reward_beam_dis_mm
from service.client import BertClient bc = BertClient() s = ['你好。', '嗨!', '今日我本来打算去公园。'] a = bc.encode(s) # a = bc.encode(['Today is good.']) print(a.shape) # print(s.type) print(a[0][1].shape) print(a[0][1]) print(a[0][4]) print(a[1][3]) print(a[0][5])