def evaluate(): # Load model weight_path = 'model/09031344_epoch_4_train_loss_3.7933.h5' # Load data X, Sources, Targets = load_test_data() de2idx, idx2de = load_de_vocab() en2idx, idx2en = load_en_vocab() model = TransformerModel(in_vocab_len=len(idx2de), out_vocab_len=len(idx2en), max_len=hp.maxlen) model.load_model(weight_path) for i in range(len(X) // hp.batch_size): x = X[i * hp.batch_size:(i + 1) * hp.batch_size] sources = Sources[i * hp.batch_size:(i + 1) * hp.batch_size] targets = Targets[i * hp.batch_size:(i + 1) * hp.batch_size] preds = model.translate(x, idx2en) for source, target, pred in zip(sources, targets, preds): print('source:', source) print('expected:', target) print('pred:', pred) print()
def eval(): # Load graph g = Graph(is_training=False) print("Graph loaded") # Load data X, Sources, Targets = load_test_data() de2idx, idx2de = load_de_vocab() en2idx, idx2en = load_en_vocab() # X, Sources, Targets = X[:33], Sources[:33], Targets[:33] # Start session with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: ## Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") ## Get model name mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # model name ## Inference if not os.path.exists('results'): os.mkdir('results') with codecs.open("results/" + mname, "w", "utf-8") as fout: list_of_refs, hypotheses = [], [] for i in range(len(X) // hp.batch_size): ### Get mini-batches x = X[i*hp.batch_size: (i+1)*hp.batch_size] sources = Sources[i*hp.batch_size: (i+1)*hp.batch_size] targets = Targets[i*hp.batch_size: (i+1)*hp.batch_size] ### Autoregressive inference preds = np.zeros((hp.batch_size, hp.maxlen), np.int32) # (32, 10) for j in range(hp.maxlen): _preds = sess.run(g.preds, {g.x: x, g.y: preds}) preds[:, j] = _preds[:, j] ### Write to file for source, target, pred in zip(sources, targets, preds): # sentence-wise #print(got) got = " ".join(idx2en[idx] for idx in pred).split("</S>")[0].strip() fout.write("- source: " + source +"\n") fout.write("- expected: " + target + "\n") fout.write("- got: " + got + "\n\n") fout.flush() # bleu score ref = target.split() hypothesis = got.split() if len(ref) > 3 and len(hypothesis) > 3: list_of_refs.append([ref]) hypotheses.append(hypothesis) ## Calculate bleu score score = corpus_bleu(list_of_refs, hypotheses) fout.write("Bleu Score = " + str(100*score))
def synthesize(): # Load data X = load_test_data() # Load graph g = Graph(training=False) print("Graph loaded") # Inference with g.graph.as_default(): with tf.Session() as sess: saver = tf.train.Saver() # Restore parameters saver.restore(sess, tf.train.latest_checkpoint(hp.puresynth)) print("Restored!") # Get model name mname = open(hp.puresynth + '/checkpoint', 'r').read().split('"')[1] # Synthesize file_id = 1 for i in range(0, len(X), hp.batch_size): x = X[i:i + hp.batch_size] # Get melspectrogram mel_output = np.zeros( (hp.batch_size, hp.T_y // hp.r, hp.n_mels * hp.r), np.float32) decoder_output = np.zeros( (hp.batch_size, hp.T_y // hp.r, hp.embed_size), np.float32) alignments_li = np.zeros( (hp.dec_layers, hp.T_x, hp.T_y // hp.r), np.float32) prev_max_attentions_li = np.zeros( (hp.dec_layers, hp.batch_size), np.int32) for j in range(hp.T_y // hp.r): _gs, _mel_output, _decoder_output, _max_attentions_li, _alignments_li = \ sess.run([g.global_step, g.mel_output, g.decoder_output, g.max_attentions_li, g.alignments_li], {g.x: x, g.y1: mel_output, g.prev_max_attentions_li:prev_max_attentions_li}) mel_output[:, j, :] = _mel_output[:, j, :] decoder_output[:, j, :] = _decoder_output[:, j, :] alignments_li[:, :, j] = np.array(_alignments_li)[:, :, j] prev_max_attentions_li = np.array(_max_attentions_li)[:, :, j] # Get magnitude mag_output = sess.run(g.mag_output, {g.decoder_output: decoder_output}) # Generate wav files if not os.path.exists(hp.sampledir): os.makedirs(hp.sampledir) for mag in mag_output: print("Working on file num ", file_id) wav = spectrogram2wav(mag) write(hp.sampledir + "/{}_{}.wav".format(mname, file_id), hp.sr, wav) file_id += 1
def eval(): # Load data X, Sources, Targets = load_test_data() de2idx, idx2de = load_de_vocab() en2idx, idx2en = load_en_vocab() enc_voc = len(de2idx) dec_voc = len(en2idx) # load model model = AttModel(hp, enc_voc, dec_voc) model.load_state_dict( torch.load(hp.model_dir + '/model_epoch_%02d' % hp.eval_epoch + '.pth')) print('Model Loaded.') model.eval() model.cuda() # Inference if not os.path.exists('results'): os.mkdir('results') with codecs.open('results/model%d.txt' % hp.eval_epoch, 'w', 'utf-8') as fout: list_of_refs, hypotheses = [], [] for i in range(len(X) // hp.batch_size): # Get mini-batches x = X[i * hp.batch_size:(i + 1) * hp.batch_size] sources = Sources[i * hp.batch_size:(i + 1) * hp.batch_size] targets = Targets[i * hp.batch_size:(i + 1) * hp.batch_size] # Autoregressive inference x_ = torch.LongTensor(x).cuda() preds_t = torch.LongTensor( np.zeros((hp.batch_size, hp.maxlen), np.int32)).cuda() preds = preds_t for j in range(hp.maxlen): _, _preds, _ = model(x_, preds) preds_t[:, j] = _preds.data[:, j] preds = preds_t.long() preds = preds.data.cpu().numpy() # Write to file for source, target, pred in zip(sources, targets, preds): # sentence-wise got = " ".join(idx2en[idx] for idx in pred).split("</S>")[0].strip() fout.write("- source: " + source + "\n") fout.write("- expected: " + target + "\n") fout.write("- got: " + got + "\n\n") fout.flush() # bleu score ref = target.split() hypothesis = got.split() if len(ref) > 3 and len(hypothesis) > 3: list_of_refs.append([ref]) hypotheses.append(hypothesis) # Calculate bleu score score = corpus_bleu(list_of_refs, hypotheses) fout.write("Bleu Score = " + str(100 * score))
def eval(): # Load graph g = Graph(is_training=False) print("Graph loaded") # Load data X, Sources, Targets = load_test_data() de2idx, idx2de = load_de_vocab() en2idx, idx2en = load_en_vocab() # X, Sources, Targets = X[:33], Sources[:33], Targets[:33] # Start session with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: ## Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") ## Get model name mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # model name ## Inference if not os.path.exists('results'): os.mkdir('results') with codecs.open("results/" + mname, "w", "utf-8") as fout: list_of_refs, hypotheses = [], [] for i in range(len(X) // hp.batch_size): ### Get mini-batches x = X[i*hp.batch_size: (i+1)*hp.batch_size] sources = Sources[i*hp.batch_size: (i+1)*hp.batch_size] targets = Targets[i*hp.batch_size: (i+1)*hp.batch_size] ### Autoregressive inference preds = np.zeros((hp.batch_size, hp.maxlen), np.int32) for j in range(hp.maxlen): _preds = sess.run(g.preds, {g.x: x, g.y: preds}) preds[:, j] = _preds[:, j] ### Write to file for source, target, pred in zip(sources, targets, preds): # sentence-wise got = " ".join(idx2en[idx] for idx in pred).split("</S>")[0].strip() fout.write("- source: " + source +"\n") fout.write("- expected: " + target + "\n") fout.write("- got: " + got + "\n\n") fout.flush() # bleu score ref = target.split() hypothesis = got.split() if len(ref) > 3 and len(hypothesis) > 3: list_of_refs.append([ref]) hypotheses.append(hypothesis) ## Calculate bleu score score = corpus_bleu(list_of_refs, hypotheses) fout.write("Bleu Score = " + str(100*score))
def eval(): # Load graph g = Graph(is_training=False) print("Graph loaded") # Load data X, Sources = load_test_data() ch2idx, idx2ch = load_ch_vocab() # X, Sources, Targets = X[:33], Sources[:33], Targets[:33] # Start session with g.graph.as_default(): sv = tf.train.Supervisor() gpu_options = tf.GPUOptions(allow_growth=True) with sv.managed_session(config=tf.ConfigProto( gpu_options=gpu_options)) as sess: ## Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") ## Get model name mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # model name ## Inference if not os.path.exists('results'): os.mkdir('results') with codecs.open("results/" + mname, "w", "utf-8") as fout: list_of_refs, hypotheses = [], [] for i in range(len(X) // hp.batch_size): ### Get mini-batches 切片得到batch x = X[i * hp.batch_size:(i + 1) * hp.batch_size] sources = Sources[i * hp.batch_size:(i + 1) * hp.batch_size] ### Autoregressive inference preds = np.zeros((hp.batch_size, hp.maxlen), np.int32) for j in range(hp.maxlen): # 通过网络预测g.preds,feed_dict的g.y,是之前定义的全为0的preds # 每次预测batch中所有句子的一个单词 # 因为multi-attention有各种mask存在,所以当预测y的第i个单词时,self-attentuon不会受后面单词的影响(seq-mask) # 同时decoder-encoder-attention不会受0 <PAD>标记影响(query-mask) # 所以可以一个一个单词训练。 _preds = sess.run(g.preds, {g.x: x, g.y: preds}) preds[:, j] = _preds[:, j] ### Write to file # 通过zip把batch中的一个句子的source, target, pred取出来 for source, pred in zip(sources, preds): # sentence-wise # " ".join获得整个句子,在</S>前的留下 got = "".join( idx2ch[idx] for idx in pred).split("</S>")[0].strip() # fout.write("- source: " + source + "\n") fout.write(got + "\n\n") fout.flush()
def synthesize(): # Load data X = load_test_data() # Load graph g = Graph(training=False); print("Graph loaded") # Inference with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: # Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)); print("Restored!") # Get model name mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # Synthesize file_id = 1 for i in range(0, len(X), hp.batch_size): x = X[i:i + hp.batch_size] # Get melspectrogram mel_output = np.zeros((hp.batch_size, hp.T_y // hp.r, hp.n_mels * hp.r), np.float32) decoder_output = np.zeros((hp.batch_size, hp.T_y // hp.r, hp.embed_size), np.float32) prev_max_attentions = np.zeros((hp.batch_size,), np.int32) max_attentions = np.zeros((hp.batch_size, hp.T_y // hp.r)) alignments = np.zeros((hp.T_x, hp.T_y // hp.r), np.float32) for j in range(hp.T_y // hp.r): _mel_output, _decoder_output, _max_attentions, _alignments = \ sess.run([g.mel_output, g.decoder_output, g.max_attentions, g.alignments], {g.x: x, g.y1: mel_output, g.prev_max_attentions: prev_max_attentions}) mel_output[:, j, :] = _mel_output[:, j, :] decoder_output[:, j, :] = _decoder_output[:, j, :] alignments[:, j] = _alignments[0].T[:, j] prev_max_attentions = _max_attentions[:, j] max_attentions[:, j] = _max_attentions[:, j] plot_alignment(alignments[::-1, :], "sanity-check", 0) # Get magnitude mags = sess.run(g.mag_output, {g.decoder_output: decoder_output}) # Generate wav files if not os.path.exists(hp.sampledir): os.makedirs(hp.sampledir) for mag in mags: print("file id=", file_id) # generate wav files mag = mag * hp.mag_std + hp.mag_mean # denormalize audio = spectrogram2wav(np.power(10, mag) ** hp.sharpening_factor) audio = signal.lfilter([1], [1, -hp.preemphasis], audio) write(hp.sampledir + "/{}_{}.wav".format(mname, file_id), hp.sr, audio) file_id += 1
def evaluate(g, maxstep=0): # Load data X, Sources, Targets = load_test_data() _en2idx, idx2en = load_en_vocab() # Start session with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto( allow_soft_placement=True)) as sess: ## Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") ## Inference with codecs.open('result.txt', "w", "utf-8") as fout: list_of_refs, hypotheses = [], [] for i in range(len(X) // hp.batch_size): if maxstep > 0 and i >= maxstep: break ### Get mini-batches x = X[i * hp.batch_size:(i + 1) * hp.batch_size] sources = Sources[i * hp.batch_size:(i + 1) * hp.batch_size] targets = Targets[i * hp.batch_size:(i + 1) * hp.batch_size] ### Autoregressive inference preds = g.get_pred(sess, g, x) ### Write to file for source, target, pred in zip(sources, targets, preds): # sentence-wise got = " ".join( idx2en[idx] for idx in pred).split("</S>")[0].strip() fout.write("- source: " + source + "\n") fout.write("- expected: " + target + "\n") fout.write("- got: " + got + "\n\n") fout.flush() # bleu score ref = target.split() hypothesis = got.split() if len(ref) > 3 and len(hypothesis) > 3: list_of_refs.append([ref]) hypotheses.append(hypothesis) ## Calculate bleu score if len(list_of_refs) > 100: score = corpus_bleu(list_of_refs, hypotheses) s = "Bleu Score = " + str(100 * score) print(s) fout.write(s)
def main(): g = Graph(is_training=False) # Load data nums, X, ys = load_test_data() pnyn2idx, idx2pnyn, hanzi2idx, idx2hanzi = load_vocab() with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto( allow_soft_placement=True)) as sess: # Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") # Get model mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # model name with codecs.open( 'eval/{}_{}.csv'.format( mname, "qwerty" if hp.isqwerty else "nine"), 'w', 'utf-8') as fout: fout.write( "NUM,EXPECTED,{}_{},# characters,edit distance\n".format( mname, "qwerty" if hp.isqwerty else "nine")) total_edit_distance, num_chars = 0, 0 for step in range(len(X) // hp.batch_size): num = nums[step * hp.batch_size:(step + 1) * hp.batch_size] #number batch x = X[step * hp.batch_size:(step + 1) * hp.batch_size] # input batch y = ys[step * hp.batch_size:(step + 1) * hp.batch_size] # batch of ground truth strings preds = sess.run(g.preds, {g.x: x}) for n, xx, pred, expected in zip(num, x, preds, y): # sentence-wise got = "".join( idx2hanzi[idx] for idx in pred)[:np.count_nonzero(xx)].replace( "_", "") edit_distance = distance.levenshtein(expected, got) total_edit_distance += edit_distance num_chars += len(expected) fout.write(u"{},{},{},{},{}\n".format( n, expected, got, len(expected), edit_distance)) fout.write(u"Total CER: {}/{}={},,,,\n".format( total_edit_distance, num_chars, round(float(total_edit_distance) / num_chars, 2)))
def synthesize_part(grp,config,gs,x_train,g_conv): if len(x_train) > hp.batch_size: x_train = random.sample(x_train, hp.batch_size) else: x_train = x_train[0] x_test = load_test_data() rand = random.randint(0,hp.batch_size-1) x_train = x_train[rand] x_test = x_test[rand] wavs = [] if g_conv is None: with grp.graph.as_default(): sv = tf.train.Supervisor(logdir=config.log_dir) with sv.managed_session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: # Restore parameters print(config.log_dir) print("Restoring checkpoint : "+tf.train.latest_checkpoint(config.log_dir)) sv.saver.restore(sess, tf.train.latest_checkpoint(config.log_dir)) wavs = create_write_files(wavs,sess,grp,x_train,"sample_"+str(gs)+"_train_",config.log_dir,"train") #wavs = create_write_files(wavs,sess,grp,x_test,"sample_"+str(gs)+"_test_",config.log_dir,"test") sess.close() else: with grp.graph.as_default(): sv = tf.train.Supervisor(logdir=config.log_dir) with sv.managed_session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: # Restore parameters print("Restoring checkpoint : "+tf.train.latest_checkpoint(config.log_dir)) sv.saver.restore(sess, tf.train.latest_checkpoint(config.log_dir)) mel_out1 = create_mel(sess,grp,x_train) mel_out2 = create_mel(sess,grp,x_test) sess.close() with g_conv.graph.as_default(): sv_conv = tf.train.Supervisor(logdir=config.load_converter) with sv_conv.managed_session(config=tf.ConfigProto(allow_soft_placement=True)) as sess_conv: # Restore parameters print("Restoring checkpoint : "+tf.train.latest_checkpoint(config.load_converter)) sv_conv.saver.restore(sess_conv, tf.train.latest_checkpoint(config.load_converter)) wavs = create_write_files_conv(wavs,sess_conv,mel_out1,g_conv,x_train,"sample_"+str(gs)+"_train_",config.log_dir,"train") wavs = create_write_files_conv(wavs,sess_conv,mel_out2,g_conv,x_test,"sample_"+str(gs)+"_test_",config.log_dir,"test") sess_conv.close() return wavs
def synthesize(): # Load data X = load_test_data() # Load graph g = Graph(training=False); print("Graph loaded") # Inference with g.graph.as_default(): with tf.Session() as sess: saver = tf.train.Saver() # Restore parameters saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)); print("Restored!") # Get model name mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # Synthesize file_id = 1 for i in range(0, len(X), hp.batch_size): x = X[i:i + hp.batch_size] # Get melspectrogram mel_output = np.zeros((hp.batch_size, hp.Ty // hp.r, hp.n_mels * hp.r), np.float32) decoder_output = np.zeros((hp.batch_size, hp.Ty // hp.r, hp.embed_size), np.float32) alignments_li = np.zeros((hp.dec_layers, hp.Tx, hp.Ty//hp.r), np.float32) prev_max_attentions_li = np.zeros((hp.dec_layers, hp.batch_size), np.int32) for j in range(hp.Ty // hp.r): _gs, _mel_output, _decoder_output, _max_attentions_li, _alignments_li = \ sess.run([g.global_step, g.mel_output, g.decoder_output, g.max_attentions_li, g.alignments_li], {g.x: x, g.y1: mel_output, g.prev_max_attentions_li:prev_max_attentions_li}) mel_output[:, j, :] = _mel_output[:, j, :] decoder_output[:, j, :] = _decoder_output[:, j, :] alignments_li[:, :, j] = np.array(_alignments_li)[:, :, j] prev_max_attentions_li = np.array(_max_attentions_li)[:, :, j] # Get magnitude mag_output = sess.run(g.mag_output, {g.decoder_output: decoder_output}) # Generate wav files if not os.path.exists(hp.sampledir): os.makedirs(hp.sampledir) for mag in mag_output: print("Working on file num ", file_id) wav = spectrogram2wav(mag) write(hp.sampledir + "/{}_{}.wav".format(mname, file_id), hp.sr, wav) file_id += 1
def predict(): print('-' * 30) print('Loading and preprocessing test data...') print('-' * 30) imgs_test = load_test_data() imgs_test = imgs_test.astype('float32') imgs_test /= 128. # scale input images to [0, 2] imgs_test = imgs_test - 1. # scale input images to [-1, 1] imgs_test = np.repeat(imgs_test, 3, axis=4) print('-' * 30) print('Loading saved weights...') print('-' * 30) # ----------------------- define model ---------------------------------- model = Unet_vgg.res_unet_vgg(image_depth=img_depth, image_rows=img_rows, image_cols=img_cols) model.summary() weight_dir = 'weights' if not os.path.exists(weight_dir): os.mkdir(weight_dir) model.load_weights(os.path.join(weight_dir, project_name + '.h5')) print('-' * 30) print('Predicting masks on test data...') print('-' * 30) imgs_mask_test = model.predict(imgs_test, batch_size=2, verbose=1) npy_mask_dir = './data/numpy/test_mask' if not os.path.exists(npy_mask_dir): os.mkdir(npy_mask_dir) np.save(os.path.join(npy_mask_dir, project_name + '_mask.npy'), imgs_mask_test) print('-' * 30) print('Saving predicted masks to files...') print('-' * 30) visualise_predicitons(imgs_mask_test, project_name)
def main(): posting_lists, pl_lengths = load_test_data(TEST_FILE, 1024, shuffling=False) pl = posting_lists[0] pl_np = pl.numpy().astype(np.int32) print(pl_np) pl_length = pl.size()[0] print(pl_length) indexes = np.arange(pl_length) points = np.array((indexes, pl_np)).T n_clusters = int(pl_length / (5*32)) print(n_clusters) kmeans = KMeans(n_clusters=n_clusters, random_state=0).fit(points) end_points = label_points(points, kmeans) print(end_points) sub_pl = segments(pl, end_points) print(sub_pl) plot_distributions(pl, end_points)
def eval(): g = train_Graph(is_training=False) print('Graph loaded') X, Sources, Targets = load_test_data() de2idx, idx2de = load_de_vocab() en2idx, idx2en = load_en_vocab() with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print('Restored') mname = open(hp.logdir + '/checkpoint', 'r').read().split('')[1] # model name if not os.path.exists('results'): os.mkdir('results') with codecs.open('results/' + mname, 'w', 'utf-8') as fout: list_of_refs, hypotheses = [], [] for i in range(len(X) // hp.batch_size): x = X[i*hp.batch_size:(i+1)*hp.batch_size] sources = Sources[i*hp.batch_size:(i+1)*hp.batch_size] targets = Targets[i*hp.batch_size:(i+1)*hp.batch_size] preds = np.zeros((hp.batch_size, hp.max_seq_len), np.int32) for j in range(hp.max_seq_len): '''每个词每个词地预测。这样,后一个词预测的时候就可以利用前面的信息来解码。 所以一共循环hp.max_len次,每次循环用之前的翻译作为解码器的输入翻译的一个词。''' _preds = sess.run(g.preds, {g.x:x, g.y:preds}) preds[:, j] = _preds[:, j] for source, target, pred in zip(sources, targets, preds): got = ''.join(idx2en[idx] for idx in pred).split('</S>')[0].strip() fout.write('-source:' + source + '\n') fout.write('-expected:' + target + '\n') fout.write('-got:' + got + '\n\n') fout.flush() # bleu score ref = target.split() hypothesis = got.split() if len(ref) > 3 and len(hypothesis) > 3: list_of_refs.append([ref]) hypotheses.append(hypothesis) score = corpus_bleu(list_of_refs, hypotheses) fout.write('Bleu Score = ' + str(100*score))
def synthesize_part(grp,config,gs,x_train): x_train = random.sample(x_train, hp.batch_size) x_test = load_test_data() rand = random.randint(0,hp.batch_size-1) x_train = x_train[rand] x_test = x_test[rand] wavs = [] with grp.graph.as_default(): sv = tf.train.Supervisor(logdir=config.log_dir) with sv.managed_session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: # Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(config.log_dir)) wavs = create_write_files(wavs,sess,grp,x_train,"sample_"+str(gs)+"_train_",config.log_dir,"train") wavs = create_write_files(wavs,sess,grp,x_test,"sample_"+str(gs)+"_test_",config.log_dir,"test") sess.close() return wavs
def synthesize(): # Load graph g = Graph(training=False) print("Graph loaded") x = load_test_data() with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto( allow_soft_placement=True)) as sess: # Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") # Get model name mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # Inference mels = np.zeros((hp.batch_size, hp.T_y // hp.r, hp.n_mels * hp.r), np.float32) prev_max_attentions = np.zeros((hp.batch_size, ), np.int32) for j in range(hp.T_x): _mels, _max_attentions = sess.run( [g.mels, g.max_attentions], { g.x: x, g.y1: mels, g.prev_max_attentions: prev_max_attentions }) mels[:, j, :] = _mels[:, j, :] prev_max_attentions = _max_attentions[:, j] mags = sess.run(g.mags, {g.mels: mels}) # Generate wav files if not os.path.exists(hp.sampledir): os.makedirs(hp.sampledir) for i, mag in enumerate(mags): # generate wav files mag = mag * hp.mag_std + hp.mag_mean # denormalize audio = spectrogram2wav(np.exp(mag)) write(hp.sampledir + "/{}_{}.wav".format(mname, i), hp.sr, audio)
def main_batches(): g = Graph(is_training=False) # Load data nums, X, ys = load_test_data() pnyn2idx, idx2pnyn, hanzi2idx, idx2hanzi = load_vocab() with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: # Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)); print("Restored!") # Get model mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # model name with codecs.open('eval/{}_{}.csv'.format(mname, "qwerty" if hp.isqwerty else "nine"), 'w', 'utf-8') as fout: fout.write("NUM,EXPECTED,{}_{},# characters,edit distance\n".format(mname, "qwerty" if hp.isqwerty else "nine")) total_edit_distance, num_chars = 0, 0 for step in range(len(X)//hp.batch_size): num = nums[step*hp.batch_size:(step+1)*hp.batch_size] #number batch x = X[step*hp.batch_size:(step+1)*hp.batch_size] # input batch y = ys[step*hp.batch_size:(step+1)*hp.batch_size] # batch of ground truth strings preds = sess.run(g.preds, {g.x: x}) for n, xx, pred, expected in zip(num, x, preds, y): # sentence-wise #got = "".join(idx2hanzi[str(idx)] for idx in pred)[:np.count_nonzero(xx)].replace("_", "") got = "".join(idx2hanzi[idx] for idx in pred)[:np.count_nonzero(xx)].replace("_", "") edit_distance = distance.levenshtein(expected, got) total_edit_distance += edit_distance num_chars += len(expected) fout.write(u"{},{},{},{},{}\n".format(n, expected, got, len(expected), edit_distance)) fout.write(u"Total CER: {}/{}={},,,,\n".format(total_edit_distance, num_chars, round(float(total_edit_distance)/num_chars, 2)))
X, Y = load_train_data(de2idx, en2idx) np.save(en_npy_path, X) np.save(zh_npy_path, Y) # load test data test_en_path = "./processed-data/test_en.npy" test_s_path = "./processed-data/t_source.npy" test_t_path = "./processed-data/t_target.npy" if os.path.exists(test_en_path) and os.path.exists( test_s_path) and os.path.exists(test_t_path): print("load testing data") X_test = np.load(test_en_path) Source_test = np.load(test_s_path) Target_test = np.load(test_t_path) else: X_test, Source_test, Target_test = load_test_data(de2idx, en2idx) np.save(test_en_path, X_test) np.save(test_s_path, Source_test) np.save(test_t_path, Target_test) # config os.environ['TF_CPP_MIN_LOG_LEVEL'] = "3" os.environ["CUDA_VISIBLE_DEVICES"] = "2" config = tf.ConfigProto(allow_soft_placement=False, log_device_placement=False) config.gpu_options.allow_growth = True class TranslateDemo(): """ """ def __init__(self, is_training=True, optimizer="Adam"):
def eval(hp): # Load graph g = Graph(hp=hp, is_training=False) print("Graph loaded") # Load data X, X_image, X_length, Y, Sources, Targets, X_turn_number, SRC_emotion, TGT_emotion, Speakers, A = load_test_data( hp) #print(X) de2idx, idx2de = load_de_vocab(hp) en2idx, idx2en = load_en_vocab(hp) # Start session with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto( allow_soft_placement=True)) as sess: ## Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") ## Get model name mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # model name #fftmp=open("tmp.txt","w") ## Inference if not os.path.exists('results'): os.mkdir('results') with codecs.open("results/" + mname, "w", "utf-8") as fout: list_of_refs, hypotheses, test_loss = [], [], [] for i in range(len(X) // hp.batch_size): ### Get mini-batches x = X[i * hp.batch_size:(i + 1) * hp.batch_size] x_length = X_length[i * hp.batch_size:(i + 1) * hp.batch_size] y = Y[i * hp.batch_size:(i + 1) * hp.batch_size] x_emotion = SRC_emotion[i * hp.batch_size:(i + 1) * hp.batch_size] speaker = Speakers[i * hp.batch_size:(i + 1) * hp.batch_size] x_image = X_image[i * hp.batch_size:(i + 1) * hp.batch_size] a = A[i * hp.batch_size:(i + 1) * hp.batch_size] sources = Sources[i * hp.batch_size:(i + 1) * hp.batch_size] targets = Targets[i * hp.batch_size:(i + 1) * hp.batch_size] eval_bath = sess.run( g.mean_loss, { g.x: x, g.x_image: x_image, g.x_length: x_length, g.y: y, g.x_emotion: x_emotion, g.speaker: speaker, g.A: a, g.x_turn_number: x_turn_number }) test_loss.append(eval_bath) ### Autoregressive inference preds = np.zeros((hp.batch_size, hp.maxlen), np.int32) for j in range(hp.maxlen): _preds = sess.run(g.preds, { g.x: x, g.x_length: x_length, g.y: preds }) preds[:, j] = _preds[:, j] ### Write to file for source, target, pred in zip(sources, targets, preds): # sentence-wise got = " ".join( idx2en[idx] for idx in pred).split("</S>")[0].strip() fout.write("- source: " + source + "\n") fout.write("- expected: " + target + "\n") fout.write("- got: " + got + "\n\n") fout.flush() # bleu score #ref = target.split() ref = target.split(u"</d>")[1].split() hypothesis = got.split() if len(ref) > 3 and len(hypothesis) > 3: list_of_refs.append([ref]) hypotheses.append(hypothesis) ## Calculate bleu score score = corpus_bleu(list_of_refs, hypotheses) fout.write("Test Bleu Score = " + str(100 * score)) print("Test Bleu Score = " + str(100 * score)) print("eval PPL = %.5lf" % (round(math.exp(np.mean(test_loss)), 4))) print("eval loss = %.5lf" % (np.mean(test_loss))) # Distinct-1, Distinct-2 candidates = [] for line in hypotheses: candidates.extend(line) distinct_1, distinct_2 = cal_Distinct(candidates) print('Distinct-1:' + str(round(distinct_1, 4)) + 'Distinct-2:' + str(round(distinct_2, 4)))
def eval(): # Load vocabulary token2idx, idx2token = load_de_en_vocab() tw2idx, idx2tw = load_tw_vocab() token2idx_len = len(token2idx) tw2idx_len = len(tw2idx) # Load vocab_overlap token_idx_list = [] con_list = np.zeros([4, token2idx_len], dtype='float32') for i in range(4, tw2idx_len): tw = idx2tw[i] token_idx_list.append(token2idx[tw]) vocab_overlap = np.append(con_list, np.eye(token2idx_len, dtype='float32')[token_idx_list], axis=0) # Load graph g = Graph(False, token2idx_len, tw2idx_len, vocab_overlap) print("Graph loaded") # Load data X, X_length, Y, TW, Sources, Targets = load_test_data() # Start session with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto( allow_soft_placement=True)) as sess: ## Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") ## Get model name mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # model name ## Inference if not os.path.exists('results'): os.mkdir('results') with codecs.open("results/" + mname, "w", "utf-8") as fout: args = parse_args() if args.a: att_f = codecs.open("results/" + "attention_vis", "w", "utf-8") for i in range(len(X) // hp.batch_size): ### Get mini-batches x = X[i * hp.batch_size:(i + 1) * hp.batch_size] x_length = X_length[i * hp.batch_size:(i + 1) * hp.batch_size] y = Y[i * hp.batch_size:(i + 1) * hp.batch_size] y_tw = TW[i * hp.batch_size:(i + 1) * hp.batch_size] sources = Sources[i * hp.batch_size:(i + 1) * hp.batch_size] targets = Targets[i * hp.batch_size:(i + 1) * hp.batch_size] preds = np.zeros((hp.batch_size, hp.maxlen), np.int32) ppls = np.zeros((hp.batch_size, hp.maxlen), np.int32) for j in range(hp.maxlen): _preds, ppl_step, att_ws, att_us, att_v = sess.run( [g.preds, g.ppl, g.att_w, g.att_u, g.att_v], { g.x: x, g.x_length: x_length, g.y: y, g.y_tw: y_tw, g.y_decoder_input: preds }) preds[:, j] = _preds[:, j] ppls[:, j] = ppl_step[:, j] if args.a: att_ws = np.mean(np.split(att_ws, hp.num_heads, axis=0), axis=0) # (N, L, L) att_us = np.mean(np.split(att_us, hp.num_heads, axis=0), axis=0) # (N, T, T) att_ws = np.reshape( att_ws, [hp.batch_size, hp.max_turn, hp.maxlen, hp.maxlen]) att_ws = np.mean(att_ws, axis=2) # N, T, L att_ws = np.reshape( att_ws, [hp.batch_size, hp.max_turn * hp.maxlen]) att_us = np.sum(att_us, axis=1) # N, T ### Write to file for source, target, pred, ppl, att_w, att_u in zip( sources, targets, preds, ppls, att_ws, att_us): # sentence-wise got = " ".join( idx2token[idx] for idx in pred).split("</S>")[0].strip() if len(got.split()) > hp.gener_maxlen: pred = pred.tolist() pred_final = list(set(pred)) pred_final.sort(key=pred.index) else: pred_final = pred got = " ".join( idx2token[idx] for idx in pred_final).split("</S>")[0].strip() fout.write("- source: " + source + "\n") fout.write("- expected: " + target + "\n") fout.write("- got: " + got + "\n\n") fout.write("- ppl_score: " + " ".join('%s' % np.mean(ppl)) + "\n\n") if args.a: att_f.write("- att_w: " + str(att_w) + "\n") att_f.write("- att_u: " + str(att_u) + "\n\n\n\n\n") fout.flush() if args.a: att_f.flush()
def eval(): # Load vocabulary token2idx, idx2token = load_de_en_vocab() tw2idx, idx2tw = load_tw_vocab() token2idx_len = len(token2idx) tw2idx_len = len(tw2idx) # Load vocab_overlap token_idx_list = [] con_list = np.zeros([4, token2idx_len],dtype='float32') for i in range(4, tw2idx_len): tw = idx2tw[i] token_idx_list.append(token2idx[tw]) vocab_overlap = np.append(con_list, np.eye(token2idx_len, dtype='float32')[token_idx_list], axis=0) # Load graph g = Graph(False, token2idx_len, tw2idx_len, vocab_overlap) print("Graph loaded") # Load data X, X_length, Y, TW, Sources, Targets = load_test_data() # Start session with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: ## Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") ## Get model name mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # model name #fftmp=open("tmp.txt","w") ## Inference if not os.path.exists('results'): os.mkdir('results') with codecs.open("results/" + mname, "w", "utf-8") as fout: list_of_refs, hypotheses = [], [] for i in range(len(X) // hp.batch_size): ### Get mini-batches x = X[i * hp.batch_size: (i + 1) * hp.batch_size] x_length= X_length[i * hp.batch_size: (i + 1) * hp.batch_size] y = Y[i * hp.batch_size: (i + 1) * hp.batch_size] y_tw = TW[i * hp.batch_size: (i + 1) * hp.batch_size] sources = Sources[i * hp.batch_size: (i + 1) * hp.batch_size] targets = Targets[i * hp.batch_size: (i + 1) * hp.batch_size] #fftmp.write("%s\n"%(" ".join(str(w) for w in x[0][0]).encode("utf-8"))) #fftmp.write("%s\n"%(sources[0].encode("utf-8"))) #fftmp.write("%s\n"%(' '.join(str(w) for w in x_length))) #print (sources) #print (targets) ### Autoregressive inference preds = np.zeros((hp.batch_size, hp.maxlen), np.int32) ppls = np.zeros((hp.batch_size, hp.maxlen), np.int32) for j in range(hp.maxlen): _preds, ppl_step, att_w, att_u, att_v = sess.run([g.preds, g.ppl_step, g.att_w, g.att_u, g.att_v], {g.x:x, g.x_length:x_length, g.y:y, g.y_tw:y_tw, g.y_decoder_input:preds}) preds[:, j] = _preds[:, j] ppls[:, j] = ppl_step[:, j] ### Write to file for source, target, pred, ppl in zip(sources, targets, preds, ppls): # sentence-wise got = " ".join(idx2token[idx] for idx in pred).split("</S>")[0].strip() ppl_score = " ".join('%s' %score for score in ppl).strip() fout.write("- source: " + source +"\n") fout.write("- expected: " + target + "\n") fout.write("- got: " + got + "\n\n") fout.write("- ppl_score: " + ppl_score + "\n\n") fout.flush() # bleu score ref = target.split() hypothesis = got.split() if len(ref) > 3 and len(hypothesis) > 3: list_of_refs.append([ref]) hypotheses.append(hypothesis) ## Calculate attention #fout.write("- att_w: " + str(att_w) + "\n") #fout.write("- att_u: " + str(att_u) + "\n") #fout.write("- att_v: " + str(att_v) + "\n") #fout.flush() ## Calculate bleu score score = corpus_bleu(list_of_refs, hypotheses) fout.write("Bleu Score = " + str(100*score))
def main(): posting_lists, pl_lengths = load_test_data(TEST_FILE, 128, shuffling=True) pl = posting_lists[0] plot_distributions(pl)
def eval(): # Load graph g = Graph(is_training=False) print("Graph loaded") # Load data nums, X, gts = load_test_data() roma2idx, idx2roma, surf2idx, idx2surf = load_vocab() with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto( allow_soft_placement=True)) as sess: # Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") # Get model mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # model name with codecs.open( 'results/{}_{}_beam_width_{}.csv'.format( hp.norm_type, mname, hp.beam_width), 'w', 'utf-8') as fout: fout.write( "NUM,EXPECTED,{}_beam_width_{},# characters,edit distance\n" .format(mname, hp.beam_width)) total_edit_distance = 0 num_chars = 0 # number of total characters for step in range(len(X) // hp.batch_size): num = nums[step * hp.batch_size:(step + 1) * hp.batch_size] #number batch x = X[step * hp.batch_size:(step + 1) * hp.batch_size] # input batch gt = gts[step * hp.batch_size:(step + 1) * hp.batch_size] # batch of ground truth strings if hp.beam_width == 1: preds = np.zeros((hp.batch_size, hp.max_len), np.int32) for j in range(hp.max_len): _preds = sess.run(g.preds, {g.x: x, g.y: preds}) preds[:, j] = _preds[:, j] else: # beam decode ## first step preds = np.zeros( (hp.beam_width * hp.batch_size, hp.max_len), np.int32) # (bw*N, T) logprobs = sess.run( g.logprobs, { g.x: x, g.y: np.zeros((hp.batch_size, hp.max_len), np.int32) }) # (N, T, V) target = logprobs[:, 0, :] # (N, V) preds_in_beam = target.argsort( )[:, ::-1][:, :hp.beam_width].flatten() # (bw*N,) preds[:, 0] = preds_in_beam logp_in_beam = np.sort( target)[:, ::-1][:, :hp.beam_width].flatten( ) # (bw*N,) logp_in_beam = np.repeat(logp_in_beam, hp.beam_width, axis=0) # (bw*bw*N, ) ## remaining steps for i in range(1, hp.max_len - 1): logprobs = sess.run( g.logprobs, { g.x: np.repeat(x, hp.beam_width, 0), g.y: preds }) # (bw*N, T, V) target = logprobs[:, i, :] # (bw*N, V) preds_in_beam = target.argsort( )[:, ::-1][:, :hp.beam_width].flatten( ) # (bw*bw*N,) logp_in_beam += np.sort( target)[:, ::-1][:, :hp.beam_width].flatten( ) # (bw*bw*N, ) preds = np.repeat( preds, hp.beam_width, axis=0 ) # (bw*bw*N, T) <- Temporary shape expansion preds[:, i] = preds_in_beam elems = [ ] # (bw*N). bw elements are selected out of bw^2 for j, cluster in enumerate( np.split( logp_in_beam, hp.batch_size)): # cluster: (bw*bw,) if i == hp.max_len - 2: # final step elem = np.argsort( cluster)[::-1][:1] # final 1 best elems.extend(list(elem + j * len(cluster))) else: elem = np.argsort( cluster)[::-1][:hp.beam_width] elems.extend(list(elem + j * len(cluster))) preds = preds[ elems] # (N, T) if final step, (bw*N, T) otherwise. <- shape restored logp_in_beam = logp_in_beam[elems] logp_in_beam = np.repeat(logp_in_beam, hp.beam_width, axis=0) # (bw*bw*N, ) # for l, pred in enumerate(preds[:hp.beam_width]): # fout.write(str(l) + " " + u"".join(idx2surf[idx] for idx in pred).split("S")[0] + "\n") for n, pred, expected in zip(num, preds, gt): # sentence-wise got = "".join(idx2surf[idx] for idx in pred).split("S")[0] edit_distance = distance.levenshtein(expected, got) total_edit_distance += edit_distance num_chars += len(expected) fout.write(u"{},{},{},{},{}\n".format( n, expected, got, len(expected), edit_distance)) fout.write(u"Total CER: {}/{}={},,,,\n".format( total_edit_distance, num_chars, round(float(total_edit_distance) / num_chars, 2)))
def eval(): # Load graph g = Graph(is_training=False) print("Graph loaded") # Load data X, Sources, Targets = load_test_data() de2idx, idx2de = load_de_vocab() en2idx, idx2en = load_en_vocab() # X, Sources, Targets = X[:33], Sources[:33], Targets[:33] # Start session with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto( allow_soft_placement=True)) as sess: ## Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") ## Get model name mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # model name ## Inference if not os.path.exists('results'): os.mkdir('results') with codecs.open("results/" + mname, "w", "utf-8") as fout: list_of_refs, hypotheses = [], [] for i in range(len(X) // hp.batch_size): ### Get mini-batches x = X[i * hp.batch_size:(i + 1) * hp.batch_size] sources = Sources[i * hp.batch_size:(i + 1) * hp.batch_size] targets = Targets[i * hp.batch_size:(i + 1) * hp.batch_size] predx = np.zeros((hp.batch_size, hp.maxlen, hp.beam_width), np.int32) predx_prob = np.zeros_like(predx, np.float64) logits = np.zeros((hp.batch_size, hp.maxlen, len(en2idx)), np.float64) print(x[1:2, :]) for j in range( hp.batch_size ): #For testing, the range will be changed to accelerate the testing for j in range(hp.maxlen) print(j) preds_sent = np.zeros((1, hp.maxlen, hp.beam_width)) probs_sent = np.zeros_like(preds_sent, np.float64) #probs_ref = np.zeros_like(preds_sent, np.float64) x_a = x[j:j + 1, :] #input one sentence each time sent_len = x_a[0, :].tolist().index(0) #print(x_a) preds = np.zeros((1, hp.maxlen), np.int32) preds_prob = np.zeros_like(preds, np.float64) _logits = np.array( sess.run(g.logits, { g.x: x_a, g.y: preds })) sent_j = _logits[0, 0] #print(sent_j) sos = sent_j.argsort( )[-1:] #retrieve the token of first character (Start of sentence) preds[ 0, 0] = sos #settle the sos token at the beginning of preds sos_prob = sent_j[sos] preds_prob[0, 0] = sos_prob #print(preds[0,0]) for bw in range(hp.beam_width): preds_sent[0, 0, bw] = preds[0, 0] probs_sent[0, 0, bw] = preds_prob[0, 0] #print(probs_sent) _logits = np.array( sess.run(g.logits, { g.x: x_a, g.y: preds })) sent_j = _logits[0] word_1 = sent_j[1] word_1 = word_1 + preds_prob[0, 0] top_bw_idx = word_1.argsort()[-hp.beam_width:] #print(top_bw_idx) top_bw_probs = word_1[top_bw_idx] #print(top_bw_probs) for bw in range(hp.beam_width): preds_sent[0, 1, bw] = np.copy(top_bw_idx[bw]) #print(top_bw_probs[bw]) probs_sent[0, 1, bw] = top_bw_probs[bw] #print(probs_sent) #settle top_bw tokens for the second character (first word) #print(probs_sent) for k in range( 2, hp.maxlen): #this part need special design added_probs = [] paths_candidate = [] preds_prob_list = [] for bw in range(hp.beam_width): preds[0, :] = preds_sent[0, :, bw].copy() preds_prob[0, :] = probs_sent[0, :, bw].copy() #print(preds_prob) if (preds_sent[0, k - 1, bw] == 3): preds_sent[0, k, bw] = 3 current_path = preds_sent[0, :, bw] new_path = np.copy(current_path) new_path[k] = 3 paths_candidate.append(new_path) preds_prob[0, k] = 0 current_preds_prob = np.copy(preds_prob) print(current_preds_prob) added_probs = np.concatenate( (added_probs, [np.sum(current_preds_prob[0])]), 0) preds_prob_list.append(current_preds_prob) if (preds_sent[0, k - 1, bw] != 3): current_path = preds_sent[0, :, bw] _logits = np.array( sess.run(g.logits, { g.x: x_a, g.y: preds })) sent_j = _logits[0] word_k = sent_j[ k] #+np.sum(preds_prob[0]) #log(a*b) = log a + log b top_bw_idx = word_k.argsort( )[-hp.beam_width:] top_bw_probs = sent_j[k][top_bw_idx] for bmw in range(hp.beam_width): new_path = np.copy(current_path) new_path[k] = top_bw_idx[bmw] current_step_probs = top_bw_probs[bmw] current_path_probs = np.copy( preds_prob[0]) current_path_probs[ k] = current_step_probs added_probs = np.concatenate( (added_probs, [np.sum(current_path_probs)]), 0) #print(new_path) paths_candidate.append(new_path) preds_prob_list.append( current_path_probs) #print("what hell is going on") #print(sub_candidates) #print("this is a =========") a_idx = np.array( added_probs).argsort()[-hp.beam_width:] a_prob = added_probs[a_idx] #print(a_prob) print(preds_prob_list) for bw in range(hp.beam_width): preds_sent[0, :, bw] = np.copy( paths_candidate[a_idx[bw]]) #print(paths_candidate[a_idx[bw]]) #print(preds_sent[0, :, bw]) probs_sent[0, :, bw] = np.copy(preds_prob_list[bw]) print(probs_sent) #print("probs_sent:") #print(probs_sent) predx[j, :, :] = preds_sent predx_prob[j, :, :] = probs_sent #print("checkpoint") #sys.exit() ### Write to file print("done") for source, target, pred, prob in zip( sources, targets, predx, predx_prob): # sentence-wise candits = [] candits_probs = [] for i in range(hp.beam_width): pres = pred[:, i] pros = prob[:, i] got = "".join( idx2en[idx] for idx in pres).split("</S>")[0].strip() candits.append(got) candits_probs.append(pros) fout.write("- source: " + source + "\n") fout.write("- expected: " + target + "\n") print(candits) for i in range(len(candits)): fout.write("- got: " + candits[i] + "\n") m = len(candits[i]) fout.write(' '.join( str(each) for each in candits_probs[i].tolist() [:m - 2])) #each for each in fout.write("\n") fout.write("\n") fout.flush() # bleu score ref = target.split() hypothesis = got.split() if len(ref) > 3 and len(hypothesis) > 3: list_of_refs.append([ref]) hypotheses.append(hypothesis)
def eval(task_name): # Load graph g = Graph(is_training=False) print("Graph loaded") # Load data X, Sources, Targets = load_test_data() #print(X, Sources, Targets) en2idx, idx2en = load_en_vocab() zh2idx, idx2zh = load_zh_vocab() # Start session with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto( allow_soft_placement=True)) as sess: ## Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") ## Get model name print('Model dir:', hp.logdir) mname = '{}'.format(''.join( hp.source_test.split('/')[-1].split('.', 3)[:-1])) + open( hp.logdir + '/checkpoint', 'r').read().split('"')[1] # model name print("Model name:", mname) ## Inference if not os.path.exists('results'): os.mkdir('results') with codecs.open("results/" + mname, "w", "utf-8") as fout: list_of_refs, hypotheses, scores = [], [], [] print("Iterator:", len(X), hp.batch_size) for i in range(len(X) // hp.batch_size): print('Step:\t', i) ### Get mini-batches x = X[i * hp.batch_size:(i + 1) * hp.batch_size] sources = Sources[i * hp.batch_size:(i + 1) * hp.batch_size] targets = Targets[i * hp.batch_size:(i + 1) * hp.batch_size] ### Autoregressive inference preds = np.zeros((hp.batch_size, hp.maxlen), np.int32) for j in range(hp.maxlen): _preds = sess.run(g.preds, {g.x: x, g.y: preds}) preds[:, j] = _preds[:, j] ### Write to file for source, target, pred in zip(sources, targets, preds): # sentence-wise #print('Inspecting:', source, target, pred) #got = " ".join(idx2zh[idx] for idx in pred).split("。", 2)[0].strip() + ' 。' #got = ''.join(idx2zh[idx] for idx in pred).split('。')[0].strip() got = ' '.join(idx2zh[idx] for idx in pred).split('</S>')[0] if task_name == 'jieba': fout.write("- source: " + source + "\n") fout.write("- expected: " + ' '.join(cut(source, target)) + "\n") fout.write("- got: " + ' '.join(cut(source, got)) + "\n\n") fout.flush() else: fout.write("- source: " + source + "\n") fout.write("- expected: " + target + "\n") fout.write("- got: " + got + "\n\n") fout.flush() # accumlate accuracty ref = cut(source, target) hypothesis = cut(source, got) acc = len([x for x in hypothesis if x in ref]) / len(ref) scores.append(min(1, acc)) ## Calculate bleu score #score = corpus_bleu(list_of_refs, hypotheses) fout.write("Tokenization Accuracy = " + str(100 * (sum(scores) / len(scores))))
def eval(task_name): # Load graph g = Graph(is_training=False) print("Graph loaded") # Load data test_data = pd.read_csv(hp.testfile) questions, contents, q_lens, p_lens, start_pos, end_pos = load_test_data() raw_passages = list(test_data['content']) reference_answers = list(test_data['answer']) word2idx, idx2word = load_vocabs() # Start session with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto( allow_soft_placement=True)) as sess: ## Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") ## Get model name print('Model dir:', hp.logdir) mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # model name print("Model name:", mname) ## Inference if not os.path.exists('results'): os.mkdir('results') with codecs.open("results/" + mname, "w", "utf-8") as fout: pred_answers, ref_answers = [], [] pred_dict, ref_dict = {}, {} ques_id = 0 eval_dict = { 'bleu_1': [], 'bleu_2': [], 'bleu_3': [], 'bleu_4': [] } for i in range(len(questions) // hp.batch_size): print("Iterator: {} / {}".format( i, len(questions) // hp.batch_size)) ### Get mini-batches q = questions[i * hp.batch_size:(i + 1) * hp.batch_size] p = contents[i * hp.batch_size:(i + 1) * hp.batch_size] q_length = q_lens[i * hp.batch_size:(i + 1) * hp.batch_size] p_length = p_lens[i * hp.batch_size:(i + 1) * hp.batch_size] s_pos = start_pos[i * hp.batch_size:(i + 1) * hp.batch_size] e_pos = end_pos[i * hp.batch_size:(i + 1) * hp.batch_size] passages = raw_passages[i * hp.batch_size:(i + 1) * hp.batch_size] ref_answers = reference_answers[i * hp.batch_size:(i + 1) * hp.batch_size] feed_dict = { g.q: q, g.p: p, g.q_length: q_length, g.p_length: p_length, g.start_label: s_pos, g.end_label: e_pos } start_probs, end_probs = sess.run( [g.start_probs, g.end_probs], feed_dict) ### Write to file for start_prob, end_prob, passage, ref in zip( start_probs, end_probs, passages, ref_answers): pred_span, prob = find_best_answer_for_passage( start_prob, end_prob) pred_answer = passage[pred_span[0]:pred_span[1] + 1] if not len(pred_answer) > 0: continue pred_dict[str(ques_id)] = [pred_answer] ref_dict[str(ques_id)] = [ref] ques_id += 1 fout.write('-ref: ' + ref) fout.write("-pred: " + pred_answer) b1, b2, b3, b4 = bleu(list(pred_answer), list(ref), 1), \ bleu(list(pred_answer), list(ref), 2), \ bleu(list(pred_answer), list(ref), 3), \ bleu(list(pred_answer), list(ref), 4) eval_dict['bleu_1'].append(b1) eval_dict['bleu_2'].append(b2) eval_dict['bleu_3'].append(b3) eval_dict['bleu_2'].append(b2) for metric in eval_dict: fout.write(metric + '\t' + str(np.mean(eval_dict[metric])) + '\n') print(metric + '\t' + str(np.mean(eval_dict[metric])))
def eval(): # Load graph g = Graph(is_training=False) print("Graph loaded") # Load data X, Sources, Targets = load_test_data() en2idx, idx2en = load_en_vocab() de2idx, idx2de = load_de_vocab() # Start session with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto( allow_soft_placement=True)) as sess: # Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") # Get model name mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # model name # Inference if not os.path.exists('results'): os.mkdir('results') with codecs.open("results/" + mname, "w", "utf-8") as fout: list_of_refs, hypotheses = [], [] for i in range(len(X) // hp.batch_size): # Get mini-batches x = X[i * hp.batch_size:(i + 1) * hp.batch_size] sources = Sources[i * hp.batch_size:(i + 1) * hp.batch_size] targets = Targets[i * hp.batch_size:(i + 1) * hp.batch_size] # Autoregressive inference preds = np.zeros((hp.batch_size, hp.maxlen), np.int32) for j in range(hp.maxlen): tensors = [g.preds] + list( g.tensors_of_interest.values()) tensors_out = sess.run(tensors, {g.x: x, g.y: preds}) _preds = tensors_out[0] preds[:, j] = _preds[:, j] print([idx2de[idx] for idx in preds[0]]) # For the first few batches, we save figures giving the attention structure in the encoder. if j == 0 and i < batches_to_visualize: tensor_keys = [None] + list( g.tensors_of_interest.keys() ) # Add a null key at the start so it lines up with the tensors_out list visualizeEncoderAttention( sources=sources, idx2en=idx2en, tensors_of_interest={ key: value for key, value in zip( tensor_keys, tensors_out) }, batch_index=i) # Write to file for source, target, pred in zip(sources, targets, preds): # sentence-wise got = " ".join( idx2de[idx] for idx in pred).split("</S>")[0].strip() fout.write("- source: " + source + "\n") fout.write("- expected: " + target + "\n") fout.write("- got: " + got + "\n\n") fout.flush() # bleu score ref = target.split() hypothesis = got.split() if len(ref) > 3 and len(hypothesis) > 3: list_of_refs.append([ref]) hypotheses.append(hypothesis) # Calculate bleu score score = corpus_bleu(list_of_refs, hypotheses) fout.write("Bleu Score = " + str(100 * score))
def eval(): # Load graph g = Graph(is_training=False) print("Graph loaded") # Load data X, Sources, Targets = load_test_data() de2idx, idx2de = load_de_vocab() en2idx, idx2en = load_en_vocab() # X, Sources, Targets = X[:33], Sources[:33], Targets[:33] gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5) # Start session with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto(allow_soft_placement=True, gpu_options=gpu_options)) as sess: ## Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") ## Get model name mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # model name ## Inference totalTransNum = 0 if not os.path.exists('results'): os.mkdir('results') with codecs.open('results/'+mname+'.trans', 'w', 'utf8') as tfout: with codecs.open("results/" + mname, "w", "utf-8") as fout: list_of_refs, hypotheses = [], [] for i in range((len(X) // hp.batch_size) + 1): ### Get mini-batches batchEnd = (i+1)*hp.batch_size readlBatchSize = hp.batch_size if batchEnd > len(X): readlBatchSize = hp.batch_size - (batchEnd - len(X)) batchEnd = len(X) x = X[i*hp.batch_size: batchEnd] sources = Sources[i*hp.batch_size: batchEnd] targets = Targets[i*hp.batch_size: batchEnd] totalTransNum += len(sources) ### Autoregressive inference preds = np.zeros((readlBatchSize, hp.maxlen), np.int32) for j in range(hp.maxlen): _preds = sess.run(g.preds, {g.x: x, g.y: preds}) preds[:, j] = _preds[:, j] ### Write to file for source, target, pred in zip(sources, targets, preds): # sentence-wise got = " ".join(idx2en[idx] for idx in pred).split("</S>")[0].strip() fout.write("- source: " + source +"\n") fout.write("- expected: " + target + "\n") fout.write("- got: " + got + "\n\n") tfout.write(got) tfout.write('\n') # bleu score ref = target.split() hypothesis = got.split() if len(ref) > 3 and len(hypothesis) > 3: list_of_refs.append([ref]) hypotheses.append(hypothesis) ## Calculate bleu score score = corpus_bleu(list_of_refs, hypotheses) fout.write("Bleu Score = " + str(100*score)) fout.write('\n') print('totalTransNum', totalTransNum, 'Bleu', str(100*score))
def eval(): # Load graph g = TransformerDecoder(is_training=False) print("Graph loaded") # Load data X, sources, actual_lengths = load_test_data() sorted_lengths = np.argsort(actual_lengths) X = X[sorted_lengths] print(X.shape) src2idx, idx2src = load_source_vocab() tgt2idx, idx2tgt = load_target_vocab() # Start session config = tf.ConfigProto() config.gpu_options.per_process_gpu_memory_fraction = 0.4 with tf.Session(graph=g.graph, config=config) as sess: saver = tf.train.Saver(tf.global_variables()) saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") ## Inference if not os.path.exists('results'): os.mkdir('results') with codecs.open("results/{}.txt".format(hp.logdir), "w", "utf-8") as fout: batch_size = hp.batch_size num_batches = math.ceil(len(X) / batch_size) Y_preds = np.zeros_like(X) + 2 for i in tqdm(range(num_batches), desc="Inference: "): indices = np.arange(i * batch_size, min((i + 1) * batch_size, len(X))) step = 0 max_steps = math.ceil( (np.max(actual_lengths[indices]) - hp.offset) / (hp.maxlen - hp.offset)) max_steps = max(1, max_steps) for step in range(max_steps): end = min(step * (hp.maxlen - hp.offset) + hp.maxlen, X.shape[1]) start = end - hp.maxlen x = X[indices, start:end] _preds = sess.run(g.preds, {g.x: x, g.dropout: False}) if step > 0: Y_preds[indices, start + hp.offset // 2:end] = _preds[:, hp.offset // 2:] else: Y_preds[indices, start:end] = _preds Y_preds = Y_preds[np.argsort(sorted_lengths)] for source, preds, actual_length in zip(sources, Y_preds, actual_lengths): formatted_pred = [ idx2tgt[idx] if src2idx.get(source[id], 1) > 8 else source[id] for id, idx in enumerate(preds[:actual_length]) ] fout.write(" ".join(formatted_pred) + "\n") model_vars = tf.trainable_variables() slim.model_analyzer.analyze_vars(model_vars, print_info=True) slim.model_analyzer.analyze_ops(g.graph, print_info=True)
def eval(task_name): # Load graph g = Graph(is_training=False) print("Graph loaded") # Load data X, _, Texts, Labels = load_test_data() word2idx, idx2word = load_vocabs() # Start session with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto( allow_soft_placement=True)) as sess: ## Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") ## Get model name print('Model dir:', hp.logdir) mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # model name print("Model name:", mname) ## Inference if not os.path.exists('results'): os.mkdir('results') with codecs.open("results/" + mname, "w", "utf-8") as fout: list_of_refs, hypotheses = [], [] print("Iterator:", len(X), hp.batch_size) predict_label = [] for i in range(len(X) // hp.batch_size + 1): print('Step:\t', i, '/', len(X) // hp.batch_size) ### Get mini-batches x = X[i * hp.batch_size:(i + 1) * hp.batch_size] sentences = Texts[i * hp.batch_size:(i + 1) * hp.batch_size] labels = Labels[i * hp.batch_size:(i + 1) * hp.batch_size] preds = sess.run(g.preds, {g.x: x}) preds = [int(x) for x in preds] predict_label.extend(preds) ### Write to file for sent, label, pred in zip(sentences, labels, preds): # sentence-wise #got = " ".join(idx2word[idx] for idx in pred).split("</S>")[0].strip() fout.write("- sent: " + sent + "\n") fout.write('- label: {}, -predict: {} \n'.format( label, pred)) fout.flush() # bleu score if task_name == 'seq2seq': ref = target.split() hypothesis = got.split() if len(ref) > 3 and len(hypothesis) > 3: list_of_refs.append([ref]) hypotheses.append(hypothesis) ## Calculate bleu score if task_name == 'seq2seq': score = corpus_bleu(list_of_refs, hypotheses) fout.write("Bleu Score = " + str(100 * score)) elif task_name == 'classfication': assert len(Labels) == len( predict_label), 'The length of label and predicts\ are not alignmentted.' res = classification_report(Labels, predict_label) print(res) fout.write(res + '\n')
from flask import Flask from flask import request from flask import jsonify import codecs import os import tensorflow as tf from hyperparams import Hyperparams as hp from data_load import load_test_data, load_de_vocab, load_en_vocab from train import Graph from nltk.translate.bleu_score import corpus_bleu app = Flask(__name__) g = Graph(is_training=False) print('Graph loaded') X, Sources, Targets = load_test_data() de2idx, idx2de = load_de_vocab() en2idx, idx2en = load_en_vocab() with g.graph.as_default(): sess = tf.Session() saver = tf.train.Saver() saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # model name if not os.path.exists('results'): os.mkdir('results') with codecs.open("results/" + mname, "w", "utf-8") as fout: list_of_refs, hypotheses = [], [] for i in range(len(X) // hp.batch_size): ### Get mini-batches x = X[i * hp.batch_size:(i + 1) * hp.batch_size]
def main(): # Configure logger LOGGER = config_logger('config/logging.conf') BASE_DIR = os.path.abspath("") LOGGER.info(f"Current directory={BASE_DIR}") # Set device DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") LOGGER.info(f"Using device:{DEVICE}") torch.set_default_tensor_type(torch.DoubleTensor) # Set the random seed manually for reproducibility. LR, EPOCHS, POSTING_LIST_SIZE, BATCH_SIZE, STOP_THRESHOLD, SCRAP_THRESHOLD, SEED, LOG_INTERVAL = read_args( ) LOGGER.info( f"LR:{LR:5.3f} | EPOCHS:{EPOCHS:5d} | POSTING_LIST_SIZE:{POSTING_LIST_SIZE:5d} | BATCH_SIZE:{BATCH_SIZE:3d} |\ STOP_THRESHOLD:{STOP_THRESHOLD:5.3f} | SCRAP_THRESHOLD:{SCRAP_THRESHOLD:5.3f} | \ SEED:{SEED:5d} | LOG_INTERVAL:{LOG_INTERVAL:5d}") torch.manual_seed(SEED) cudnn.deterministic = True cudnn.benchmark = False BIAS = True NON_LINEARITY = "leaky_relu" # Open files for saving weights and deltas DELTA_DIR = os.path.join(BASE_DIR, "deltas") delta_freqs_file = os.path.join( DELTA_DIR, "{}_{}lr{}_b{}_e{}_deltas.freqs".format(NON_LINEARITY, "bias_" if BIAS else "", LR, BATCH_SIZE, EPOCHS)) delta_docs_file = os.path.join( DELTA_DIR, "{}_{}lr{}_b{}_e{}_deltas.docs".format(NON_LINEARITY, "bias_" if BIAS else "", LR, BATCH_SIZE, EPOCHS)) LOGGER.info("Saving deltas to: {} and {}".format(delta_freqs_file, delta_docs_file)) delta_freqs_binfile = open(delta_freqs_file, "wb") delta_docs_binfile = open(delta_docs_file, "wb") deltas_dummy = np.array([1, 1], dtype=np.uint32).ravel() deltas_dummy.tofile(delta_docs_binfile) # Load data TEST_FILE = "test_data/test_collection" posting_lists, pl_lengths = load_test_data(TEST_FILE, POSTING_LIST_SIZE, shuffling=False, sampling_size_ratio=0) pl_ds = DatasetDataList(posting_lists, pl_lengths) LOGGER.info("Loaded {} posting lists".format(len(pl_ds))) # Run training train_posting_lists(device=DEVICE, delta_freqs_file=delta_freqs_binfile, delta_docs_file=delta_docs_binfile, deltas_path=DELTA_DIR, logger=LOGGER, posting_lists_dataset=pl_ds, bias=BIAS, non_linearity=NON_LINEARITY, lr=LR, batch_size=BATCH_SIZE, stop_threshold=STOP_THRESHOLD, scrap_threshold=SCRAP_THRESHOLD, epochs=EPOCHS, log_interval=LOG_INTERVAL) # Close files delta_freqs_binfile.close() delta_docs_binfile.close()
def eval(): # Load graph g = Graph(is_training=False) print("Graph loaded") # Load data X, Sources, Targets = load_test_data() de2idx, idx2de = load_de_vocab() en2idx, idx2en = load_en_vocab() # X, Sources, Targets = X[:33], Sources[:33], Targets[:33] # Start session with g.graph.as_default(): sv = tf.train.Supervisor() with sv.managed_session(config=tf.ConfigProto( allow_soft_placement=True)) as sess: ## Restore parameters sv.saver.restore(sess, tf.train.latest_checkpoint(hp.logdir)) print("Restored!") ## Get model name mname = open(hp.logdir + '/checkpoint', 'r').read().split('"')[1] # model name ## Inference if not os.path.exists('results'): os.mkdir('results') with codecs.open("results/" + mname, "w", "utf-8") as fout: print("open reults success\n") list_of_refs, hypotheses = [], [] print("length of batch is" + str(len(X) // hp.batch_size)) for i in range(len(X) // hp.batch_size): print('translating') ### Get mini-batches x = X[i * hp.batch_size:(i + 1) * hp.batch_size] sources = Sources[i * hp.batch_size:(i + 1) * hp.batch_size] targets = Targets[i * hp.batch_size:(i + 1) * hp.batch_size] ### Autoregressive inference # 循环结束后,这个batch的句子的翻译保存在preds中 preds = np.zeros((hp.batch_size, hp.maxlen), np.int32) for j in range(hp.maxlen): _preds = sess.run(g.preds, {g.x: x, g.y: preds}) preds[:, j] = _preds[:, j] ### Write to file # 翻译完成后把句子的翻译保存到preds中 for source, target, pred in zip(sources, targets, preds): # sentence-wise #将pred(preds中的每个句子)的每个id转化为其对应的而英文单词,然后将这些单词字符串用一个空格字符链接起来,同时去掉句尾结束符。即得到了翻译的由词组成的句子。 got = " ".join( idx2en[idx] for idx in pred).split("</S>")[0].strip() # 分别将原句子、期望翻译的结果、实际翻译的结果写入文件 fout.write("- source: " + source + "\n") print('\n' + '\n' + '\n' + source + '\n' + '\n' + '\n') fout.write("- expected: " + target + "\n") fout.write("- got: " + got + "\n\n") fout.flush() # bleu score ref = target.split() hypothesis = got.split() if len(ref) > 3 and len(hypothesis) > 3: list_of_refs.append([ref]) hypotheses.append(hypothesis) ## Calculate bleu score # 最后计算bleu score并写入文件 # 将两者长度都大于3的句子加入到总的列表中,作为计算Bleu的参数,由此得到bleu socre.可以用以评估模型。 str_hyp = ",".join(hypotheses) print("len of hypothese is :" + len(hypotheses)) score = corpus_bleu(list_of_refs, hypotheses) fout.write("Bleu Score = " + str(100 * score)) #将bleu score写入文件末尾