def calc_test_loss(test_set = Dataset(test_x,test_y),display=True): accs = [] worksum = int(len(test_x) / batch_size) loss_list = [] predict_list = [] target_list = [] source_list = [] pb = ProgressBar(worksum=worksum,info="validating...",auto_display=display) pb.startjob() #test_set = Dataset(test_x,test_y) for j in range(worksum): batch_x,batch_y = test_set.next_batch(batch_size) lx = [seq_max_len] * batch_size ly = [seq_max_len] * batch_size bx = [np.sum(m > 0) for m in batch_x] by = [np.sum(m > 0) for m in batch_y] tmp_loss,tran = session.run([train_loss,translations],feed_dict={x:batch_x,y:batch_y, y_in: np.concatenate(( np.ones((batch_y.shape[0],1),dtype=np.int) * ch2ind['<go>'],batch_y[:,:-1]) ,axis=1) ,x_len:lx,y_len:ly, y_real_len:by, x_real_len:bx}) loss_list.append(tmp_loss) tmp_acc = cal_acc(tran,batch_y) accs.append(tmp_acc) predict_list += [i for i in tran] target_list += [i for i in batch_y] source_list += [i for i in batch_x] pb.complete(1) return np.average(loss_list),np.average(accs),get_bleu_score(predict_list,target_list),predict_list,target_list,source_list
def calc_test_loss(test_x, test_y, display=True): accs = [] worksum = int(len(test_x) / batch_size) loss_list = [] predict_list = [] target_list = [] source_list = [] pb = ProgressBar(worksum=worksum, info="validating...", auto_display=display) pb.startjob() #test_set = Dataset(test_x,test_y) for j in range(0, len(test_x), batch_size): batch_x, batch_y = test_x[j:j + batch_size], test_y[ j:j + batch_size] #test_set.next_batch(batch_size) if len(batch_x) < batch_size: continue bx = [len(m) + 1 for m in batch_x] by = [len(m) + 1 for m in batch_y] lx = [max(bx)] * batch_size ly = [max(by)] * batch_size batch_x = preprocessing.sequence.pad_sequences( batch_x, max(bx), padding='post', value=en2ind_oov['<eos>']) batch_y = preprocessing.sequence.pad_sequences( batch_y, max(by), padding='post', value=ch2ind_oov['<eos>']) tmp_loss, tran = session.run( [train_loss, translations], feed_dict={ x: batch_x, y: batch_y, y_in: np.concatenate((np.ones( (batch_y.shape[0], 1), dtype=np.int) * ch2ind['<go>'], batch_y[:, :-1]), axis=1), x_len: lx, y_len: ly, y_real_len: by, x_real_len: bx, y_max_len: max(by) }) loss_list.append(tmp_loss) tmp_acc = cal_acc(tran[:, :, 0], batch_y) accs.append(tmp_acc) predict_list += [i for i in tran[:, :, 0]] target_list += [i for i in batch_y] source_list += [i for i in batch_x] pb.complete(1) return np.average(loss_list), np.average(accs), get_bleu_score( predict_list, target_list), predict_list, target_list, source_list
def get_most_common(a1, a2): temp_dict1 = {} temp_dict2 = {} pb = ProgressBar(worksum=len(a1), auto_display=False) pb.startjob() num = 0 for s1, s2 in zip(a1, a2): num += 1 pb.complete(1) if args.max_words != -1 and (len(s1) > args.max_words or len(s2) > args.max_words): continue for w1 in s1: temp_dict1.setdefault(w1, 0) temp_dict1[w1] += 1 for w2 in s2: temp_dict2.setdefault(w2, 0) temp_dict2[w2] += 1 if num % 32 == 0: pb.display_progress_bar() sorted1 = sorted(temp_dict1.items(), key=lambda i: i[1], reverse=True) sorted2 = sorted(temp_dict2.items(), key=lambda i: i[1], reverse=True) #print(sorted1[:100]) #print(sorted2[:100]) return [i[0] for i in sorted1[:args.vac_dict_ch] ], [i[0] for i in sorted2[:args.vac_dict_en]]
loss = tf.reduce_mean(loss) optimizer = tf.train.AdamOptimizer( learning_rate=learning_rate).minimize(loss) # optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate).minimize(loss) # In[] # ============================================================================= # RUN INFERENCE # ============================================================================= sess.run(tf.global_variables_initializer()) from utils import ProgressBar losses = [] beginning_lr = 0.1 gen = data_generator(X_train, y_train, batch_size) for one_epoch in range(1, 10): pb = ProgressBar(worksum=MAX_SEQUENCE_LENGTH) pb.startjob() for one_batch in range(0, len(X_train), batch_size): batch_x, batch_y = gen.__next__() batch_x_len = np.asarray([len(x) for x in batch_x]) batch_lr = beginning_lr _, batch_loss = sess.run([optimizer, loss], feed_dict={ X: batch_x, y: batch_y, X_len: batch_x_len, learning_rate: batch_lr }) pb.info = "EPOCH {} batch{} lr {} loss {}".format(one_epoch, one_batch, batch_lr, batch_loss) pb.complete(batch_size)