def generator_train(): for data in train_data: x, y = data[0], data[1] x_emb = embedding_bert.get_batch_emb([x], doc_len, sen_len, tokenizer, estimator) doc_seq_len, sen_seq_len = embedding_bert.get_batch_seq([x], doc_len, sen_len, tokenizer, tol=2) yield x_emb[0], y, doc_seq_len, sen_seq_len
#acc_train /= n_batch loss_train /= n_batch rank_train /= n_batch #if (epoch + 1) % (epochs//10) == 0: #if (epoch + 0) % 10 == 0: #print('Epoch:', '%d' % (epoch + 0), 'cost =', '{:.6f}'.format(loss)) # testing n_batch = len(X_test_doc) // batch_size #for i in range(n_batch): for i in tqdm(range(n_batch), total=n_batch): X_batch, y_batch = utils.get_batch_test(X_test_doc, y_test, i, batch_size) X_batch_emb = embedding_bert.get_batch_emb(X_batch, doc_len, sen_len, tokenizer, estimator) #X_batch_emb = X_batch_emb[:, :, 0, :] #X_batch_seq, _ = embedding_bert.get_batch_seq(X_batch, doc_len, sen_len, tokenizer, tol = 2) doc_seq_len, sen_seq_len = embedding_bert.get_batch_seq(X_batch, doc_len, sen_len, tokenizer, tol=2) #batch_seq_len = np.array([list(x).index(0) + 1 for x in X_batch]) loss, y_pred_val = sess.run( [cost, y_pred], feed_dict={ X_emb: X_batch_emb, y: y_batch, doc_seq_length: doc_seq_len,
doc_len = 5 sen_len = 10 batch_size = 32 # In[63]: tokenizer, estimator = embedding_bert.prepare_bert(bert_vocab_file, bert_config_file, init_checkpoint, sen_len, select_layers, batch_size, graph_file, model_dir) # In[11]: xx = embedding_bert.get_batch_emb([data_part[0][0]], doc_len, sen_len, tokenizer, estimator) # In[12]: xx.shape # In[ ]: # In[9]: a, b = embedding_bert.get_batch_seq([data_part[0][0], data_part[1][0]], doc_len, sen_len, tokenizer, tol=2) a, b