def fit(self, sess, train, dev=None):
train_batches = data_utils.batch_iter(train, self.batch_size, self.num_epochs)
data_size = len(train)
num_batches_per_epoch = int((data_size - 1) / self.batch_size) + 1
best_dev_pacc = 0.0
best_dev_eacc = 0.0
best_dev_loss = 1e10
best_dev_epoch = 0
for batch in train_batches:
words_batch, textlen_batch, mentions_batch, mentionlen_batch, positions_batch, labels_batch = zip(*batch)
self.train_on_batch(sess, words_batch, textlen_batch, mentions_batch, mentionlen_batch, positions_batch, labels_batch)
current_step = tf.train.global_step(sess, self.global_step)
if (current_step % num_batches_per_epoch == 0) and (dev is not None):
print("\nEvaluation:")
print("previous best dev epoch {}, best exact acc {:g} with partial acc {:g}".format(best_dev_epoch, best_dev_eacc, best_dev_pacc))
loss, pacc, eacc = self.evaluation_on_dev(sess, dev)
print("")
if eacc > best_dev_eacc:
best_dev_loss = loss
best_dev_pacc = pacc
best_dev_eacc = eacc
best_dev_epoch = current_step // num_batches_per_epoch
if current_step // num_batches_per_epoch - best_dev_epoch > 3:
break
return best_dev_epoch, best_dev_loss, best_dev_pacc, best_dev_eacc
def predict(self, sess, test):
batches = data_utils.batch_iter(test,
self.batch_size,
1,
shuffle=False)
all_probs = np.zeros((0, self.num_classes - 1))
all_labels = []
total_cnt = 0
total_size = 0
for batch in batches:
words_batch, textlen_batch, positions_batch, heads_batch, tails_batch, labels_batch = zip(
*batch) # noqa
feed = self.create_feed_dict(words_batch, textlen_batch,
positions_batch, heads_batch,
tails_batch, labels_batch)
loss, probs, size, cnt = sess.run(
[self.loss, self.probs, self.valid_size, self.correct_num],
feed_dict=feed)
total_cnt += cnt
total_size += size
all_probs = np.concatenate((all_probs, probs[:, 1:]))
for l in labels_batch:
tmp = np.zeros(self.num_classes - 1)
if l > 0:
tmp[l - 1] = 1.0
all_labels.append(tmp)
all_probs = np.reshape(all_probs, (-1))
all_labels = np.reshape(np.array(all_labels), (-1))
return all_labels, all_probs, total_cnt / total_size
def fit(self, sess, train_triples, valid_triples=None):
train_batches = data_utils.batch_iter(train_triples, self.batch_size,
self.num_epochs)
data_size = len(train_triples)
num_batches_per_epoch = int((data_size - 1) / self.batch_size) + 1
best_valid_acc = 0.0
best_valid_loss = 1e10
best_valid_epoch = 0
for batch in train_batches:
self.train_on_batch(sess, batch)
current_step = tf.train.global_step(sess, self.global_step)
if (current_step % num_batches_per_epoch == 0) and (valid_triples
is not None):
print("\nValidation:")
print(
"previous best valid epoch %d, best valid acc %.3f with loss %.3f"
% (best_valid_epoch, best_valid_acc, best_valid_loss))
loss, acc = self.validation(sess, valid_triples)
print("")
if acc > best_valid_acc:
best_valid_loss = loss
best_valid_acc = acc
best_valid_epoch = current_step // num_batches_per_epoch
if current_step // num_batches_per_epoch - best_valid_epoch >= 3:
break
return best_valid_epoch, best_valid_loss, best_valid_acc
def fit(self, sess, train, valid=None):
train_batches = data_utils.batch_iter(train, self.batch_size,
self.num_epochs)
data_size = len(train)
num_batches_per_epoch = int((data_size - 1) / self.batch_size) + 1
best_valid_acc = 0.0
best_valid_loss = 1e10
best_valid_ap = 0.0
best_valid_epoch = 0
for batch in train_batches:
words_batch, textlen_batch, positions_batch, heads_batch, tails_batch, labels_batch = zip(
*batch) # noqa
self.train_on_batch(sess, words_batch, textlen_batch,
positions_batch, heads_batch, tails_batch,
labels_batch)
current_step = tf.train.global_step(sess, self.global_step)
if (current_step % num_batches_per_epoch == 0) and (valid
is not None):
print("\nEvaluation:")
print(
"previous best valid epoch %d, best valid ap %.3f with loss %.3f acc %.3f"
% (best_valid_epoch, best_valid_ap, best_valid_loss,
best_valid_acc))
loss, acc, ap = self.validation(sess, valid)
print("")
if ap > best_valid_ap:
best_valid_loss = loss
best_valid_acc = acc
best_valid_ap = ap
best_valid_epoch = current_step // num_batches_per_epoch
if current_step // num_batches_per_epoch - best_valid_epoch > 3:
break
return best_valid_epoch, best_valid_loss, best_valid_acc, best_valid_ap
def visdial_evaluate(dataloader, params, eval_batch_size):
sparse_metrics = SparseGTMetrics()
ndcg = NDCG()
dialog_encoder.eval()
batch_idx = 0
with torch.no_grad():
batch_size = 500 * (params['n_gpus'] / 8)
batch_size = min([1, 2, 4, 5, 100, 1000, 200, 8, 10, 40, 50, 500, 20, 25, 250, 125], \
key=lambda x: abs(x - batch_size) if x <= batch_size else float("inf"))
if params['overfit']:
batch_size = 100
for epoch_id, _, batch in batch_iter(dataloader, params):
if epoch_id == 1:
break
tokens = batch['tokens']
num_rounds = tokens.shape[1]
num_options = tokens.shape[2]
tokens = tokens.view(-1, tokens.shape[-1])
segments = batch['segments']
segments = segments.view(-1, segments.shape[-1])
sep_indices = batch['sep_indices']
sep_indices = sep_indices.view(-1, sep_indices.shape[-1])
mask = batch['mask']
mask = mask.view(-1, mask.shape[-1])
hist_len = batch['hist_len']
hist_len = hist_len.view(-1)
gt_option_inds = batch['gt_option_inds']
gt_relevance = batch['gt_relevance']
gt_relevance_round_id = batch['round_id'].squeeze(1)
assert tokens.shape[0] == segments.shape[0] == sep_indices.shape[0] == mask.shape[0] == \
hist_len.shape[0] == num_rounds * num_options * eval_batch_size
output = []
assert (eval_batch_size * num_rounds * num_options) // batch_size == (
eval_batch_size * num_rounds * num_options) / batch_size
for j in range((eval_batch_size * num_rounds * num_options) // batch_size):
# create chunks of the original batch
item = {}
item['tokens'] = tokens[j * batch_size:(j + 1) * batch_size, :]
item['segments'] = segments[j * batch_size:(j + 1) * batch_size, :]
item['sep_indices'] = sep_indices[j * batch_size:(j + 1) * batch_size, :]
item['mask'] = mask[j * batch_size:(j + 1) * batch_size, :]
item['hist_len'] = hist_len[j * batch_size:(j + 1) * batch_size]
_, _, _, nsp_scores = forward(dialog_encoder, item, params, output_nsp_scores=True, evaluation=True)
# normalize nsp scores
nsp_probs = F.softmax(nsp_scores, dim=1)
output.append(nsp_probs[:, 0])
output = torch.cat(output, 0).view(eval_batch_size, num_rounds, num_options)
sparse_metrics.observe(output, gt_option_inds)
output = output[torch.arange(output.size(0)), gt_relevance_round_id - 1, :]
ndcg.observe(output, gt_relevance)
batch_idx += 1
dialog_encoder.train()
all_metrics = {}
all_metrics.update(sparse_metrics.retrieve(reset=True))
all_metrics.update(ndcg.retrieve(reset=True))
return all_metrics
def get_rank_loss(self, test):
batches = data_utils.batch_iter(test,
self.batch_size,
1,
shuffle=False)
cnt = 0.0
total_loss = 0.0
for batch in batches:
words_batch, labels_batch = zip(*batch)
batch_size = len(words_batch)
words = self._variable(words_batch)
probs = self.model(words, batch_size)
if self.use_cuda:
probs = probs.cpu()
probs = probs.data.numpy()
cnt += batch_size
for i in range(batch_size):
s1 = self.get_score(probs[i])
s2 = self.get_score(labels_batch[i])
# s1 = np.argmax(probs[i])
# s2 = labels_batch[i]
total_loss += abs(s1 - s2)
return total_loss / cnt
def predict(self, sess, test):
batches = data_utils.batch_iter(test, self.batch_size, 1, shuffle=False)
all_predictions = []
for batch in batches:
words_batch, textlen_batch, mentions_batch, mentionlen_batch, positions_batch, labels_batch = zip(*batch)
feed = self.create_feed_dict(words_batch, textlen_batch, mentions_batch, mentionlen_batch, positions_batch)
batch_predictions = sess.run(self.predictions, feed_dict=feed)
all_predictions = np.concatenate([all_predictions, batch_predictions])
return all_predictions
def evaluate(self, sess, train, test):
train_batches = data_utils.batch_iter(train, self.batch_size, self.num_epochs)
data_size = len(train)
num_batches_per_epoch = int((data_size - 1) / self.batch_size) + 1
for batch in train_batches:
words_batch, textlen_batch, mentions_batch, mentionlen_batch, positions_batch, labels_batch = zip(*batch)
self.train_on_batch(sess, words_batch, textlen_batch, mentions_batch, mentionlen_batch, positions_batch, labels_batch)
current_step = tf.train.global_step(sess, self.global_step)
if current_step % num_batches_per_epoch == 0:
yield self.predict(sess, test)
def test_accuracy(test_x, test_y):
'''
:param test_x: testing dataset
:param test_y: testing label
:return:
eval_loss: loss
accuracy: accuracy
ave_precison_score: average precison
'''
true_onehot_labels = []
predicted_onehot_scores = []
predicted_onehot_labels_t2 = []
test_batches = batch_iter(test_x, test_y, BATCH_SIZE, 1)
eval_loss, eval_counter = 0., 0
for test_batch_x, test_batch_y in test_batches:
scores, cur_loss = sess.run(
[model.scores, model.loss],
feed_dict={
model.x: test_batch_x,
model.y: test_batch_y,
model.keep_prob: 1.0
})
for i in test_batch_y:
true_onehot_labels.append(i)
for j in scores:
predicted_onehot_scores.append(j)
batch_predicted_onehot_labels = get_onehot_label_topk(
scores=scores, top_num=NUM_LABEL)
for i in batch_predicted_onehot_labels:
predicted_onehot_labels_t2.append(i)
eval_loss = eval_loss + cur_loss
eval_counter = eval_counter + 1
#metrics
eval_loss = float(eval_loss / eval_counter)
ave_precision_score = average_precision_score(
y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores),
average='micro')
accuracy = accuracy_score(np.array(true_onehot_labels),
np.array(predicted_onehot_labels_t2))
return eval_loss, accuracy, ave_precision_score
def predict(self, sess, test_triples):
batches = data_utils.batch_iter(test_triples,
self.batch_size,
1,
shuffle=False)
preds = []
for batch in batches:
feed = self.create_feed_dict(**batch)
pred = sess.run(self.preds, feed_dict=feed)
preds = np.concatenate([preds, pred])
return preds
def evaluation_on_dev(self, sess, dev):
batches = data_utils.batch_iter(dev, self.batch_size, 1, shuffle=False)
total_loss = 0.0
total_pacc = 0.0
total_eacc = 0.0
total_len = 0
for batch in batches:
words_batch, textlen_batch, mentions_batch, mentionlen_batch, positions_batch, labels_batch = zip(*batch)
feed = self.create_feed_dict(words_batch, textlen_batch, mentions_batch, mentionlen_batch, positions_batch, labels_batch)
loss, pacc, eacc = sess.run([self.loss, self.partial_accuracy, self.exact_accuracy], feed_dict=feed)
total_loss += loss * len(labels_batch)
total_pacc += pacc * len(labels_batch)
total_eacc += eacc * len(labels_batch)
total_len += len(labels_batch)
time_str = datetime.datetime.now().isoformat()
print("{}: loss {:g} partial acc {:g} exact acc {:g}".format(time_str, total_loss/total_len, total_pacc / total_len, total_eacc / total_len))
return total_loss / total_len, total_pacc / total_len, total_eacc / total_len
def eval_test(sess,
model,
dataset,
video_features,
configs,
epoch=None,
global_step=None,
name="test"):
num_test_batches = math.ceil(len(dataset) / configs.batch_size)
ious = list()
for data in tqdm(batch_iter(dataset, video_features, configs.batch_size,
configs.extend, False),
total=num_test_batches,
desc="evaluate {}".format(name)):
raw_data, feed_dict = get_feed_dict(data, model, mode=name)
start_indexes, end_indexes = sess.run(
[model.start_index, model.end_index], feed_dict=feed_dict)
for record, start_index, end_index in zip(raw_data, start_indexes,
end_indexes):
start_time, end_time = convert_to_time(start_index, end_index,
record["feature_shape"],
record["duration"])
iou = calculate_iou(i0=[start_time, end_time],
i1=[record["start_time"], record["end_time"]])
ious.append(iou)
r1i3 = calculate_iou_accuracy(ious, threshold=0.3)
r1i5 = calculate_iou_accuracy(ious, threshold=0.5)
r1i7 = calculate_iou_accuracy(ious, threshold=0.7)
mi = np.mean(ious) * 100.0
value_pairs = [("{}/Rank@1, IoU=0.3".format(name), r1i3),
("{}/Rank@1, IoU=0.5".format(name), r1i5),
("{}/Rank@1, IoU=0.7".format(name), r1i7),
("{}/mean IoU".format(name), mi)]
# write the scores
score_str = "Epoch {}, Step {}:\n".format(epoch, global_step)
score_str += "Rank@1, IoU=0.3: {:.2f}\t".format(r1i3)
score_str += "Rank@1, IoU=0.5: {:.2f}\t".format(r1i5)
score_str += "Rank@1, IoU=0.7: {:.2f}\t".format(r1i7)
score_str += "mean IoU: {:.2f}\n".format(mi)
return r1i3, r1i5, r1i7, mi, value_pairs, score_str
def validation(self, sess, valid_triples):
batches = data_utils.batch_iter(valid_triples,
self.batch_size,
1,
shuffle=False)
total_loss = 0.0
total_acc = 0.0
total_len = 0
for batch in batches:
feed = self.create_feed_dict(**batch)
loss, acc = sess.run([self.loss, self.accuracy], feed_dict=feed)
total_loss += loss * len(batch["heads"])
total_acc += acc * len(batch["heads"])
total_len += len(batch["heads"])
time_str = datetime.datetime.now().isoformat()
print("{}: loss {:g} acc {:g}".format(time_str, total_loss / total_len,
total_acc / total_len))
return total_loss / total_len, total_acc / total_len
def fit(self, train, valid=None):
batches = data_utils.batch_iter(train, self.batch_size,
self.num_epochs)
data_size = len(train)
num_batches_per_epoch = int((data_size - 1) / self.batch_size) + 1
best_valid_loss = 1e5
best_valid_epoch = 0
step = 0
for batch in batches:
step += 1
words_batch, labels_batch = zip(*batch)
batch_size = len(words_batch)
self.model.zero_grad()
words = self._variable(words_batch)
labels = self._variable(labels_batch)
probs = self.model(words, batch_size)
# print(probs.data.numpy()[0])
loss = self.loss_fn(probs, labels)
time_str = datetime.now().isoformat()
print("{}: step {}, loss {:g}".format(time_str, step,
loss.data[0]))
loss.backward()
self.optimizer.step()
if (step % num_batches_per_epoch == 0) and (valid is not None):
print("\nValidation:")
print("previous best valid loss {:g} at epoch {}".format(
best_valid_loss, best_valid_epoch))
rloss = self.get_rank_loss(valid)
print("epoch: {}, loss {:g}".format(
step // num_batches_per_epoch, rloss))
print("")
if rloss < best_valid_loss:
best_valid_loss = rloss
best_valid_epoch = step // num_batches_per_epoch
# if step // num_batches_per_epoch - best_valid_epoch > 3:
# break
return best_valid_epoch, best_valid_loss
def save_preds(self, sess, test):
batches = data_utils.batch_iter(test,
self.batch_size,
1,
shuffle=False)
all_labels = []
all_preds = []
for batch in batches:
words_batch, textlen_batch, positions_batch, labels_batch = zip(
*batch)
feed = self.create_feed_dict(words_batch, textlen_batch,
positions_batch, labels_batch)
preds = sess.run(self.predictions, feed_dict=feed)
all_labels = np.concatenate((all_labels, labels_batch))
all_preds = np.concatenate((all_preds, preds))
outfile = open("preds.txt", "w")
for x, y in zip(all_preds, all_labels):
if y == 0:
continue
outfile.write("%d %d\n" % (x, y))
outfile.close()
def validation(self, sess, valid):
batches = data_utils.batch_iter(valid,
self.batch_size,
1,
shuffle=False)
total_loss = 0.0
total_len = 0
total_cnt = 0
total_size = 0
all_probs = np.zeros((0, self.num_classes - 1))
all_labels = []
for batch in batches:
words_batch, textlen_batch, positions_batch, heads_batch, tails_batch, labels_batch = zip(
*batch) # noqa
feed = self.create_feed_dict(words_batch, textlen_batch,
positions_batch, heads_batch,
tails_batch, labels_batch)
loss, size, cnt, probs = sess.run(
[self.loss, self.valid_size, self.correct_num, self.probs],
feed_dict=feed)
total_loss += loss * len(labels_batch)
total_len += len(labels_batch)
total_cnt += cnt
total_size += size
all_probs = np.concatenate((all_probs, probs[:, 1:]))
for l in labels_batch:
tmp = np.zeros(self.num_classes - 1)
if l > 0:
tmp[l - 1] = 1.0
all_labels.append(tmp)
all_probs = np.reshape(all_probs, (-1))
all_labels = np.reshape(np.array(all_labels), (-1))
average_precision = average_precision_score(all_labels, all_probs)
time_str = datetime.datetime.now().isoformat()
print("{}: loss {:g} acc {:g} ap {:g}".format(time_str,
total_loss / total_len,
total_cnt / total_size,
average_precision))
return total_loss / total_len, total_cnt / total_size, average_precision
def get_types(model_name, input_file, dev_file, output_file, options):
checkpoint_file = os.path.join(config.CHECKPOINT_DIR, model_name)
type2id, typeDict = pkl_utils._load(config.WIKI_TYPE)
id2type = {type2id[x]: x for x in type2id.keys()}
#different way? -> data is different!
# words, mentions, positions, labels = data_utils.load(input_file)
# n = len(words)
embedding = embedding_utils.Embedding.restore(checkpoint_file)
test_set, test_labels, test_tokenized = create_labelset_input(
*data_utils.load(input_file), embedding)
dev_set, dev_labels, dev_tokenized = create_labelset_input(
*data_utils.load(dev_file), embedding)
store = StructuredLogitsStore(
model_name,
idx2label=id2type,
hierarchical=True if "hier" in model_name else False,
nested=False)
graph = tf.Graph()
with graph.as_default():
sess = tf.Session()
saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
# DEFINE operations
input_words = graph.get_operation_by_name("input_words").outputs[0]
input_textlen = graph.get_operation_by_name("input_textlen").outputs[0]
input_mentions = graph.get_operation_by_name(
"input_mentions").outputs[0]
input_mentionlen = graph.get_operation_by_name(
"input_mentionlen").outputs[0]
input_positions = graph.get_operation_by_name(
"input_positions").outputs[0]
phase = graph.get_operation_by_name("phase").outputs[0]
dense_dropout = graph.get_operation_by_name("dense_dropout").outputs[0]
rnn_dropout = graph.get_operation_by_name("rnn_dropout").outputs[0]
pred_op = graph.get_operation_by_name("output/predictions").outputs[0]
#proba_op = graph.get_operation_by_name("output/proba").outputs[0] #proba
logit_op = graph.get_operation_by_name("output/scores").outputs[
0] #proba
tune_op = graph.get_operation_by_name("tune").outputs[0] # K x K
# results_op = graph.get_operation_by_name("results").outputs[0] # require labels
# DO THE SAME FOR DEV set!
test_batches = data_utils.batch_iter(test_set, 512, 1, shuffle=False)
all_predictions = []
all_logits = []
for batch in test_batches:
words_batch, textlen_batch, mentions_batch, mentionlen_batch, positions_batch, labels_batch = zip(
*batch)
feed = {
input_words: words_batch,
input_textlen: textlen_batch,
input_mentions: mentions_batch,
input_mentionlen: mentionlen_batch,
input_positions: positions_batch,
phase: False,
dense_dropout: 1.0,
rnn_dropout: 1.0
}
batch_predictions = sess.run(pred_op, feed_dict=feed)
all_predictions = np.concatenate(
[all_predictions, batch_predictions])
#probas = sess.run(logit_op, feed_dict=feed)
logit_predictions = sess.run(logit_op, feed_dict=feed)
if all_logits == []:
all_logits = logit_predictions
else:
all_logits = np.concatenate([all_logits, logit_predictions])
store.create_labelset(
StructuredLogits(f_x=all_logits,
y_true=test_labels,
tokenized=test_tokenized,
y_hat=None,
probas=None,
c=None,
document_masks=None,
idx2label=id2type), "test")
store.score_set("test")
dev_batches = data_utils.batch_iter(dev_set, 512, 1, shuffle=False)
all_predictions = []
all_logits = []
for batch in dev_batches:
words_batch, textlen_batch, mentions_batch, mentionlen_batch, positions_batch, labels_batch = zip(
*batch)
feed = {
input_words: words_batch,
input_textlen: textlen_batch,
input_mentions: mentions_batch,
input_mentionlen: mentionlen_batch,
input_positions: positions_batch,
phase: False,
dense_dropout: 1.0,
rnn_dropout: 1.0
}
batch_predictions = sess.run(pred_op, feed_dict=feed)
all_predictions = np.concatenate(
[all_predictions, batch_predictions])
#probas = sess.run(logit_op, feed_dict=feed)
logit_predictions = sess.run(logit_op, feed_dict=feed)
if all_logits == []:
all_logits = logit_predictions
else:
all_logits = np.concatenate([all_logits, logit_predictions])
store.create_labelset(
StructuredLogits(f_x=all_logits,
y_true=dev_labels,
tokenized=dev_tokenized,
y_hat=None,
probas=None,
c=None,
document_masks=None,
idx2label=id2type), "dev")
store.score_set("dev")
#np.transpose(prior_utils.create_prior(type_info, hparams.alpha)
# all_logits.append(logit_predictions)
# save as pickle
with open(os.path.join(os.path.dirname(checkpoint_file), "logits.pickle"),
"wb") as f:
pickle.dump(store, f)
"""
def visdial_evaluate(dataloader, params, eval_batch_size, dialog_encoder):
sparse_metrics = SparseGTMetrics()
ndcg = NDCG()
dialog_encoder.eval()
batch_idx = 0
with torch.no_grad():
# we can fit approximately 500 sequences of length 256 in 8 gpus with 12 GB of memory during inference.
batch_size = 500 * (params['n_gpus'] / 8)
batch_size = min([1, 2, 4, 5, 100, 1000, 200, 8, 10, 40, 50, 500, 20, 25, 250, 125], \
key=lambda x: abs(x-batch_size) if x <= batch_size else float("inf"))
print("batch size for evaluation", batch_size)
for epoch_id, _, batch in batch_iter(dataloader, params):
if epoch_id == 1:
break
tokens = batch['tokens']
num_rounds = tokens.shape[1]
num_options = tokens.shape[2]
tokens = tokens.view(-1, tokens.shape[-1])
segments = batch['segments']
segments = segments.view(-1, segments.shape[-1])
sep_indices = batch['sep_indices']
sep_indices = sep_indices.view(-1, sep_indices.shape[-1])
mask = batch['mask']
mask = mask.view(-1, mask.shape[-1])
hist_len = batch['hist_len']
hist_len = hist_len.view(-1)
gt_option_inds = batch['gt_option_inds']
gt_relevance = batch['gt_relevance']
gt_relevance_round_id = batch['round_id'].squeeze(1)
# get image features
features = batch['image_feat']
spatials = batch['image_loc']
image_mask = batch['image_mask']
max_num_regions = features.shape[-2]
features = features.unsqueeze(1).unsqueeze(1).expand(
eval_batch_size, num_rounds, num_options, max_num_regions,
2048).contiguous()
spatials = spatials.unsqueeze(1).unsqueeze(1).expand(
eval_batch_size, num_rounds, num_options, max_num_regions,
5).contiguous()
image_mask = image_mask.unsqueeze(1).unsqueeze(1).expand(
eval_batch_size, num_rounds, num_options,
max_num_regions).contiguous()
features = features.view(-1, max_num_regions, 2048)
spatials = spatials.view(-1, max_num_regions, 5)
image_mask = image_mask.view(-1, max_num_regions)
assert tokens.shape[0] == segments.shape[0] == sep_indices.shape[0] == mask.shape[0] == \
hist_len.shape[0] == features.shape[0] == spatials.shape[0] == \
image_mask.shape[0] == num_rounds * num_options * eval_batch_size
output = []
assert (eval_batch_size * num_rounds *
num_options) // batch_size == (eval_batch_size * num_rounds
* num_options) / batch_size
for j in range(
(eval_batch_size * num_rounds * num_options) // batch_size):
# create chunks of the original batch
item = {}
item['tokens'] = tokens[j * batch_size:(j + 1) * batch_size, :]
item['segments'] = segments[j * batch_size:(j + 1) *
batch_size, :]
item['sep_indices'] = sep_indices[j * batch_size:(j + 1) *
batch_size, :]
item['mask'] = mask[j * batch_size:(j + 1) * batch_size, :]
item['hist_len'] = hist_len[j * batch_size:(j + 1) *
batch_size]
item['image_feat'] = features[j * batch_size:(j + 1) *
batch_size, :, :]
item['image_loc'] = spatials[j * batch_size:(j + 1) *
batch_size, :, :]
item['image_mask'] = image_mask[j * batch_size:(j + 1) *
batch_size, :]
_, _, _, _, nsp_scores = forward(dialog_encoder,
item,
params,
output_nsp_scores=True,
evaluation=True)
# normalize nsp scores
nsp_probs = F.softmax(nsp_scores, dim=1)
assert nsp_probs.shape[-1] == 2
output.append(nsp_probs[:, 0])
output = torch.cat(output, 0).view(eval_batch_size, num_rounds,
num_options)
sparse_metrics.observe(output, gt_option_inds)
output = output[torch.arange(output.size(0)),
gt_relevance_round_id - 1, :]
ndcg.observe(output, gt_relevance)
batch_idx += 1
dialog_encoder.train()
print("tot eval batches", batch_idx)
all_metrics = {}
all_metrics.update(sparse_metrics.retrieve(reset=True))
all_metrics.update(ndcg.retrieve(reset=True))
return all_metrics
del pretrained_dict, pretrained_dict_model, pretrained_dict_optimizer, pretrained_dict_scheduler, \
model_dict, optimizer_dict
torch.cuda.empty_cache()
num_iter_epoch = dataset.numDataPoints['train'] // (params['batch_size'] // params['sequences_per_image'] if (params['batch_size'] // params['sequences_per_image']) \
else 1 if not params['overfit'] else 5 )
print('\n%d iter per epoch.' % num_iter_epoch)
dialog_encoder = nn.DataParallel(dialog_encoder)
dialog_encoder.to(device)
start_t = timer()
optimizer.zero_grad()
for epoch_id, idx, batch in batch_iter(dataloader, params):
iter_id = start_iter_id + idx + (epoch_id * num_iter_epoch)
dialog_encoder.train()
# expand image features,
orig_features = batch['image_feat']
orig_spatials = batch['image_loc']
orig_image_mask = batch['image_mask']
orig_image_target = batch['image_target']
orig_image_label = batch['image_label']
num_rounds = batch["tokens"].shape[1]
num_samples = batch["tokens"].shape[2]
features = orig_features.unsqueeze(1).unsqueeze(1).expand(
orig_features.shape[0], num_rounds, num_samples,
def get_types(model_name, input_file, output_file):
checkpoint_file = os.path.join(config.CHECKPOINT_DIR, model_name)
type2id, typeDict = pkl_utils._load(config.WIKI_TYPE)
id2type = {type2id[x]:x for x in type2id.keys()}
df = pd.read_csv(input_file, sep="\t", names=["r", "e1", "x1", "y1", "e2", "x2", "y2", "s"])
n = df.shape[0]
words1 = np.array(df.s)
mentions1 = np.array(df.e1)
positions1 = np.array([[x, y] for x, y in zip(df.x1, df.y1+1)])
words2 = np.array(df.s)
mentions2 = np.array(df.e2)
positions2 = np.array([[x, y] for x, y in zip(df.x2, df.y2+1)])
words = np.concatenate([words1, words2])
mentions = np.concatenate([mentions1, mentions2])
positions = np.concatenate([positions1, positions2])
embedding = embedding_utils.Embedding.restore(checkpoint_file)
textlen = np.array([embedding.len_transform1(x) for x in words])
words = np.array([embedding.text_transform1(x) for x in words])
mentionlen = np.array([embedding.len_transform2(x) for x in mentions])
mentions = np.array([embedding.text_transform2(x) for x in mentions])
positions = np.array([embedding.position_transform(x) for x in positions])
labels = np.zeros(2*n)
test_set = list(zip(words, textlen, mentions, mentionlen, positions, labels))
graph = tf.Graph()
with graph.as_default():
sess = tf.Session()
saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
input_words = graph.get_operation_by_name("input_words").outputs[0]
input_textlen = graph.get_operation_by_name("input_textlen").outputs[0]
input_mentions = graph.get_operation_by_name("input_mentions").outputs[0]
input_mentionlen = graph.get_operation_by_name("input_mentionlen").outputs[0]
input_positions = graph.get_operation_by_name("input_positions").outputs[0]
phase = graph.get_operation_by_name("phase").outputs[0]
dense_dropout = graph.get_operation_by_name("dense_dropout").outputs[0]
rnn_dropout = graph.get_operation_by_name("rnn_dropout").outputs[0]
pred_op = graph.get_operation_by_name("output/predictions").outputs[0]
batches = data_utils.batch_iter(test_set, 512, 1, shuffle=False)
all_predictions = []
for batch in batches:
words_batch, textlen_batch, mentions_batch, mentionlen_batch, positions_batch, labels_batch = zip(*batch)
feed = {
input_words: words_batch,
input_textlen: textlen_batch,
input_mentions: mentions_batch,
input_mentionlen: mentionlen_batch,
input_positions: positions_batch,
phase: False,
dense_dropout: 1.0,
rnn_dropout: 1.0
}
batch_predictions = sess.run(pred_op, feed_dict=feed)
all_predictions = np.concatenate([all_predictions, batch_predictions])
df["t1"] = all_predictions[:n]
df["t2"] = all_predictions[n:]
df["t1"] = df["t1"].map(id2type)
df["t2"] = df["t2"].map(id2type)
df.to_csv(output_file, sep="\t", header=False, index=False)
log_device_placement=False)
sess = tf.Session(config=session_conf)
with sess.as_default():
# Load the saved meta graph and restore variables
saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
# Get the placeholders from the graph by name
input_x = graph.get_operation_by_name("input_x").outputs[0]
attention_map = graph.get_operation_by_name("attention_map").outputs[0]
# Tensors we want to evaluate
scores = graph.get_operation_by_name("output/scores").outputs[0]
# Generate batches for one epoch
batches = utils.batch_iter(list(zip(x_test, x_test_map)), batch_size,
1)
# Collect the prediction scores here
pred_scores = []
for batch in batches:
x_test_batch, x_test_batch_map = zip(*batch)
batch_scores = sess.run(scores, {
input_x: x_test_batch,
attention_map: x_test_batch_map
})
pred_scores = np.concatenate([pred_scores, batch_scores])
predictions = np.array([1 if score > 0.05 else 0 for score in pred_scores])
mae = mean_absolute_error(y_test_prob, pred_scores)
def eval_ai_generate(dataloader, params, eval_batch_size, split='test'):
sparse_metrics = SparseGTMetrics()
ranks_json = []
dialog_encoder.eval()
batch_idx = 0
with torch.no_grad():
batch_size = 500 * (params['n_gpus'] / 8)
batch_size = min([1, 2, 4, 5, 100, 1000, 200, 8, 10, 40, 50, 500, 20, 25, 250, 125], \
key=lambda x: abs(x-batch_size) if x <= batch_size else float("inf"))
print("batch size for evaluation", batch_size)
for epochId, _, batch in tqdm(batch_iter(dataloader, params)):
if epochId == 1:
break
tokens = batch['tokens']
num_rounds = tokens.shape[1]
num_options = tokens.shape[2]
tokens = tokens.view(-1, tokens.shape[-1])
segments = batch['segments']
segments = segments.view(-1, segments.shape[-1])
sep_indices = batch['sep_indices']
sep_indices = sep_indices.view(-1, sep_indices.shape[-1])
mask = batch['mask']
mask = mask.view(-1, mask.shape[-1])
hist_len = batch['hist_len']
hist_len = hist_len.view(-1)
gt_option_inds = batch['gt_option_inds']
output = []
assert (eval_batch_size * num_rounds *
num_options) // batch_size == (eval_batch_size * num_rounds
* num_options) / batch_size
for j in range(
(eval_batch_size * num_rounds * num_options) // batch_size):
# create chunks of the original batch
item = {}
item['tokens'] = tokens[j * batch_size:(j + 1) * batch_size, :]
item['segments'] = segments[j * batch_size:(j + 1) *
batch_size, :]
item['sep_indices'] = sep_indices[j * batch_size:(j + 1) *
batch_size, :]
item['mask'] = mask[j * batch_size:(j + 1) * batch_size, :]
item['hist_len'] = hist_len[j * batch_size:(j + 1) *
batch_size]
_, _, _, nsp_scores = forward(dialog_encoder,
item,
params,
output_nsp_scores=True,
evaluation=True)
# normalize nsp scores
nsp_probs = F.softmax(nsp_scores, dim=1)
assert nsp_probs.shape[-1] == 2
output.append(nsp_probs[:, 0])
output = torch.cat(output, 0).view(eval_batch_size, num_rounds,
num_options)
sparse_metrics.observe(output, gt_option_inds)
batch_idx += 1
all_metrics = {}
all_metrics.update(sparse_metrics.retrieve(reset=True))
'''
for i in range(eval_batch_size):
ranks_json.append(
{
#"image_id": batch["image_id"][i].item(),
#"round_id": int(batch["round_id"][i].item()),
"ranks": [rank.item() for rank in ranks[i][:]],
}
)
'''
#return ranks_json
return all_metrics
def eval_ai_generate(dataloader, params, eval_batch_size, split='test'):
ranks_json = []
dialog_encoder.eval()
batch_idx = 0
with torch.no_grad():
batch_size = 500 * (params['n_gpus'] / 8)
batch_size = min([1, 2, 4, 5, 100, 1000, 200, 8, 10, 40, 50, 500, 20, 25, 250, 125], \
key=lambda x: abs(x-batch_size) if x <= batch_size else float("inf"))
print("batch size for evaluation", batch_size)
for epochId, _, batch in batch_iter(dataloader, params):
if epochId == 1:
break
tokens = batch['tokens']
num_rounds = tokens.shape[1]
num_options = tokens.shape[2]
tokens = tokens.view(-1, tokens.shape[-1])
segments = batch['segments']
segments = segments.view(-1, segments.shape[-1])
sep_indices = batch['sep_indices']
sep_indices = sep_indices.view(-1, sep_indices.shape[-1])
mask = batch['mask']
mask = mask.view(-1, mask.shape[-1])
hist_len = batch['hist_len']
hist_len = hist_len.view(-1)
# get image features
features = batch['image_feat']
spatials = batch['image_loc']
image_mask = batch['image_mask']
# expand the image features to match those of tokens etc.
max_num_regions = features.shape[-2]
features = features.unsqueeze(1).unsqueeze(1).expand(
eval_batch_size, num_rounds, num_options, max_num_regions,
2048).contiguous()
spatials = spatials.unsqueeze(1).unsqueeze(1).expand(
eval_batch_size, num_rounds, num_options, max_num_regions,
5).contiguous()
image_mask = image_mask.unsqueeze(1).unsqueeze(1).expand(
eval_batch_size, num_rounds, num_options,
max_num_regions).contiguous()
features = features.view(-1, max_num_regions, 2048)
spatials = spatials.view(-1, max_num_regions, 5)
image_mask = image_mask.view(-1, max_num_regions)
assert tokens.shape[0] == segments.shape[0] == sep_indices.shape[0] == mask.shape[0] == \
hist_len.shape[0] == features.shape[0] == spatials.shape[0] == \
image_mask.shape[0] == num_rounds * num_options * eval_batch_size
output = []
assert (eval_batch_size * num_rounds *
num_options) // batch_size == (eval_batch_size * num_rounds
* num_options) / batch_size
for j in range(
(eval_batch_size * num_rounds * num_options) // batch_size):
# create chunks of the original batch
item = {}
item['tokens'] = tokens[j * batch_size:(j + 1) * batch_size, :]
item['segments'] = segments[j * batch_size:(j + 1) *
batch_size, :]
item['sep_indices'] = sep_indices[j * batch_size:(j + 1) *
batch_size, :]
item['mask'] = mask[j * batch_size:(j + 1) * batch_size, :]
item['hist_len'] = hist_len[j * batch_size:(j + 1) *
batch_size]
item['image_feat'] = features[j * batch_size:(j + 1) *
batch_size, :, :]
item['image_loc'] = spatials[j * batch_size:(j + 1) *
batch_size, :, :]
item['image_mask'] = image_mask[j * batch_size:(j + 1) *
batch_size, :]
_, _, _, _, nsp_scores = forward(dialog_encoder,
item,
params,
output_nsp_scores=True,
evaluation=True)
# normalize nsp scores
nsp_probs = F.softmax(nsp_scores, dim=1)
assert nsp_probs.shape[-1] == 2
output.append(nsp_probs[:, 0])
# print("output shape",torch.cat(output,0).shape)
output = torch.cat(output, 0).view(eval_batch_size, num_rounds,
num_options)
ranks = scores_to_ranks(output)
ranks = ranks.squeeze(1)
for i in range(eval_batch_size):
ranks_json.append({
"image_id":
batch["image_id"][i].item(),
"round_id":
int(batch["round_id"][i].item()),
"ranks": [rank.item() for rank in ranks[i][:]],
})
batch_idx += 1
return ranks_json
def train_cnn():
# Data Preparation
# ==================================================
if FLAGS.init_embedding_path is not None:
embedding = np.load(FLAGS.init_embedding_path)
print("Using pre-trained word embedding which shape is {}\n".format(
embedding.shape))
FLAGS.vocab_size = embedding.shape[0]
FLAGS.embedding_size = embedding.shape[1]
if FLAGS.init_model_path is not None:
assert os.path.isdir(
FLAGS.init_model_path), "init_model_path must be a directory\n"
ckpt = tf.train.get_checkpoint_state(FLAGS.init_model_path)
assert ckpt, "No checkpoint found in {}\n".format(
FLAGS.init_model_path)
assert ckpt.model_checkpoint_path, "No model_checkpoint_path found in checkpoint\n"
# Create root directory
timestamp = str(int(time.time()))
root_dir = os.path.join(os.path.curdir, 'runs', 'textcnn',
'trained_result_' + timestamp)
os.makedirs(root_dir)
# Load data
print("Loading data...\n")
x_data = np.loadtxt(FLAGS.x_data_file)
x_data = x_data.reshape(20480, 20, 30)
x_data = x_data.reshape(20480, 600)
y_data = np.loadtxt(FLAGS.y_data_file)
print("data load finished")
# Split dataset
# x_train, x_test, y_train, y_test = train_test_split(x_data, y_data, test_size=FLAGS.test_size, stratify=y_data, random_state=0)
# x_val, x_test, y_val, y_test = train_test_split(x_test, y_test, test_size=0.5, random_state=0)
# Training
# ==================================================
with tf.Graph().as_default():
tf_config = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
tf_config.gpu_options.allow_growth = FLAGS.gpu_allow_growth
with tf.Session(config=tf_config).as_default() as sess:
cnn = TextCNN(vocab_size=FLAGS.vocab_size,
embedding_size=FLAGS.embedding_size,
sequence_length=FLAGS.sequence_length,
filter_sizes=list(
map(int, FLAGS.filter_sizes.split(","))),
num_filters=FLAGS.num_filters,
num_classes=FLAGS.num_classes,
learning_rate=FLAGS.learning_rate,
grad_clip=FLAGS.grad_clip,
l2_reg_lambda=FLAGS.l2_reg_lambda)
# Output directory for models and summaries
out_dir = os.path.abspath(root_dir)
print("Writing to {}...\n".format(out_dir))
# Summaries for loss and accuracy
tf.summary.scalar("loss", cnn.loss)
tf.summary.scalar("accuracy", cnn.accuracy)
merged_summary = tf.summary.merge_all()
# Summaries dictionary
train_summary_dir = os.path.join(out_dir, 'summaries', 'train')
val_summary_dir = os.path.join(out_dir, 'summaries', 'val')
train_summary_writer = tf.summary.FileWriter(
train_summary_dir, sess.graph)
val_summary_writer = tf.summary.FileWriter(val_summary_dir,
sess.graph)
# Checkpoint directory, will not create itself
checkpoint_dir = os.path.abspath(
os.path.join(out_dir, 'checkpoints'))
checkpoint_prefix = os.path.join(checkpoint_dir, 'model.ckpt')
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=1)
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Using pre-trained word embedding
# if FLAGS.init_embedding_path is not None:
# sess.run(cnn.embedding.assign(embedding))
# del embedding
# Continue training from saved model
if FLAGS.init_model_path is not None:
saver.restore(sess, ckpt.model_checkpoint_path)
# Training start
print("Start training...\n")
best_at_step = 0
best_val_accuracy = 0
#****************************************
# Generate train batches
train_batches = data_utils.batch_iter(list(zip(x_data, y_data)),
FLAGS.batch_size)
start = time.time()
cnn_feature_temp = []
for batch in train_batches:
# Training model on x_batch and y_batch
x_batch, y_batch = zip(*batch)
feed_dict = {
cnn.input_x: x_batch,
cnn.input_y: y_batch,
cnn.keep_prob: FLAGS.dropout_keep_prob,
cnn.is_training: True
}
pooled_concat_flat, _, global_step, train_summaries, train_loss, train_accuracy = sess.run(
[
cnn.pooled_concat_flat, cnn.train_op, cnn.global_step,
merged_summary, cnn.loss, cnn.accuracy
],
feed_dict=feed_dict)
cnn_feature_temp.append(pooled_concat_flat.tolist())
np.savetxt(
"../data/char_data/char_dim/char_cnn_embeddings_20_30_dim256.txt",
np.array(cnn_feature_temp).reshape(20480, 192))
# cnn_feature.append(cnn_feature_temp)
# with open('./embeddings.txt','w', encoding='utf-8')as f:
# for line in cnn_feature_temp:
# for content in line :
# f.write(str(content).lstrip('[').rstrip(']') + '\n')
print('finished training')
def train(args: dict):
model_name = args['--model']
batch_size = int(args['--batch-size'])
max_patience = int(args['--max-patience'])
max_num_trial = int(args['--max-num-trial'])
lr_decay = float(args['--lr-decay'])
train_ratio = float(args['--train-ratio'])
model_save_to = args['--model-save-to']
max_epoch = int(args['--max-epoch'])
model_path = args['model-path']
train_data_path = args['train-data-path']
pid2pidx_path = args['pid2pidx-path']
cid2cidx_path = args['cid2cidx-path']
pidx2cidx_path = args['pidx2cidx-path']
cidx2cname_path = args['cidx2cname-path']
raw_data, problem_map = read_data_from_file(train_data_path, pid2pidx_path,
cid2cidx_path, pidx2cidx_path,
cidx2cname_path)
num_problems = len(problem_map)
num_concepts = problem_map.num_concepts
train_size = int(train_ratio * len(raw_data))
val_size = len(raw_data) - train_size
train_dataset, val_dataset = random_split(raw_data, [train_size, val_size])
unpacked_train_dataset = unpack_data(train_dataset)
unpacked_val_dataset = unpack_data(val_dataset)
if model_name == 'DKT':
model = DKT(num_problems=num_problems,
hidden_size=int(args['--hidden-size']),
dropout_rate=float(args['--dropout']))
criterion = torch.nn.BCELoss(reduction='mean')
optimizer = optim.Adam(model.parameters(), lr=float(args['--lr']))
elif model_name == "DKTEnhanced":
model = DKTEnhanced(problem_map=problem_map,
hidden_size=int(args['--hidden-size']),
dropout_rate=float(args['--dropout']))
criterion = PredictionConsistentBCELoss(
lambda_r=float(args['--lambda-r']))
optimizer = optim.Adam(model.parameters(), lr=float(args['--lr']))
else:
raise ValueError("wrong value of model_name[{}]".format(model_name))
if args['--gpu'] is not None:
device = torch.device(
"cuda:" + args['--gpu'] if torch.cuda.is_available() else "cpu")
else:
device = torch.device("cpu")
logging.info("Using {}".format(device))
model.to(device)
# for p in model.parameters():
# p.data.uniform_(-0.1, 0.1)
train_iter = 0
patience = 0
num_trial = 0
hist_valid_scores = []
# print statistic infomation
with torch.no_grad():
model.eval()
t_auc_n, t_auc_c, t_acc, t_rmse, t_mae = evaluate(
unpacked_train_dataset, num_problems, num_concepts, model)
v_auc_n, v_auc_c, v_acc, v_rmse, v_mae = evaluate(
unpacked_val_dataset, num_problems, num_concepts, model)
print("Init:")
print("valid_auc(n) %.6f\ttrain_auc(n) %.6f\t" % (v_auc_n, t_auc_n))
print("valid_auc(c) %.6f\ttrain_auc(c) %.6f\t" % (v_auc_c, t_auc_c))
print("valid_acc %.6f\ttrain_acc %.6f\t" % (v_acc, t_acc))
print("valid_rmse %.6f\ttrain_rmse %.6f\t" % (v_rmse, t_rmse))
print("valid_mae %.6f\ttrain_mae %.6f\t" % (v_mae, t_mae))
batch_num = math.ceil(len(train_dataset) / batch_size)
for epoch in range(max_epoch): # loop over the dataset multiple times
for batch_data in tqdm.tqdm(batch_iter(train_dataset,
batch_size=batch_size),
desc="Epoch[{}]".format(epoch + 1),
total=batch_num):
train_iter += 1
model.train()
problem_seqs, concept_seqs, answer_seqs = batch_data
# get y_next_true ans y_cur_true
next_answer_seqs = copy.deepcopy(answer_seqs)
for n_ans_seq in next_answer_seqs:
del n_ans_seq[0]
y_next_true = list(itertools.chain.from_iterable(next_answer_seqs))
y_next_true = torch.tensor(y_next_true,
dtype=torch.float,
device=model.device)
y_cur_true = list(itertools.chain.from_iterable(answer_seqs))
y_cur_true = torch.tensor(y_cur_true,
dtype=torch.float,
device=model.device)
# process the input data
response_data, concept_data, seq_lengths = process_data(
batch_data, num_problems, num_concepts, device=model.device)
# forward + backward + optimize
y_next_pred, y_cur_pred, batch_future_pred, batch_concept_mastery = model(
response_data, concept_data, seq_lengths)
if model_name == 'DKT':
loss = criterion(y_next_pred, y_next_true)
elif model_name == 'DKTEnhanced':
loss = criterion(y_next_pred, y_next_true, y_cur_pred,
y_cur_true)
else:
raise ValueError
loss.backward()
# clip gradient
nn.utils.clip_grad_norm_(model.parameters(),
max_norm=float(args['--clip-grad']))
optimizer.step()
# zero the parameter gradients
optimizer.zero_grad()
# print statistic information
with torch.no_grad():
model.eval()
t_auc_n, t_auc_c, t_acc, t_rmse, t_mae = evaluate(
unpacked_train_dataset, num_problems, num_concepts, model)
v_auc_n, v_auc_c, v_acc, v_rmse, v_mae = evaluate(
unpacked_val_dataset, num_problems, num_concepts, model)
print("valid_auc(n) %.6f\ttrain_auc(n) %.6f\t" %
(v_auc_n, t_auc_n))
print("valid_auc(c) %.6f\ttrain_auc(c) %.6f\t" %
(v_auc_c, t_auc_c))
print("valid_acc %.6f\ttrain_acc %.6f\t" % (v_acc, t_acc))
print("valid_rmse %.6f\ttrain_rmse %.6f\t" % (v_rmse, t_rmse))
print("valid_mae %.6f\ttrain_mae %.6f\t" % (v_mae, t_mae))
# check for early stop
is_better = len(
hist_valid_scores) == 0 or v_auc_n > max(hist_valid_scores)
hist_valid_scores.append(v_auc_n)
if is_better:
patience = 0
if not os.path.exists(model_save_to):
os.makedirs(model_save_to)
model.save(model_path)
# also save the optimizers' state
torch.save(optimizer.state_dict(), model_path + '.optim')
elif patience < int(max_patience):
patience += 1
print('hit patience %d' % patience)
if patience == int(max_patience):
num_trial += 1
print('hit #%d trial' % num_trial)
if num_trial == int(max_num_trial):
print('early stop!')
break
# decay lr, and restore from previously best checkpoint
lr = optimizer.param_groups[0]['lr'] * float(lr_decay)
# load model
params = torch.load(model_path,
map_location=lambda storage, loc: storage)
model.load_state_dict(params['state_dict'])
model = model.to(device)
optimizer.load_state_dict(torch.load(model_path + '.optim'))
# set new lr
for param_group in optimizer.param_groups:
param_group['lr'] = lr
# reset patience
patience = 0
print('Finished Training')
def train_rnn():
# Data Preparation
# ==================================================
if FLAGS.init_embedding_path is not None:
embedding = np.load(FLAGS.init_embedding_path)
print("Using pre-trained word embedding which shape is {}\n".format(embedding.shape))
FLAGS.vocab_size = embedding.shape[0]
FLAGS.embedding_size = embedding.shape[1]
if FLAGS.init_model_path is not None:
assert os.path.isdir(FLAGS.init_model_path), "init_model_path must be a directory\n"
ckpt = tf.train.get_checkpoint_state(FLAGS.init_model_path)
assert ckpt, "No checkpoint found in {}\n".format(FLAGS.init_model_path)
assert ckpt.model_checkpoint_path, "No model_checkpoint_path found in checkpoint\n"
# Create root directory
timestamp = str(int(time.time()))
root_dir = os.path.join(os.path.curdir, 'runs', 'textrnn', 'trained_result_' + timestamp)
os.makedirs(root_dir)
# Load data
# print("Loading data...\n")
# x, y = data_helpers.load_data(FLAGS.data_file, FLAGS.sequence_length, FLAGS.vocab_size, root_dir=root_dir)
# FLAGS.num_classes = len(y[0])
print("Loading data...\n")
x_data = np.loadtxt(FLAGS.x_data_file)
y_data = np.loadtxt(FLAGS.y_data_file)
print("data load finished")
# Split dataset
# x_train, x_test, y_train, y_test = train_test_split(x_data, y_data, test_size=FLAGS.test_size, stratify=y_data, random_state=0)
# x_val, x_test, y_val, y_test = train_test_split(x_test, y_test, test_size=0.5, random_state=0)
# Training
# ==================================================
with tf.Graph().as_default():
tf_config = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
tf_config.gpu_options.allow_growth = FLAGS.gpu_allow_growth
with tf.Session(config=tf_config).as_default() as sess:
rnn = TextRNN(
vocab_size=FLAGS.vocab_size,
embedding_size=FLAGS.embedding_size,
sequence_length=FLAGS.sequence_length,
rnn_size=FLAGS.rnn_size,
num_layers=FLAGS.num_layers,
attention_size=FLAGS.attention_size,
num_classes=FLAGS.num_classes,
learning_rate=FLAGS.learning_rate,
grad_clip=FLAGS.grad_clip)
# Output directory for models and summaries
out_dir = os.path.abspath(root_dir)
print("Writing to {}...\n".format(out_dir))
# Summaries for loss and accuracy
tf.summary.scalar("loss", rnn.loss)
tf.summary.scalar("accuracy", rnn.accuracy)
merged_summary = tf.summary.merge_all()
# Summaries dictionary
train_summary_dir = os.path.join(out_dir, 'summaries', 'train')
val_summary_dir = os.path.join(out_dir, 'summaries', 'val')
train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph)
val_summary_writer = tf.summary.FileWriter(val_summary_dir, sess.graph)
# Checkpoint directory, will not create itself
checkpoint_dir = os.path.abspath(os.path.join(out_dir, 'checkpoints'))
checkpoint_prefix = os.path.join(checkpoint_dir, 'model.ckpt')
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=1)
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Using pre-trained word embedding
if FLAGS.init_embedding_path is not None:
sess.run(rnn.embedding.assign(embedding))
del embedding
# Continue training from saved model
if FLAGS.init_model_path is not None:
saver.restore(sess, ckpt.model_checkpoint_path)
# Training start
print("Start training...\n")
best_at_step = 0
best_val_accuracy = 0
train_batches = data_utils.batch_iter(list(zip(x_data, y_data)), FLAGS.batch_size)
start = time.time()
rnn_feature_temp = []
for batch in train_batches:
# Training model on x_batch and y_batch
x_batch, y_batch = zip(*batch)
# seq_len_train = data_helpers.real_len(x_batch)
seq_len_train = data_utils.real_len(x_batch)
feed_dict = {rnn.input_x: x_batch, rnn.input_y: y_batch, rnn.seq_len: seq_len_train, rnn.keep_prob: FLAGS.dropout_keep_prob}
attention_output, _, global_step, train_summaries, train_loss, train_accuracy = sess.run([rnn.attention_output,rnn.train_op, rnn.global_step,
merged_summary, rnn.loss, rnn.accuracy], feed_dict=feed_dict)
rnn_feature_temp.append(attention_output.tolist())
print(rnn_feature_temp[0:2])
print(len(rnn_feature_temp))
np.savetxt("../data/word_data/word_dim/word_rnn_attention_embeddings_600_dim256.txt", np.array(rnn_feature_temp).reshape(20480,200))
log_device_placement=False)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
saver = tf.train.Saver(max_to_keep=5)
writer = tf.summary.FileWriter(log_dir)
sess.run(tf.global_variables_initializer())
best_r1i7 = -1.0
score_writer = open(os.path.join(model_dir, "eval_results.txt"),
mode="w",
encoding="utf-8")
for epoch in range(configs.epochs):
for data in tqdm(
batch_iter(train_set, video_features,
configs.batch_size, configs.extend, True),
total=num_train_batches,
desc="Epoch %d / %d" % (epoch + 1, configs.epochs)):
# run the model
feed_dict = get_feed_dict(data, model, configs.drop_rate)
_, loss, h_loss, global_step = sess.run(
[
model.train_op, model.loss, model.highlight_loss,
model.global_step
],
feed_dict=feed_dict)
if global_step % configs.period == 0:
write_tf_summary(writer,
[("train/loss", loss),
def train_rnn():
# Data Preparation
# ==================================================
if FLAGS.init_embedding_path is not None:
embedding = np.load(FLAGS.init_embedding_path)
print("Using pre-trained word embedding which shape is {}\n".format(embedding.shape))
FLAGS.vocab_size = embedding.shape[0]
FLAGS.embedding_size = embedding.shape[1]
if FLAGS.init_model_path is not None:
assert os.path.isdir(FLAGS.init_model_path), "init_model_path must be a directory\n"
ckpt = tf.train.get_checkpoint_state(FLAGS.init_model_path)
assert ckpt, "No checkpoint found in {}\n".format(FLAGS.init_model_path)
assert ckpt.model_checkpoint_path, "No model_checkpoint_path found in checkpoint\n"
# Create root directory
timestamp = str(int(time.time()))
root_dir = os.path.join(os.path.curdir, 'runs', 'textrnn', 'trained_result_' + timestamp)
os.makedirs(root_dir)
# Load data
# print("Loading data...\n")
# x, y = data_helpers.load_data(FLAGS.data_file, FLAGS.sequence_length, FLAGS.vocab_size, root_dir=root_dir)
# FLAGS.num_classes = len(y[0])
print("Loading data...\n")
x_data = np.loadtxt(FLAGS.x_data_file)
y_data = np.loadtxt(FLAGS.y_data_file)
print("data load finished")
# Split dataset
x_train, x_test, y_train, y_test = train_test_split(x_data, y_data, test_size=FLAGS.test_size, stratify=y_data, random_state=0)
x_val, x_test, y_val, y_test = train_test_split(x_test, y_test, test_size=0.5, random_state=0)
# Training
# ==================================================
with tf.Graph().as_default():
tf_config = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
tf_config.gpu_options.allow_growth = FLAGS.gpu_allow_growth
with tf.Session(config=tf_config).as_default() as sess:
rnn = TextRNN(
vocab_size=FLAGS.vocab_size,
embedding_size=FLAGS.embedding_size,
sequence_length=FLAGS.sequence_length,
rnn_size=FLAGS.rnn_size,
num_layers=FLAGS.num_layers,
attention_size=FLAGS.attention_size,
num_classes=FLAGS.num_classes,
learning_rate=FLAGS.learning_rate,
grad_clip=FLAGS.grad_clip)
# Output directory for models and summaries
out_dir = os.path.abspath(root_dir)
print("Writing to {}...\n".format(out_dir))
# Summaries for loss and accuracy
tf.summary.scalar("loss", rnn.loss)
tf.summary.scalar("accuracy", rnn.accuracy)
merged_summary = tf.summary.merge_all()
# Summaries dictionary
train_summary_dir = os.path.join(out_dir, 'summaries', 'train')
val_summary_dir = os.path.join(out_dir, 'summaries', 'val')
train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph)
val_summary_writer = tf.summary.FileWriter(val_summary_dir, sess.graph)
# Checkpoint directory, will not create itself
checkpoint_dir = os.path.abspath(os.path.join(out_dir, 'checkpoints'))
checkpoint_prefix = os.path.join(checkpoint_dir, 'model.ckpt')
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=1)
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Using pre-trained word embedding
if FLAGS.init_embedding_path is not None:
sess.run(rnn.embedding.assign(embedding))
del embedding
# Continue training from saved model
if FLAGS.init_model_path is not None:
saver.restore(sess, ckpt.model_checkpoint_path)
# Training start
print("Start training...\n")
best_at_step = 0
best_val_accuracy = 0
for epoch in range(FLAGS.num_epochs):
# Generate train batches
train_batches = data_utils.batch_iter(list(zip(x_train, y_train)), FLAGS.batch_size)
start = time.time()
for batch in train_batches:
# Training model on x_batch and y_batch
x_batch, y_batch = zip(*batch)
# seq_len_train = data_helpers.real_len(x_batch)
seq_len_train = data_utils.real_len(x_batch)
feed_dict = {rnn.input_x: x_batch, rnn.input_y: y_batch, rnn.seq_len: seq_len_train, rnn.keep_prob: FLAGS.dropout_keep_prob}
_, global_step, train_summaries, train_loss, train_accuracy = sess.run([rnn.train_op, rnn.global_step,
merged_summary, rnn.loss, rnn.accuracy], feed_dict=feed_dict)
# Evaluates model on val set
if global_step % FLAGS.evaluate_every == 0:
end = time.time()
train_summary_writer.add_summary(train_summaries, global_step)
seq_len_val = data_utils.real_len(x_val)
feed_dict = {rnn.input_x: x_val, rnn.input_y: y_val, rnn.seq_len: seq_len_val, rnn.keep_prob: 1.0}
val_summaries, val_loss, val_accuracy = sess.run([merged_summary, rnn.loss, rnn.accuracy], feed_dict=feed_dict)
val_summary_writer.add_summary(val_summaries, global_step)
print("Epoch: {}, global step: {}, training speed: {:.3f}sec/batch".format(epoch,
global_step, (end - start) / FLAGS.evaluate_every))
print("train loss: {:.3f}, train accuracy: {:.3f}, val loss: {:.3f}, val accuracy: {:.3f}\n".format(train_loss,
train_accuracy, val_loss, val_accuracy))
# If improved, save the model
if val_accuracy > best_val_accuracy:
print("Get a best val accuracy at step {}, model saving...\n".format(global_step))
saver.save(sess, checkpoint_prefix, global_step=global_step)
best_val_accuracy = val_accuracy
best_at_step = global_step
start = time.time()
# Rename the checkpoint
best_model_prefix = checkpoint_prefix + '-' + str(best_at_step)
os.rename(best_model_prefix + '.index', os.path.join(checkpoint_dir, 'best_model.index'))
os.rename(best_model_prefix + '.meta', os.path.join(checkpoint_dir, 'best_model.meta'))
os.rename(best_model_prefix + '.data-00000-of-00001', os.path.join(checkpoint_dir, 'best_model.data-00000-of-00001'))
# Testing on test set
print("\nTraining complete, testing the best model on test set...\n")
saver.restore(sess, os.path.join(checkpoint_dir, 'best_model'))
seq_len_test = data_utils.real_len(x_test)
feed_dict = {rnn.input_x: x_test, rnn.input_y: y_test, rnn.seq_len: seq_len_test, rnn.keep_prob: 1.0}
# y_logits, test_accuracy = sess.run([rnn.logits, rnn.accuracy], feed_dict=feed_dict)
# label_transformer = joblib.load(os.path.join(out_dir, 'label_transformer.pkl'))
# y_test_original = label_transformer.inverse_transform(y_test)
# y_logits_original = label_transformer.inverse_transform(y_logits)
# print("Precision, Recall and F1-Score:\n\n", classification_report(y_test_original, y_logits_original))
y_pred, test_accuracy = sess.run([rnn.predictions, rnn.accuracy], feed_dict=feed_dict)
print("Testing Accuracy: {:.3f}\n".format(test_accuracy))
y_test_original = np.argmax(y_test, 1)
print(y_test_original.shape)
print("Precision, Recall and F1-Score:\n\n",
classification_report(y_test_original, y_pred, target_names=['体育', '军事', '医学', '文化', '汽车', '经济']))
def train(train_x, train_y, test_x, test_y, vocab_size, embedding_size,
pretrained_embedding, trainset_embedding_matrix, args):
'''
traing process + testing process
:param train_x: training dataset
:param train_y: training label
:param test_x: testing dataset
:param test_y: testing label
:param vocab_size: number of vocabulary in embedding matrx
:param embedding_size: embedding size for each word
:param pretrained_embedding: pretrained word embedding matrix
:param trainset_embedding_matrix: oov word(from train set) embedding matrix
:param args:
:return:
print testing result by fixed epoch interval
'''
with tf.Session() as sess:
model = WordRNN(MAX_DOCUMENT_LEN,
NUM_CLASS,
vocab_size=vocab_size,
embedding_size=embedding_size,
trainset_embedding=trainset_embedding_matrix)
# Define training procedure
global_step = tf.Variable(0, trainable=False)
params = tf.trainable_variables()
gradients = tf.gradients(model.loss, params)
clipped_gradients, _ = tf.clip_by_global_norm(gradients, 5.0)
optimizer = tf.train.AdamOptimizer(model.lr)
train_op = optimizer.apply_gradients(zip(clipped_gradients, params),
global_step=global_step)
# Initialize all variables
feed_dict_emb = {model.x_init: np.float32(pretrained_embedding)}
sess.run(tf.global_variables_initializer(), feed_dict=feed_dict_emb)
def train_step(batch_x, batch_y):
feed_dict = {
model.x: batch_x,
model.y: batch_y,
model.keep_prob: 0.8,
}
_, step, loss = sess.run([train_op, global_step, model.loss],
feed_dict=feed_dict)
return loss
def test_accuracy(test_x, test_y):
'''
:param test_x: testing dataset
:param test_y: testing label
:return:
eval_loss: loss
accuracy: accuracy
ave_precison_score: average precison
'''
true_onehot_labels = []
predicted_onehot_scores = []
predicted_onehot_labels_t2 = []
test_batches = batch_iter(test_x, test_y, BATCH_SIZE, 1)
eval_loss, eval_counter = 0., 0
for test_batch_x, test_batch_y in test_batches:
scores, cur_loss = sess.run(
[model.scores, model.loss],
feed_dict={
model.x: test_batch_x,
model.y: test_batch_y,
model.keep_prob: 1.0
})
for i in test_batch_y:
true_onehot_labels.append(i)
for j in scores:
predicted_onehot_scores.append(j)
batch_predicted_onehot_labels = get_onehot_label_topk(
scores=scores, top_num=NUM_LABEL)
for i in batch_predicted_onehot_labels:
predicted_onehot_labels_t2.append(i)
eval_loss = eval_loss + cur_loss
eval_counter = eval_counter + 1
#metrics
eval_loss = float(eval_loss / eval_counter)
ave_precision_score = average_precision_score(
y_true=np.array(true_onehot_labels),
y_score=np.array(predicted_onehot_scores),
average='micro')
accuracy = accuracy_score(np.array(true_onehot_labels),
np.array(predicted_onehot_labels_t2))
return eval_loss, accuracy, ave_precision_score
# Training loop
start = time.time()
batches = batch_iter(train_x, train_y, BATCH_SIZE, NUM_EPOCHS)
st = time.time()
steps_per_epoch = int(num_train / BATCH_SIZE)
for batch_x, batch_y in batches:
step = tf.train.global_step(sess, global_step)
num_epoch = int(step / steps_per_epoch)
loss = train_step(batch_x, batch_y)
if step % 50 == 0:
eval_loss, acc, eval_prc = test_accuracy(test_x, test_y)
mode = "w" if step == 0 else "a"
with open(args.summary_dir + "-accuracy.txt", mode) as f:
print("epo: {}, step: {}, loss: {}, accuracy: {}".format(
num_epoch, step, eval_loss, acc),
file=f)
print(
"epoch: {}, step: {}, loss: {}, steps_per_epoch: {}, batch size: {}"
.format(num_epoch, step, eval_loss, steps_per_epoch,
BATCH_SIZE))
print("Accuracy:{},Avg_Precision: {}, loss:{}".format(
acc, eval_prc, eval_loss))
print("time of one epoch: {}\n".format(time.time() - st))
st = time.time()
print('training time', time.time() - start)
# #
test_start_time = time.time()
eval_loss, acc, eval_prc = test_accuracy(test_x, test_y)
print('testing time', time.time() - test_start_time)
print(eval_loss, acc)
feed_dict = {
quinn.input_x: x_batch,
quinn.input_y: y_batch,
quinn.attention_map: x_map
}
step, loss, mae, _update_op = sess.run(
[global_step, quinn.loss, quinn.mae, quinn.update_op],
feed_dict)
time_str = datetime.datetime.now().isoformat()
print("{}: step {}, loss {:g}, mae {:g}".format(
time_str, step, loss, mae))
# Generate batches
batches = utils.batch_iter(
list(zip(x_train, x_train_map, y_train_prob)), batch_size,
num_epochs)
# Training loop. For each batch...
for batch in batches:
x_batch, x_map, y_batch = zip(*batch)
train_step(x_batch, x_map, y_batch)
current_step = tf.train.global_step(sess, global_step)
epoch_step = (int((len(x_train[0]) - 1) / batch_size) + 1)
if current_step % epoch_step == 0:
print("\nValidation:")
# Randomly draw a validation batch
shuff_idx = np.random.permutation(np.arange(batch_size))
