def fit(
self,
train_iteractions: interactions.MatchInteraction,
verbose=True, # for printing out evaluation during training
val_interactions: interactions.MatchInteraction = None,
test_interactions: interactions.MatchInteraction = None):
"""
Fit the model.
Parameters
----------
train_iteractions: :class:`matchzoo.DataPack` The input sequence dataset.
val_interactions: :class:`matchzoo.DataPack`
test_interactions: :class:`matchzoo.DataPack`
"""
self._initialize()
best_ce, best_epoch, test_ce = sys.maxsize, 0, 0
test_results_dict = None
iteration_counter = 0
count_patience_epochs = 0
for epoch_num in range(self._n_iter):
# ------ Move to here ----------------------------------- #
self._net.train(True)
query_ids, left_contents, left_lengths, \
doc_ids, right_contents, target_contents, right_lengths = self._sampler.get_instances(train_iteractions)
queries, query_content, query_lengths, \
docs, doc_content, target_contents, doc_lengths = my_utils.shuffle(query_ids, left_contents, left_lengths,
doc_ids, right_contents, target_contents, right_lengths)
epoch_loss, total_pairs = 0.0, 0
t1 = time.time()
for (minibatch_num, (batch_query, batch_query_content, batch_query_len,
batch_doc, batch_doc_content, batch_doc_target, batch_docs_lens)) \
in enumerate(my_utils.minibatch(queries, query_content, query_lengths,
docs, doc_content, target_contents, doc_lengths,
batch_size = self._batch_size)):
t3 = time.time()
batch_query = my_utils.gpu(torch.from_numpy(batch_query),
self._use_cuda)
batch_query_content = my_utils.gpu(
torch.from_numpy(batch_query_content), self._use_cuda)
# batch_query_len = my_utils.gpu(torch.from_numpy(batch_query_len), self._use_cuda)
batch_doc = my_utils.gpu(torch.from_numpy(batch_doc),
self._use_cuda)
batch_doc_content = my_utils.gpu(
torch.from_numpy(batch_doc_content), self._use_cuda)
batch_doc_target = my_utils.gpu(
torch.from_numpy(batch_doc_target), self._use_cuda)
# batch_docs_lens = my_utils.gpu(torch.from_numpy(batch_docs_lens), self._use_cuda)
total_pairs += batch_query.size(0) # (batch_size)
self._optimizer.zero_grad()
loss = self._get_loss(batch_query, batch_query_content,
batch_doc, batch_doc_content,
batch_query_len, batch_docs_lens,
batch_doc_target)
epoch_loss += loss.item()
iteration_counter += 1
# if iteration_counter % 2 == 0: break
TensorboardWrapper.mywriter().add_scalar(
"loss/minibatch_loss", loss.item(), iteration_counter)
loss.backward()
torch.nn.utils.clip_grad_norm_(self._net.parameters(),
self._clip)
self._optimizer.step()
t4 = time.time()
# if iteration_counter % 100 == 0: print("Running time for each mini-batch: ", (t4 - t3), "s")
epoch_loss /= float(total_pairs)
TensorboardWrapper.mywriter().add_scalar("loss/epoch_loss_avg",
epoch_loss, epoch_num)
# print("Number of Minibatches: ", minibatch_num, "Avg. loss of epoch: ", epoch_loss)
t2 = time.time()
epoch_train_time = t2 - t1
if verbose: # validation after each epoch
t1 = time.time()
result_val = self.evaluate(val_interactions)
val_ce = result_val["cross_entropy"]
t2 = time.time()
validation_time = t2 - t1
TensorboardWrapper.mywriter().add_scalar(
"cross_entropy/val_ce", val_ce, epoch_num)
FileHandler.myprint(
'|Epoch %03d | Train time: %04.1f(s) | Train loss: %.3f'
'| Val loss = %.5f | Validation time: %04.1f(s)' %
(epoch_num, epoch_train_time, epoch_loss, val_ce,
validation_time))
if val_ce < best_ce:
count_patience_epochs = 0
with open(self.saved_model, "wb") as f:
torch.save(self._net.state_dict(), f)
# test_results_dict = result_test
best_ce, best_epoch = val_ce, epoch_num
else:
count_patience_epochs += 1
if self._early_stopping_patience and count_patience_epochs > self._early_stopping_patience:
FileHandler.myprint(
"Early Stopped due to no better performance in %s epochs"
% count_patience_epochs)
break
if np.isnan(epoch_loss) or epoch_loss == 0.0:
raise ValueError(
'Degenerate epoch loss: {}'.format(epoch_loss))
FileHandler.myprint("Closing tensorboard")
TensorboardWrapper.mywriter().close()
FileHandler.myprint(
'Best result: | vad cross_entropy = %.5f | epoch = %d' %
(best_ce, best_epoch))
FileHandler.myprint_details(
json.dumps(test_results_dict, sort_keys=True, indent=2))
def fit(
self,
train_iteractions: interactions.MatchInteraction,
verbose=True, # for printing out evaluation during training
topN=10,
val_interactions: interactions.MatchInteraction = None,
test_interactions: interactions.MatchInteraction = None):
"""
Fit the model.
Parameters
----------
train_iteractions: :class:`matchzoo.DataPack` The input sequence dataset.
val_interactions: :class:`matchzoo.DataPack`
test_interactions: :class:`matchzoo.DataPack`
"""
self._initialize(train_iteractions)
best_hit, best_ndcg, best_epoch, test_ndcg, test_hit = 0, 0, 0, 0, 0
test_results_dict = None
iteration_counter = 0
count_patience_epochs = 0
for epoch_num in range(self._n_iter):
# ------ Move to here ----------------------------------- #
self._net.train(True)
query_ids, left_contents, left_lengths, \
doc_ids, right_contents, right_lengths, \
neg_docs_ids, neg_docs_contents, neg_docs_lens = self._sampler.get_train_instances(train_iteractions, self._num_negative_samples)
queries, query_content, query_lengths, \
docs, doc_content, doc_lengths, \
neg_docs, neg_docs_contents, neg_docs_lens = my_utils.shuffle(query_ids, left_contents, left_lengths,
doc_ids, right_contents, right_lengths,
neg_docs_ids, neg_docs_contents, neg_docs_lens)
epoch_loss, total_pairs = 0.0, 0
t1 = time.time()
for (minibatch_num,
(batch_query, batch_query_content, batch_query_len,
batch_doc, batch_doc_content, batch_docs_lens,
batch_neg_docs, batch_neg_doc_content, batch_neg_docs_lens)) \
in enumerate(my_utils.minibatch(queries, query_content, query_lengths,
docs, doc_content, doc_lengths,
neg_docs, neg_docs_contents, neg_docs_lens,
batch_size = self._batch_size)):
# add idf here...
query_idfs = None
if len(TFIDF.get_term_idf()) != 0:
query_idf_dict = TFIDF.get_term_idf()
query_idfs = [[
query_idf_dict.get(int(word_idx), 0.0)
for word_idx in row
] for row in batch_query_content]
query_idfs = torch_utils.gpu(
torch.from_numpy(np.array(query_idfs)).float(),
self._use_cuda)
batch_query = my_utils.gpu(torch.from_numpy(batch_query),
self._use_cuda)
batch_query_content = my_utils.gpu(
torch.from_numpy(batch_query_content), self._use_cuda)
batch_doc = my_utils.gpu(torch.from_numpy(batch_doc),
self._use_cuda)
batch_doc_content = my_utils.gpu(
torch.from_numpy(batch_doc_content), self._use_cuda)
batch_neg_doc_content = my_utils.gpu(
torch.from_numpy(batch_neg_doc_content), self._use_cuda)
total_pairs += self._batch_size * self._num_negative_samples
self._optimizer.zero_grad()
if self._loss in ["bpr", "hinge", "pce", "bce"]:
loss = self._get_multiple_negative_predictions_normal(
batch_query,
batch_query_content,
batch_doc,
batch_doc_content,
batch_neg_docs,
batch_neg_doc_content,
batch_query_len,
batch_docs_lens,
batch_neg_docs_lens,
self._num_negative_samples,
query_idf=query_idfs)
epoch_loss += loss.item()
iteration_counter += 1
# if iteration_counter % 2 == 0: break
TensorboardWrapper.mywriter().add_scalar(
"loss/minibatch_loss", loss.item(), iteration_counter)
loss.backward()
self._optimizer.step()
epoch_loss /= float(total_pairs)
TensorboardWrapper.mywriter().add_scalar("loss/epoch_loss_avg",
epoch_loss, epoch_num)
# print("Number of Minibatches: ", minibatch_num, "Avg. loss of epoch: ", epoch_loss)
t2 = time.time()
epoch_train_time = t2 - t1
if verbose: # validation after each epoch
t1 = time.time()
assert len(val_interactions.unique_queries_test
) in KeyWordSettings.QueryCountVal, len(
val_interactions.unique_queries_test)
result_val = self.evaluate(val_interactions, topN)
hits = result_val["hits"]
ndcg = result_val["ndcg"]
t2 = time.time()
valiation_time = t2 - t1
if epoch_num and epoch_num % self._testing_epochs == 0:
t1 = time.time()
assert len(test_interactions.unique_queries_test
) in KeyWordSettings.QueryCountTest
result_test = self.evaluate(test_interactions, topN)
hits_test = result_test["hits"]
ndcg_test = result_test["ndcg"]
t2 = time.time()
testing_time = t2 - t1
TensorboardWrapper.mywriter().add_scalar(
"hit/hit_test", hits_test, epoch_num)
TensorboardWrapper.mywriter().add_scalar(
"ndcg/ndcg_test", ndcg_test, epoch_num)
FileHandler.myprint(
'|Epoch %03d | Test hits@%d = %.5f | Test ndcg@%d = %.5f | Testing time: %04.1f(s)'
% (epoch_num, topN, hits_test, topN, ndcg_test,
testing_time))
TensorboardWrapper.mywriter().add_scalar(
"hit/hits_val", hits, epoch_num)
TensorboardWrapper.mywriter().add_scalar(
"ndcg/ndcg_val", ndcg, epoch_num)
FileHandler.myprint(
'|Epoch %03d | Train time: %04.1f(s) | Train loss: %.3f'
'| Vad hits@%d = %.5f | Vad ndcg@%d = %.5f | Validation time: %04.1f(s)'
% (epoch_num, epoch_train_time, epoch_loss, topN, hits,
topN, ndcg, valiation_time))
if hits > best_hit or (hits == best_hit and ndcg > best_ndcg):
# if (hits + ndcg) > (best_hit + best_ndcg):
count_patience_epochs = 0
with open(self.saved_model, "wb") as f:
torch.save(self._net.state_dict(), f)
# test_results_dict = result_test
best_hit, best_ndcg, best_epoch = hits, ndcg, epoch_num
# test_hit, test_ndcg = hits_test, ndcg_test
else:
count_patience_epochs += 1
if self._early_stopping_patience and count_patience_epochs > self._early_stopping_patience:
FileHandler.myprint(
"Early Stopped due to no better performance in %s epochs"
% count_patience_epochs)
break
if np.isnan(epoch_loss) or epoch_loss == 0.0:
raise ValueError(
'Degenerate epoch loss: {}'.format(epoch_loss))
FileHandler.myprint("Closing tensorboard")
TensorboardWrapper.mywriter().close()
FileHandler.myprint(
'Best result: | vad hits@%d = %.5f | vad ndcg@%d = %.5f | epoch = %d'
% (topN, best_hit, topN, best_ndcg, best_epoch))
FileHandler.myprint_details(
json.dumps(test_results_dict, sort_keys=True, indent=2))