def evaluate_batch(self, test_batch, sess):
y = None
y_prob = None
seq, time, seq_mask, label_n, label_t = test_batch
y_ = label_n
if self.options['time_loss'] == 'mse':
time_pred = 0
else:
time_pred = self.predict_time(sess, time, label_t, seq)
if self.node_pred:
rnn_args = {
self.input_nodes: seq,
self.input_times: time
# self.init_state: np.zeros((2, self.batch_size, self.state_size))
}
y_prob_ = sess.run([self.probs], feed_dict=rnn_args)
y_prob_ = y_prob_[0]
# print(y_prob_.shape, log_lik.shape)
for j, p in enumerate(y_prob_):
test_seq_len = test_batch[2][j]
test_seq = test_batch[0][j][0:int(sum(test_seq_len))]
p[test_seq.astype(int)] = 0
y_prob_[j, :] = p / float(np.sum(p))
if y_prob is None:
y_prob = y_prob_
y = y_
else:
y = np.concatenate((y, y_), axis=0)
y_prob = np.concatenate((y_prob, y_prob_), axis=0)
node_score = metrics.portfolio(y_prob, y, k_list=[10, 50, 100])
else:
node_score = {}
return node_score, time_pred
def evaluate(f_prob, test_loader, k_list=[10, 50, 100]):
'''
Evaluates trained model.
'''
n_batches = len(test_loader)
y = None
y_prob = None
for _ in range(n_batches):
batch_data = test_loader()
y_ = batch_data[-1]
y_prob_ = f_prob(*batch_data[:-1])
# excludes activated nodes when predicting.
for i, p in enumerate(y_prob_):
length = int(np.sum(batch_data[1][:, i]))
sequence = batch_data[0][:length, i]
assert y_[i] not in sequence, str(sequence) + str(y_[i])
p[sequence] = 0.
y_prob_[i, :] = p / float(np.sum(p))
if y_prob is None:
y_prob = y_prob_
y = y_
else:
y = np.concatenate((y, y_), axis=0)
y_prob = np.concatenate((y_prob, y_prob_), axis=0)
return metrics.portfolio(y_prob, y, k_list=k_list)
def evaluate_topk(f_prob,
test_loader,
f_tprob,
tdim,
node_reverse_index,
savefile,
k_list=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]):
'''
Evaluates trained model.
'''
n_batches = len(test_loader)
y = None
y_prob = None
yt = None
yt_prob = None
alltopk = None
for _ in range(n_batches):
batch_data = test_loader()
y_ = batch_data[-2]
y_prob_ = f_prob(*batch_data[:-3])
yt_ = batch_data[-1]
yt_prob_ = f_tprob(*batch_data[:-2])
topk = np.zeros((y_.size, 10), dtype=int)
# excludes activated nodes when predicting.
for i, p in enumerate(y_prob_):
length = int(np.sum(batch_data[1][:, i]))
#sequence = batch_data[0][: length, i]
#assert y_[i] not in sequence, str(sequence) + str(y_[i])
#p[sequence] = 0.
y_prob_[i, :] = p / float(np.sum(p))
tmp_prob = p / float(np.sum(p))
topk_ = sorted(range(tmp_prob.size),
key=lambda k: tmp_prob[k],
reverse=True)[:10]
topk[i, :] = [node_reverse_index[d] for d in topk_]
if y_prob is None:
y_prob = y_prob_
y = y_
yt_prob = yt_prob_
yt = yt_
alltopk = topk
else:
y = np.concatenate((y, y_), axis=0)
y_prob = np.concatenate((y_prob, y_prob_), axis=0)
yt = np.concatenate((yt, yt_), axis=0)
yt_prob = np.concatenate((yt_prob, yt_prob_), axis=0)
alltopk = np.concatenate((alltopk, topk), axis=0)
np.savetxt(savefile + "topk", alltopk, fmt='%d', delimiter=',')
return metrics.portfolio(y_prob, y, yt_prob, yt, tdim, k_list=k_list)
def evaluate(f_prob, test_loader, k_list=[10, 50, 100], test_batch=False):
'''
Evaluates trained model.
'''
n_batches = len(test_loader)
print("n_batches---", n_batches)
y = None
y_prob = None
y_hate = None
for _ in range(n_batches):
batch_data = test_loader()
y_ = batch_data[-1]
# print y_
# print y_h_
# print len(y_)
# print len(y_h_)
# print type(y_)
# print type(y_h_)
# if test_batch:
y_prob_ = f_prob(*batch_data[:-2])
y_h_ = batch_data[-2]
# else:
# y_prob_ = f_prob(*batch_data[:-1])
# y_h_=None
# excludes activated nodes when predicting.
for i, p in enumerate(y_prob_):
length = int(np.sum(batch_data[1][:, i]))
sequence = batch_data[0][:length, i]
assert y_[i] not in sequence, str(sequence) + str(y_[i])
p[sequence] = 0.
y_prob_[i, :] = p / float(np.sum(p))
if y_prob is None:
y_prob = y_prob_
y = y_
y_hate = y_h_
# if y_prob is None and test_batch:
# y_hate = y_h_
else:
y = np.concatenate((y, y_), axis=0)
y_prob = np.concatenate((y_prob, y_prob_), axis=0)
# if test_batch:
y_hate = np.concatenate((y_hate, y_h_), axis=0)
return metrics.portfolio(y_prob,
y,
y_hate,
k_list=k_list,
test_batch=test_batch)
def evaluate_model(self, sess, test_it):
test_batch_size = len(test_it)
y = None
y_prob = None
node_scores = []
time_scores = []
for i in range(0, test_batch_size):
test_batch = test_it()
seq, time, seq_mask, label_n, label_t = test_batch
if seq.shape[0] < self.batch_size:
continue
'''else:
node_score, time_score = self.evaluate_batch(test_batch, sess)
node_scores.append(node_score)
time_scores.append(time_score)'''
if self.loss_type == 'mse':
time_pred = 0.0
else:
time_pred = self.predict_time(sess, time, label_t, seq)
time_scores.append(time_pred)
y_ = label_n
rnn_args = {
# self.init_state: np.zeros((2, self.batch_size, self.state_size)),
self.input_nodes:
seq,
self.input_times:
time
}
y_prob_ = sess.run([self.probs], feed_dict=rnn_args)
y_prob_ = y_prob_[0]
for j, p in enumerate(y_prob_):
test_seq_len = test_batch[3][j]
test_seq = test_batch[0][j][0:int(test_seq_len)]
assert y_[j] not in test_seq, str(test_seq) + str(y_[j])
p[test_seq.astype(int)] = 0.
y_prob_[j, :] = p / float(np.sum(p))
if y_prob is None:
y_prob = y_prob_
y = y_
else:
y = np.concatenate((y, y_), axis=0)
y_prob = np.concatenate((y_prob, y_prob_), axis=0)
scores = metrics.portfolio(y_prob, y, k_list=[10, 50, 100])
scores['time_mse'] = np.mean(
np.asarray(time_scores)) // test_batch_size
return scores
def test_epoch(model, test_data, opt, k_list=[1, 5, 10, 20, 50, 100]):
''' Epoch operation in evaluation phase '''
model.eval()
scores = {}
for k in k_list:
scores['hits@' + str(k)] = 0
scores['map@' + str(k)] = 0
n_total_words = 0
reward = 0.0
batch_num = 0.0
for batch in tqdm(test_data,
mininterval=2,
desc=' - (Test) ',
leave=False):
# prepare data
tgt = batch
gold = tgt[:, 1:]
# forward
pred, *_ = model(tgt, RL_train=False)
scores_batch, scores_len = metrics.portfolio(
pred.detach().cpu().numpy(),
gold.contiguous().view(-1).detach().cpu().numpy(), k_list)
n_total_words += scores_len
for k in k_list:
scores['hits@' +
str(k)] += scores_batch['hits@' + str(k)] * scores_len
scores['map@' +
str(k)] += scores_batch['map@' + str(k)] * scores_len
if opt.rl:
pred_ids, pred_probs = model(tgt, RL_train=True)
for i in range(10):
loss_rl, expect_batch, reward_batch = get_performance_rl(
pred_ids, pred_probs, tgt, expect=0) # with start ids
reward += reward_batch
batch_num += tgt.size(0)
for k in k_list:
scores['hits@' + str(k)] = scores['hits@' + str(k)] / n_total_words
scores['map@' + str(k)] = scores['map@' + str(k)] / n_total_words
return scores, reward / batch_num if opt.rl else 0
def evaluate(f_prob, test_loader, f_tprob, tdim, k_list=[1,5,10]):
'''
Evaluates trained model.
'''
n_batches = len(test_loader)
y = None
y_prob = None
yt = None
yt_prob = None
for _ in range(n_batches):
batch_data = test_loader()
y_ = batch_data[-2]
y_prob_ = f_prob(*batch_data[:-3])
yt_ = batch_data[-1]
yt_prob_ = f_tprob(*batch_data[:-2])
# excludes activated nodes when predicting.
for i, p in enumerate(y_prob_):
length = int(np.sum(batch_data[1][:, i]))
#sequence = batch_data[0][: length, i]
#assert y_[i] not in sequence, str(sequence) + str(y_[i])
#p[sequence] = 0.
y_prob_[i, :] = p / float(np.sum(p))
if y_prob is None:
y_prob = y_prob_
y = y_
yt_prob = yt_prob_
yt = yt_
else:
y = np.concatenate((y, y_), axis=0)
y_prob = np.concatenate((y_prob, y_prob_), axis=0)
yt = np.concatenate((yt, yt_), axis=0)
yt_prob = np.concatenate((yt_prob, yt_prob_), axis=0)
#np.savetxt('twitterpred',yt_prob)
#np.savetxt('twittergt', yt)
return metrics.portfolio(y_prob, y, yt_prob, yt, tdim, k_list=k_list)
