def evaluate_batch(self, test_batch, sess): y = None y_prob = None seq, time, seq_mask, label_n, label_t = test_batch y_ = label_n if self.options['time_loss'] == 'mse': time_pred = 0 else: time_pred = self.predict_time(sess, time, label_t, seq) if self.node_pred: rnn_args = { self.input_nodes: seq, self.input_times: time # self.init_state: np.zeros((2, self.batch_size, self.state_size)) } y_prob_ = sess.run([self.probs], feed_dict=rnn_args) y_prob_ = y_prob_[0] # print(y_prob_.shape, log_lik.shape) for j, p in enumerate(y_prob_): test_seq_len = test_batch[2][j] test_seq = test_batch[0][j][0:int(sum(test_seq_len))] p[test_seq.astype(int)] = 0 y_prob_[j, :] = p / float(np.sum(p)) if y_prob is None: y_prob = y_prob_ y = y_ else: y = np.concatenate((y, y_), axis=0) y_prob = np.concatenate((y_prob, y_prob_), axis=0) node_score = metrics.portfolio(y_prob, y, k_list=[10, 50, 100]) else: node_score = {} return node_score, time_pred
def evaluate(f_prob, test_loader, k_list=[10, 50, 100]): ''' Evaluates trained model. ''' n_batches = len(test_loader) y = None y_prob = None for _ in range(n_batches): batch_data = test_loader() y_ = batch_data[-1] y_prob_ = f_prob(*batch_data[:-1]) # excludes activated nodes when predicting. for i, p in enumerate(y_prob_): length = int(np.sum(batch_data[1][:, i])) sequence = batch_data[0][:length, i] assert y_[i] not in sequence, str(sequence) + str(y_[i]) p[sequence] = 0. y_prob_[i, :] = p / float(np.sum(p)) if y_prob is None: y_prob = y_prob_ y = y_ else: y = np.concatenate((y, y_), axis=0) y_prob = np.concatenate((y_prob, y_prob_), axis=0) return metrics.portfolio(y_prob, y, k_list=k_list)
def evaluate_topk(f_prob, test_loader, f_tprob, tdim, node_reverse_index, savefile, k_list=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]): ''' Evaluates trained model. ''' n_batches = len(test_loader) y = None y_prob = None yt = None yt_prob = None alltopk = None for _ in range(n_batches): batch_data = test_loader() y_ = batch_data[-2] y_prob_ = f_prob(*batch_data[:-3]) yt_ = batch_data[-1] yt_prob_ = f_tprob(*batch_data[:-2]) topk = np.zeros((y_.size, 10), dtype=int) # excludes activated nodes when predicting. for i, p in enumerate(y_prob_): length = int(np.sum(batch_data[1][:, i])) #sequence = batch_data[0][: length, i] #assert y_[i] not in sequence, str(sequence) + str(y_[i]) #p[sequence] = 0. y_prob_[i, :] = p / float(np.sum(p)) tmp_prob = p / float(np.sum(p)) topk_ = sorted(range(tmp_prob.size), key=lambda k: tmp_prob[k], reverse=True)[:10] topk[i, :] = [node_reverse_index[d] for d in topk_] if y_prob is None: y_prob = y_prob_ y = y_ yt_prob = yt_prob_ yt = yt_ alltopk = topk else: y = np.concatenate((y, y_), axis=0) y_prob = np.concatenate((y_prob, y_prob_), axis=0) yt = np.concatenate((yt, yt_), axis=0) yt_prob = np.concatenate((yt_prob, yt_prob_), axis=0) alltopk = np.concatenate((alltopk, topk), axis=0) np.savetxt(savefile + "topk", alltopk, fmt='%d', delimiter=',') return metrics.portfolio(y_prob, y, yt_prob, yt, tdim, k_list=k_list)
def evaluate(f_prob, test_loader, k_list=[10, 50, 100], test_batch=False): ''' Evaluates trained model. ''' n_batches = len(test_loader) print("n_batches---", n_batches) y = None y_prob = None y_hate = None for _ in range(n_batches): batch_data = test_loader() y_ = batch_data[-1] # print y_ # print y_h_ # print len(y_) # print len(y_h_) # print type(y_) # print type(y_h_) # if test_batch: y_prob_ = f_prob(*batch_data[:-2]) y_h_ = batch_data[-2] # else: # y_prob_ = f_prob(*batch_data[:-1]) # y_h_=None # excludes activated nodes when predicting. for i, p in enumerate(y_prob_): length = int(np.sum(batch_data[1][:, i])) sequence = batch_data[0][:length, i] assert y_[i] not in sequence, str(sequence) + str(y_[i]) p[sequence] = 0. y_prob_[i, :] = p / float(np.sum(p)) if y_prob is None: y_prob = y_prob_ y = y_ y_hate = y_h_ # if y_prob is None and test_batch: # y_hate = y_h_ else: y = np.concatenate((y, y_), axis=0) y_prob = np.concatenate((y_prob, y_prob_), axis=0) # if test_batch: y_hate = np.concatenate((y_hate, y_h_), axis=0) return metrics.portfolio(y_prob, y, y_hate, k_list=k_list, test_batch=test_batch)
def evaluate_model(self, sess, test_it): test_batch_size = len(test_it) y = None y_prob = None node_scores = [] time_scores = [] for i in range(0, test_batch_size): test_batch = test_it() seq, time, seq_mask, label_n, label_t = test_batch if seq.shape[0] < self.batch_size: continue '''else: node_score, time_score = self.evaluate_batch(test_batch, sess) node_scores.append(node_score) time_scores.append(time_score)''' if self.loss_type == 'mse': time_pred = 0.0 else: time_pred = self.predict_time(sess, time, label_t, seq) time_scores.append(time_pred) y_ = label_n rnn_args = { # self.init_state: np.zeros((2, self.batch_size, self.state_size)), self.input_nodes: seq, self.input_times: time } y_prob_ = sess.run([self.probs], feed_dict=rnn_args) y_prob_ = y_prob_[0] for j, p in enumerate(y_prob_): test_seq_len = test_batch[3][j] test_seq = test_batch[0][j][0:int(test_seq_len)] assert y_[j] not in test_seq, str(test_seq) + str(y_[j]) p[test_seq.astype(int)] = 0. y_prob_[j, :] = p / float(np.sum(p)) if y_prob is None: y_prob = y_prob_ y = y_ else: y = np.concatenate((y, y_), axis=0) y_prob = np.concatenate((y_prob, y_prob_), axis=0) scores = metrics.portfolio(y_prob, y, k_list=[10, 50, 100]) scores['time_mse'] = np.mean( np.asarray(time_scores)) // test_batch_size return scores
def test_epoch(model, test_data, opt, k_list=[1, 5, 10, 20, 50, 100]): ''' Epoch operation in evaluation phase ''' model.eval() scores = {} for k in k_list: scores['hits@' + str(k)] = 0 scores['map@' + str(k)] = 0 n_total_words = 0 reward = 0.0 batch_num = 0.0 for batch in tqdm(test_data, mininterval=2, desc=' - (Test) ', leave=False): # prepare data tgt = batch gold = tgt[:, 1:] # forward pred, *_ = model(tgt, RL_train=False) scores_batch, scores_len = metrics.portfolio( pred.detach().cpu().numpy(), gold.contiguous().view(-1).detach().cpu().numpy(), k_list) n_total_words += scores_len for k in k_list: scores['hits@' + str(k)] += scores_batch['hits@' + str(k)] * scores_len scores['map@' + str(k)] += scores_batch['map@' + str(k)] * scores_len if opt.rl: pred_ids, pred_probs = model(tgt, RL_train=True) for i in range(10): loss_rl, expect_batch, reward_batch = get_performance_rl( pred_ids, pred_probs, tgt, expect=0) # with start ids reward += reward_batch batch_num += tgt.size(0) for k in k_list: scores['hits@' + str(k)] = scores['hits@' + str(k)] / n_total_words scores['map@' + str(k)] = scores['map@' + str(k)] / n_total_words return scores, reward / batch_num if opt.rl else 0
def evaluate(f_prob, test_loader, f_tprob, tdim, k_list=[1,5,10]): ''' Evaluates trained model. ''' n_batches = len(test_loader) y = None y_prob = None yt = None yt_prob = None for _ in range(n_batches): batch_data = test_loader() y_ = batch_data[-2] y_prob_ = f_prob(*batch_data[:-3]) yt_ = batch_data[-1] yt_prob_ = f_tprob(*batch_data[:-2]) # excludes activated nodes when predicting. for i, p in enumerate(y_prob_): length = int(np.sum(batch_data[1][:, i])) #sequence = batch_data[0][: length, i] #assert y_[i] not in sequence, str(sequence) + str(y_[i]) #p[sequence] = 0. y_prob_[i, :] = p / float(np.sum(p)) if y_prob is None: y_prob = y_prob_ y = y_ yt_prob = yt_prob_ yt = yt_ else: y = np.concatenate((y, y_), axis=0) y_prob = np.concatenate((y_prob, y_prob_), axis=0) yt = np.concatenate((yt, yt_), axis=0) yt_prob = np.concatenate((yt_prob, yt_prob_), axis=0) #np.savetxt('twitterpred',yt_prob) #np.savetxt('twittergt', yt) return metrics.portfolio(y_prob, y, yt_prob, yt, tdim, k_list=k_list)