def run_tests(predictor, model_file, dataset, get_full_recommendation_list=False, k=10): # Load model predictor.load(model_file) #predictor.load_last(os.path.dirname(model_file) + '/') # Prepare evaluator evaluator = evaluation.Evaluator(dataset, k=k) if get_full_recommendation_list: k = dataset.n_items count = 0 for sequence, user_id in dataset.test_set(epochs=1): count += 1 num_viewed = int(len(sequence) / 2) viewed = sequence[:num_viewed] goal = [i[0] for i in sequence[num_viewed:]] recommendations = predictor.top_k_recommendations(viewed, user_id=user_id, k=k) evaluator.add_instance(goal, recommendations) if len(goal) == 0: raise ValueError return evaluator
def _compute_validation_metrics(self, metrics): clusters = np.zeros(self.n_clusters, dtype="int") used_items = [] ev = evaluation.Evaluator(self.dataset, k=10) ev_clusters = evaluation.Evaluator(self.dataset, k=10) try: for batch, goal in self._gen_mini_batch( self.dataset.validation_set(epochs=1), test=True): pred1, pred2, cl, i = self.test_function(batch) ev.add_instance(goal, pred1) ev_clusters.add_instance(goal, pred2) clusters[cl] += 1 used_items.append(i) except Exception as e: print(e) if self.cluster_type == 'softmax': ignored_items = 0 cluster_size = np.histogram( self.cluster_repartition.get_value(borrow=True).argmax(axis=1), bins=range(self.n_clusters + 1))[0].tolist() elif self.cluster_type == 'mix': ignored_items = 0 sig_clusters = self.cluster_repartition.get_value(borrow=True) > 0. softmax_clusters = self.cluster_repartition.get_value( borrow=True).argmax(axis=1) for i in range(self.n_items): sig_clusters[i, softmax_clusters[i]] = True cluster_size = sig_clusters.sum(axis=0) else: ignored_items = (self.cluster_repartition.get_value( borrow=True).max(axis=1) < 0.).sum() cluster_size = (self.cluster_repartition.get_value(borrow=True) > 0.).sum(axis=0) metrics['recall'].append(ev.average_recall()) metrics['cluster_recall'].append(ev_clusters.average_recall()) metrics['sps'].append(ev.sps()) metrics['cluster_sps'].append(ev_clusters.sps()) metrics['assr'].append(self.n_items / np.mean(used_items)) metrics['ignored_items'].append(ignored_items) metrics['cluster_use'].append(clusters) metrics['cluster_use_std'].append(np.std(clusters)) metrics['cluster_size'].append(cluster_size) return metrics
def _compute_validation_metrics(self, metrics): """ add value to lists in metrics dictionary """ ev = evaluation.Evaluator(self.dataset, k=10) if not self.iter: for batch_input, goal in self._gen_mini_batch( self.dataset.validation_set(epochs=1), test=True): predictions = self.test_function( [batch_input[0], batch_input[1]]) # print("predictions") # print(predictions) ev.add_instance(goal, predictions) else: for sequence, user in self.dataset.validation_set(epochs=1): seq_lengths = list(range( 1, len(sequence))) # 1, 2, 3, ... len(sequence)-1 for length in seq_lengths: X = np.zeros( (1, self.max_length, self._input_size()), dtype=self._input_type) # input shape of the RNN mask = np.zeros( (1, self.max_length) ) # mask of the input (to deal with sequences of different length) seq_by_max_length = sequence[max( length - self.max_length, 0):length] # last max length or all X[0, :len(seq_by_max_length), :] = np.array( map(lambda x: self._get_features(x), seq_by_max_length)) mask[0, :len(seq_by_max_length)] = 1 predictions = self.test_function( [X, mask.astype(theano.config.float)]) # print("predictions") # print(predictions) goal = sequence[length:][0] ev.add_instance(goal, predictions) metrics['recall'].append(ev.average_recall()) metrics['sps'].append(ev.sps()) metrics['precision'].append(ev.average_precision()) metrics['ndcg'].append(ev.average_ndcg()) metrics['user_coverage'].append(ev.user_coverage()) metrics['item_coverage'].append(ev.item_coverage()) metrics['blockbuster_share'].append(ev.blockbuster_share()) # del ev ev.instances = [] return metrics
def compute_validation_metrics(model, dataset, metrics): """ add value to lists in metrics dictionary """ ev = evaluation.Evaluator(dataset, k=10) print(iter) if not iter: for batch_input, goal in gen_mini_batch(dataset.validation_set(), test=False): # test=True # print(batch_input[0].shape()) # output = model.predict_on_batch(batch_input[0]) output = model.predict_on_batch(batch_input) predictions = np.argpartition(-output, list(range(10)), axis=-1)[0, :10] # print("predictions") # print(predictions) ev.add_instance(goal, predictions) else: for sequence, user in dataset.validation_set(epochs=1): seq_lengths = list(range( 1, len(sequence))) # 1, 2, 3, ... len(sequence)-1 for length in seq_lengths: X = np.zeros((1, max_length, n_items), dtype=input_type) # input shape of the RNN seq_by_max_length = sequence[max(length - max_length, 0): length] # last max length or all X[0, :len(seq_by_max_length), :] = np.array( map(lambda x: get_features(x), seq_by_max_length)) output = model.predict_on_batch(X) predictions = np.argpartition(-output, list(range(10)), axis=-1)[0, :10] # print("predictions") # print(predictions) goal = sequence[length:][0] ev.add_instance(goal, predictions) metrics['recall'].append(ev.average_recall()) metrics['sps'].append(ev.sps()) metrics['precision'].append(ev.average_precision()) metrics['ndcg'].append(ev.average_ndcg()) metrics['user_coverage'].append(ev.user_coverage()) metrics['item_coverage'].append(ev.item_coverage()) metrics['blockbuster_share'].append(ev.blockbuster_share()) # del ev ev.instances = [] return metrics
def run_tests(predictor, model_file, dataset, args, get_full_recommendation_list=False, k=10): predictor._load(model_file) #predictor.load_last(os.path.dirname(model_file) + '/') # Prepare evaluator evaluator = evaluation.Evaluator(dataset, k=k) if get_full_recommendation_list: k = dataset.n_items nb_of_dp = [] start = time.clock() for sequence, user_id in dataset.test_set(epochs=1): if not args.test_iter: num_viewed = int(len(sequence) / 2) viewed = sequence[:num_viewed] goal = [i[0] for i in sequence[num_viewed:]] #list of movie ids recommendations = predictor.top_k_recommendations(viewed, k=k) #recommendations(movie ids) 잘 추가되게 하면 됨 #print(recommendations) evaluator.add_instance(goal, recommendations) if len(goal) == 0: raise ValueError else: #seq_lengths = sorted(random.sample(xrange(1, len(sequence)),len(sequence) - 1)) seq_lengths = list(range(1, len(sequence))) for length in seq_lengths: viewed = sequence[:length] goal = sequence[length:][0] recommendations = predictor.top_k_recommendations(viewed, k=k) evaluator.add_instance(goal, recommendations) end = time.clock() print('Timer: ', end - start) if len(nb_of_dp) == 0: evaluator.nb_of_dp = dataset.n_items else: evaluator.nb_of_dp = np.mean(nb_of_dp) return evaluator
def _compute_validation_metrics(self, metrics): ev = evaluation.Evaluator(self.dataset, k=10) for batch_input, goal in self._gen_mini_batch( self.dataset.validation_set(epochs=1), test=True): predictions = self.test_function(batch_input) ev.add_instance(goal, predictions) metrics['recall'].append(ev.average_recall()) metrics['sps'].append(ev.sps()) metrics['ndcg'].append(ev.average_ndcg()) metrics['user_coverage'].append(ev.user_coverage()) metrics['item_coverage'].append(ev.item_coverage()) metrics['blockbuster_share'].append(ev.blockbuster_share()) return metrics
def _compute_validation_metrics(self, metrics): ev = evaluation.Evaluator(self.dataset, k=10) for sequence, user_id in self.dataset.validation_set(epochs=1): top_k = self.top_k_recommendations(sequence[:len(sequence) // 2], user_id=int(user_id)) goal = [i[0] for i in sequence[len(sequence) // 2:]] ev.add_instance(goal, top_k) metrics['recall'].append(ev.average_recall()) metrics['sps'].append(ev.sps()) metrics['ndcg'].append(ev.average_ndcg()) metrics['user_coverage'].append(ev.user_coverage()) metrics['item_coverage'].append(ev.item_coverage()) metrics['blockbuster_share'].append(ev.blockbuster_share()) return metrics
def run_tests(predictor, model_file, dataset, args, get_full_recommendation_list=False, k=10): # Load model predictor.load(model_file) #predictor.load_last(os.path.dirname(model_file) + '/') # Prepare evaluator evaluator = evaluation.Evaluator(dataset, k=k) if get_full_recommendation_list: k = dataset.n_items count = 0 nb_of_dp = [] start = time.clock() for sequence, user_id in dataset.test_set(epochs=1): count += 1 num_viewed = int(len(sequence) / 2) viewed = sequence[:num_viewed] goal = [i[0] for i in sequence[num_viewed:]] if args.clusters > 0: recommendations, n = predictor.top_k_recommendations( viewed, user_id=user_id, k=k) nb_of_dp.append(n) else: recommendations = predictor.top_k_recommendations(viewed, user_id=user_id, k=k) evaluator.add_instance(goal, recommendations) if len(goal) == 0: raise ValueError end = time.clock() print('Timer: ', end - start) if len(nb_of_dp) == 0: evaluator.nb_of_dp = dataset.n_items else: evaluator.nb_of_dp = np.mean(nb_of_dp) return evaluator
def _compute_validation_metrics(self, dataset, metrics): """ add value to lists in v_metrics dictionary """ self.dataset = dataset ev = evaluation.Evaluator(self.dataset, k=10) # for batch_input, goal in gen_mini_batch(dataset.validation_set(epochs=1)): # test=True for sequence, user_id in self.dataset.validation_set(epochs=1): sequence = sequence[-min(self.max_length, len(sequence)):] num_viewed = int(len(sequence) / 2) viewed = sequence[:num_viewed] goal = [i[0] for i in sequence[num_viewed:]] # list of movie ids X = np.zeros((1, self.max_length), dtype=np.int32) # ktf embedding requires movie-id sequence, not one-hot X[0, :len(viewed)] = np.array([item[0] for item in viewed]) output = self.model.predict_on_batch(X) # output[[i[0] for i in viewed]] = -np.inf predictions = np.argpartition(-output, list(range(10)), axis=-1)[0, :10] # print("predictions") # print(predictions) ev.add_instance(goal, predictions) # metrics['recall'].append(ev.average_recall()) metrics['sps'].append(ev.sps()) metrics['sps_short'].append(ev.sps_short()) metrics['sps_long'].append(ev.sps_long()) metrics['precision'].append(ev.average_precision()) metrics['ndcg'].append(ev.average_ndcg()) metrics['user_coverage'].append(ev.user_coverage()) metrics['item_coverage'].append(ev.item_coverage()) metrics['total_item_coverage'].append(ev.total_item_coverage()) metrics['uniq_rec'].append(ev.uniq_rec()) metrics['blockbuster_share'].append(ev.blockbuster_share()) metrics['intra_list_similarity'].append(ev.average_intra_list_similarity()) # del ev ev.nb_of_dp = self.dataset.n_items ev.instances = [] return metrics
def _compute_validation_metrics(self, metrics): """ add value to lists in metrics dictionary """ ev = evaluation.Evaluator(self.dataset, k=10) if not self.iter: for batch_input, goal in self._gen_mini_batch( self.dataset.validation_set(epochs=1), test=True): output = self.sess.run(self.softmax, feed_dict={ self.X: batch_input[0], self.length: batch_input[2] }) predictions = np.argpartition(-output, list(range(10)), axis=-1)[0, :10] # print("predictions") # print(predictions) ev.add_instance(goal, predictions) else: for sequence, user in self.dataset.validation_set(epochs=1): seq_lengths = list(range( 1, len(sequence))) # 1, 2, 3, ... len(sequence)-1 for seq_length in seq_lengths: X = np.zeros( (1, self.max_length, self._input_size()), dtype=self._input_type) # input shape of the RNN # Y = np.zeros((1, self.n_items)) # Y가 왜 있는지????? 안쓰임 length = np.zeros((1, ), dtype=np.int32) seq_by_max_length = sequence[ max(length - self.max_length, 0):seq_length] # last max length or all X[0, :len(seq_by_max_length), :] = np.array( map(lambda x: self._get_features(x), seq_by_max_length)) length[0] = len(seq_by_max_length) output = self.sess.run(self.softmax, feed_dict={ self.X: X, self.length: length }) predictions = np.argpartition(-output, list(range(10)), axis=-1)[0, :10] # print("predictions") # print(predictions) goal = sequence[seq_length:][0] ev.add_instance(goal, predictions) metrics['recall'].append(ev.average_recall()) metrics['sps'].append(ev.sps()) metrics['precision'].append(ev.average_precision()) metrics['ndcg'].append(ev.average_ndcg()) metrics['user_coverage'].append(ev.user_coverage()) metrics['item_coverage'].append(ev.item_coverage()) metrics['blockbuster_share'].append(ev.blockbuster_share()) # del ev ev.instances = [] return metrics