def __init__(self, urm, ucm, binary=False, verbose=True, mode='offline', datareader=None, verbose_evaluation=True, bm25=False, similarity='tversky'): assert (mode in ('offline', 'online')) if binary: ucm.data = np.ones(ucm.data.shape[0]) self.urm = urm self.binary = binary self.verbose = verbose self.verbose_ev = verbose_evaluation self.dr = datareader self.mode = mode self.similarity = similarity self.bm25 = bm25 ucm_aux = ucm.copy() ut.inplace_set_rows_zero(X=ucm_aux, target_rows=self.dr.get_test_pids() ) #don't learn from challange set ucm_aux.eliminate_zeros() if self.bm25: self.m_ui = bm25_row(ucm.copy()).tocsr() else: self.m_ui = ucm.copy().tocsr() if self.bm25: self.m_iu = bm25_col(ucm_aux.T.copy()).tocsr() else: self.m_iu = ucm_aux.T.copy().tocsr() if mode == 'offline': self.ev = Evaluator(self.dr)
def __init__(self, urm, ucm=None, stopwords=[], load_ucm=False, save_ucm=False, binary=False, verbose=True, mode='offline', datareader=None, verbose_evaluation=True): assert(mode in ('offline', 'online')) if binary: urm.data=np.ones(urm.data.shape[0]) # best: norm, wor, split, skipw, porter2, lanca2 norm = True work = True split = True skip_words = True date = False porter = False porter2 = True lanca = False lanca2 = True data1 = False self.ucm=ucm if self.ucm is None and not load_ucm: nlp = NLP(datareader, stopwords=stopwords, norm=norm, work=work, split=split, date=date, skip_words=skip_words, porter=porter, porter2=porter2, lanca=lanca, lanca2=lanca2) self.ucm = nlp.get_UCM(data1=data1) elif self.ucm is None and load_ucm: self.load_ucm('ucm_nlp.npz') if save_ucm: self.save_ucm('ucm_nlp.npz') self.m_uc = pre.bm25_row(self.ucm.copy()).tocsr() self.m_cu = pre.bm25_row(self.ucm.copy()).T.tocsr() self.urm = urm self.binary = binary self.verbose = verbose self.verbose_ev = verbose_evaluation self.dr = datareader self.mode = mode if mode == 'offline': self.ev = Evaluator(self.dr)
def grid_holeboost(): datareader = Datareader(mode='offline', only_load=True, verbose=False) ev = Evaluator(datareader) # LOAD AND COMBINE eurm_lele = sparse.load_npz( ROOT_DIR + '/data/lele/ensembled_CLUSTERARTISTScat4-5-6-8-10_offline.npz') eurm_std = sparse.load_npz(ROOT_DIR + '/data/lele/ensembled_SUBCREATIVA_offline.npz') eurm_ens = combine_two_eurms(eurm_lele, eurm_std, cat_first=[4, 5, 6, 8, 10]) sim_offline = sparse.load_npz(ROOT_DIR + '/data/sim_offline.npz') for k in [50, 100, 150, 200, 250, 300, 350, 400]: for gamma in [1, 2, 5, 10]: h = HoleBoost(similarity=sim_offline, eurm=eurm_ens, datareader=datareader, norm=norm_l1_row) eurm_ens_boosted = h.boost_eurm(categories=[8, 10], k=k, gamma=gamma) rec_list = eurm_to_recommendation_list(eurm_ens_boosted, datareader=datareader) print( '--------------------------------------------------------------------------' ) print('K =', k) print('G =', gamma) ev.evaluate(rec_list, name='hb', save=False)
def offline(): # INIT dr = Datareader(mode='offline', only_load=True, verbose=False) ev = Evaluator(dr) # LOAD AND COMBINE eurm_lele = sparse.load_npz( ROOT_DIR + '/data/lele/ensembled_CLUSTERARTISTScat4-5-6-8-10_offline.npz') eurm_std = sparse.load_npz(ROOT_DIR + '/data/lele/ensembled_SUBCREATIVA_offline.npz') eurm_ens = combine_two_eurms(eurm_lele, eurm_std, cat_first=[4, 5, 6, 8, 10]) # LOAD # eurm_ens = sparse.load_npz(ROOT_DIR + '/data/ensembled_creativeFIRE_offline.npz') sim = sparse.load_npz(ROOT_DIR + '/data/sim_offline.npz') # TOPBOOST # topb = TopBoost(dr, eurm_ens, sim) # eurm_ens = topb.boost_eurm(categories=[9], top_k=100, gamma=0.01) # HOLEBOOST hb = HoleBoost(similarity=sim, eurm=eurm_ens, datareader=dr, norm=norm_l1_row) eurm_ens = hb.boost_eurm(categories=[8], k=300, gamma=1) hb = HoleBoost(similarity=sim, eurm=eurm_ens, datareader=dr, norm=norm_l1_row) eurm_ens = hb.boost_eurm(categories=[10], k=150, gamma=1) # TAILBOOST tb = TailBoost(similarity=sim, eurm=eurm_ens, datareader=dr, norm=norm_l2_row) eurm_ens = tb.boost_eurm(categories=[9, 7, 6, 5], last_tracks=[10, 3, 3, 3], k=[100, 80, 100, 100], gamma=[0.01, 0.01, 0.01, 0.01]) # ALBUMBOOST ab = AlbumBoost(dr, eurm_ens) eurm_ens = ab.boost_eurm(categories=[3, 4, 7, 9], gamma=2, top_k=[3, 3, 10, 40]) # MATCHBOOST # mb = MatchBoost(datareader=dr, eurm=eurm_ens, top_k_alb=5000, top_k_art=10000) # eurm_ens, pids = mb.boost_eurm(categories='all', k_art=300, k_alb=300, gamma_art=0.1, gamma_alb=0.1) # EVALUATION rec_list = eurm_to_recommendation_list(eurm_ens, datareader=dr) sparse.save_npz('FINAL.npz', eurm_ens) ev.evaluate(rec_list, name='LELE_boosts.csv')
def __init__(self, UCM, URM_train, test_playlists_indices, logFile, bestIndividualFile, mode="selection", numGenerations=30, populationSize=30, initialRandomDistribution=np.random.uniform(0, 1), verbose=True): self.UCM = UCM self.URM_train = URM_train self.test_playlists_indices = test_playlists_indices.astype(np.int) self.logFile = open(logFile, "a") self.bestIndividualFile = open(bestIndividualFile, "a") self.initialRandomDistribution = initialRandomDistribution self.verbose = verbose self.top = 0 self.current = 0 self.evaluator = Evaluator(Datareader(mode='offline', only_load=True, verbose=False)) self.NUM_VARIABLES = UCM.shape[1] if (mode == "weighting" or mode == "selection"): self.mode = mode # Crossover probability self.CXPB = 0.5 # Mutation probability self.MUTPB = 0.2 # Number of generations for which the evolution runs self.NGEN = numGenerations self.POPULATION_SIZE = populationSize
def icm(): datareader = Datareader(mode='offline', only_load=True) evaluator = Evaluator(datareader) print('NLP...') stopwords = STOP_WORDS token_weights = np.array(TOKEN_WEIGHTS) test_playlists = datareader.get_test_pids() nlp = NLP(datareader=datareader, stopwords=[], mode='tracks') print('Getting ucm and icm...') icm = nlp.get_icm() icm = bm25_row(icm) print('Computing similarity...') start = time.time() # Compute similarity similarity = tversky_similarity(icm, shrink=200, alpha=0.1, beta=1) similarity = similarity.tocsr() print(time.time() - start) urm = datareader.get_urm() print('Computing eurm...') start = time.time() # Compute eurm eurm_nlp = dot_product(urm[test_playlists, :], similarity, k=500) eurm_nlp = eurm_nlp.tocsr() # sparse.save_npz(ROOT_DIR + '/data/eurm_nlp_weighted_offline.npz', eurm_nlp) evaluator.evaluate(eurm_to_recommendation_list(eurm_nlp), name='nlp_enriched')
def __init__(self, matrices_names, matrices_array, dr, cat, start, end, n_calls=1000, n_random_starts=0.1, n_points=50, step=0.001, verbose=True): self.target_metric = 'ndcg' self.best_score = 0 self.best_params = 0 self.norm = norm_max_row self.verbose = verbose self.n_cpu = int(multiprocessing.cpu_count() / 10) if self.n_cpu == 0: self.n_cpu = 1 # Do not edit self.start = start self.end = end self.cat = cat self.global_counter = 0 self.start_index = (cat - 1) * 1000 self.end_index = cat * 1000 self.matrices_array = list() self.matrices_names = matrices_names self.n_calls = n_calls self.global_counter = 0 self.x0 = None self.y0 = None self.n_random_starts = int(n_calls * n_random_starts) self.n_points = n_points self.step = step # memory_on_disk= False self.memory_on_notebook = True self.dr = dr self.ev = Evaluator(self.dr) for matrix in matrices_array: self.matrices_array.append( self.norm( eurm_remove_seed( matrix, datareader=dr)[self.start_index:self.end_index])) del self.dr, matrices_array
def classification(): """ Classification. """ # create data X, y = DataCreator().make_classification(sklearn=True, n_classes=2) # train kNN classifier # clf = kNN(n_neighbors=3) # clf.fit(X, y) # train SVM classifier # clf = SVM(kernel="polynomial", C=1.0, p=2, s=5.0) # y[np.where(y == 0)] = -1 # clf.fit(X, y) # train logistic regression classifier # clf = LogisticRegression(poly=False) # clf.fit(X, y, batch_size=X.shape[0]) # train one-vs-one logistic regression classifier # clf = LogRegOneVsOne(poly=True) # clf.fit(X, y) # train a neural network # clf = NeuralNetwork(layers=[10, 5]) # clf.fit(X, y) # train tensorflow mlp clf = MLP() clf.fit(X, y) # train Fisher's linear discriminant # clf = LDA(n_dims=1) # y[np.where(y == 0)] = -1 # clf.fit(X, y) # Expectation Maximization # em = EM() # em.fit(X, n_comp=3, n_iter=30) # plot boundary BoundaryPlotter(X, y).plot_boundary(clf, step_size=0.005) # evaluation evaluator = Evaluator() acc = evaluator.accuracy(clf, X, y) print("Accuracy: {} %".format(acc)) evaluator.conf_mat(clf, X, y)
def prova(): dr = Datareader(mode='offline', only_load=True) print(dr.get_artist_to_tracks_dict()) exit() dr = Datareader(mode='offline', only_load=True, verbose=False) test_playlists = dr.get_test_pids() stopwords = STOP_WORDS token_weights = np.array(TOKEN_WEIGHTS) nlp = NLP(mode='playlists', datareader=dr, stopwords=STOP_WORDS) s = nlp.get_ucm() print(s.shape) evaluator = Evaluator(dr) ucm = nlp.get_ucm() sim = sparse.load_npz(ROOT_DIR + '/data/cf_user_similarity.npz') print('Computing dot...') ucm = dot_product(sim, ucm, k=200) print('NNZ', ucm.nnz) exit() urm = dr.get_urm() # ucm = ucm.astype(np.float64) # inplace_csr_column_scale(ucm, token_weights) print('Computing similarity...') start = time.time() # Compute similarity similarity = tversky_similarity(ucm, shrink=200, alpha=0.1, beta=1) similarity = similarity.tocsr() print(time.time() - start) print('Computing eurm...') start = time.time() # Compute eurm eurm_nlp = dot_product(similarity, urm, k=500) eurm_nlp = eurm_nlp.tocsr() eurm_nlp = eurm_nlp[test_playlists, :] #sparse.save_npz(ROOT_DIR + '/data/eurm_nlp_weighted_offline.npz', eurm_nlp) evaluator.evaluate(eurm_to_recommendation_list(eurm_nlp), name='nlp_enriched')
def _load_evaluator(self): print('loading evaluator...') from utils.evaluator import Evaluator self.evaluator = Evaluator() print('evaluator load finished!')
def configuration(cls, plm=None, method='lgesql', table_path='data/tables.json', tables='data/tables.bin', db_dir='data/database'): cls.plm, cls.method = plm, method cls.grammar = ASDLGrammar.from_filepath(GRAMMAR_FILEPATH) cls.trans = TransitionSystem.get_class_by_lang('sql')(cls.grammar) cls.tables = pickle.load(open(tables, 'rb')) if type(tables) == str else tables cls.evaluator = Evaluator(cls.trans, table_path, db_dir) if plm is None: cls.word2vec = Word2vecUtils() cls.tokenizer = lambda x: x cls.word_vocab = Vocab( padding=True, unk=True, boundary=True, default=UNK, filepath='./pretrained_models/glove.42b.300d/vocab.txt', specials=SCHEMA_TYPES) # word vocab for glove.42B.300d else: cls.tokenizer = AutoTokenizer.from_pretrained( os.path.join('./pretrained_models', plm)) cls.word_vocab = cls.tokenizer.get_vocab() cls.relation_vocab = Vocab(padding=False, unk=False, boundary=False, iterable=RELATIONS, default=None) cls.graph_factory = GraphFactory(cls.method, cls.relation_vocab)
def evaluate_from_pred_dict(predictions, refvg_split, analyze_subset=True, exp_name_in_summary=None, save_result_to_path=None, update_predictions=False, verbose=False): evaluator = Evaluator(refvg_split=refvg_split, analyze_subset=analyze_subset) mb = list() for img_id, im_preds in predictions.items(): pred_mask_tag = None pred_boxes_tag = None correct_tag = None for task_id, task_pred in im_preds.items(): if 'pred_mask' in task_pred: pred_mask_tag = 'pred_mask' if 'pred_boxlist' in task_pred: pred_boxes_tag = 'pred_boxlist' if 'correct' in task_pred: correct_tag = 'correct' break if len(mb) == 0: if pred_mask_tag is not None: mb.append('mask') if pred_boxes_tag is not None: mb.append('box') im, ib = evaluator.eval_single_img(img_id=img_id, im_pred_dict=im_preds, pred_mask_tag=pred_mask_tag, pred_boxes_tag=pred_boxes_tag, correct_tag=correct_tag, verbose=verbose) if update_predictions: for task_id, task_pred in im_preds.items(): if task_id in im: task_pred['mask_iou'] = im[task_id] if task_id in ib: task_pred['box_iou'] = ib[task_id] evaluator.analyze_stats(mask_box=mb, exp_name_in_summary=exp_name_in_summary, save_result_to_path=save_result_to_path) return predictions
def main(args): sys.stdout = Logger(args.log_dir) train_loader, test_loader = get_data(args) model = BaseModel(args) evaluator = Evaluator(model=model, data_loader=test_loader) best_acc = evaluator.evaluate() accuracies = [best_acc] losses = [] for e in range(1, args.epochs + 1): epoch_loss = 0 print("Epoch", e) for data in tqdm(train_loader): model.set_input(data) model.optimize_parameters() epoch_loss += model.get_loss() print("Epoch finished with loss", epoch_loss) losses.append(epoch_loss) if e % args.eval_step == 0: acc = evaluator.evaluate() accuracies.append(acc) best_acc = max(acc, best_acc) print("[Epoch {}] Accuracy:{:.2f}, Best Accuracy:{:.2f}".format( e, acc, best_acc)) if e % args.save_step == 0: model.save_model(e) model.update_lr() plt.figure() plt.plot(range(len(losses)), losses) plt.xlabel('Epochs') plt.ylabel('Training Loss') plt.savefig(os.path.join(args.exp_dir, 'losses.png')) plt.figure() plt.plot(range(len(accuracies)), accuracies) plt.xlabel('Epochs') plt.ylabel('Test Accuracy') plt.savefig(os.path.join(args.exp_dir, 'accuracies.png'))
def polyfit_upperbound(dataset, degree): evaluator = Evaluator(dataset, '/tmp', degree) print('Predicting with upperbound...') for i, anno in enumerate(progressbar(dataset.annotations)): label = anno['label'] pred = np.zeros((label.shape[0], 1 + 2 + degree + 1)) pred[:, :3] = label[:, :3] for j, lane in enumerate(label): if lane[0] == 0: continue xy = lane[3:] x = xy[:(len(xy) // 2)] y = xy[(len(xy) // 2):] ind = x > 0 pred[j, -(degree + 1):] = np.polyfit(y[ind], x[ind], degree) evaluator.add_prediction([i], pred, 0.0005) # 0.0005 = dummy runtime _, result = evaluator.eval(label='upperbound', only_metrics=True) return result
def __init__(self, args): self.args = args # 初始化tensorboard summary self.summary = TensorboardSummary(directory=args.save_path) self.writer = self.summary.create_summary() # 初始化dataloader kwargs = {'num_workers': args.workers, 'pin_memory': True} self.train_dataset = Apolloscapes('train_dataset.csv', '/home/aistudio/data/data1919/Image_Data', '/home/aistudio/data/data1919/Gray_Label', args.crop_size, type='train') self.dataloader = DataLoader(self.train_dataset, batch_size=args.batch_size, shuffle=True, drop_last=True, **kwargs) self.val_dataset = Apolloscapes('val_dataset.csv', '/home/aistudio/data/data1919/Image_Data', '/home/aistudio/data/data1919/Gray_Label', args.crop_size, type='val') self.val_loader = DataLoader(self.val_dataset, batch_size=args.batch_size, shuffle=False, drop_last=False, **kwargs) # 初始化model self.model = DeeplabV3Plus(backbone=args.backbone, output_stride=args.out_stride, batch_norm=args.batch_norm, num_classes=args.num_classes, pretrain=True) # 初始化优化器 self.optimizer = torch.optim.SGD(self.model.parameters(), momentum=args.momentum, nesterov=args.nesterov, weight_decay=args.weight_decay, lr=args.lr) # 定义损失函数 self.loss = CELoss(num_class=args.num_classes, cuda=args.cuda) # 定义验证器 self.evaluator = Evaluator(args.num_classes) # 定义学习率 self.scheduler = LR_Scheduler('poly', args.lr, args.epochs, len(self.dataloader)) # 使用cuda if args.cuda: self.model = self.model.cuda(device=args.gpus[0]) self.model = torch.nn.DataParallel(self.model, device_ids=args.gpus)
def __init__(self, icm, urm, binary=False, verbose=True, mode='offline', datareader=None, verbose_evaluation=True): assert (mode in ('offline', 'online')) if binary: urm.data = np.ones(urm.data.shape[0]) self.urm = urm self.m_ic = pre.bm25_col(icm.copy()).tocsr() self.m_ci = pre.bm25_col(icm.T.copy()).tocsr() self.binary = binary self.verbose = verbose self.verbose_ev = verbose_evaluation self.dr = datareader self.mode = mode if mode == 'offline': self.ev = Evaluator(self.dr)
def hseq_eval(): with open(FLAGS.config, 'r') as f: test_config = yaml.load(f, Loader=yaml.FullLoader) # Configure dataset hseq_utils = HSeqUtils(test_config['hseq']) prepare_reg_feat( hseq_utils, os.path.join(test_config['eval']['reg_model'], 'model.ckpt-550000')) # Configure evaluation evaluator = Evaluator(test_config['eval']) # Construct inference networks. output_tensors = inference(test_config['network']) # Create the initializier. config = tf.compat.v1.ConfigProto() config.gpu_options.allow_growth = True with tf.compat.v1.Session(config=config) as sess: # Restore pre-trained model. recoverer( sess, os.path.join(test_config['eval']['loc_model'], 'model.ckpt-400000')) producer_queue = Queue(maxsize=18) consumer_queue = Queue() producer0 = Thread(target=loader, args=(hseq_utils, test_config['network']['ori_est'], test_config['network']['dense_desc'], producer_queue)) producer0.daemon = True producer0.start() producer1 = Thread(target=extractor, args=(producer_queue, sess, output_tensors, test_config['network'], consumer_queue)) producer1.daemon = True producer1.start() consumer = Thread(target=matcher, args=(consumer_queue, sess, evaluator)) consumer.daemon = True consumer.start() producer0.join() producer1.join() consumer.join() evaluator.stats['all_avg_recall'] /= max(hseq_utils.seq_num, 1) evaluator.stats['i_avg_recall'] /= max(hseq_utils.seq_i_num, 1) evaluator.stats['v_avg_recall'] /= max(hseq_utils.seq_v_num, 1) print(evaluator.stats)
def evaluate(self, category, train=True): print "Evaluting discriminator pipeline for category", category evaluated_train = Evaluator("sigmoid_normal3", "ade20k") evaluated_val = Evaluator("sigmoid_normal3", "ade20k_val") im_list_train = evaluated_train.get_im_list_by_category(category) im_list_val = evaluated_val.get_im_list_by_category(category) print "Train instances:", len(im_list_train) print "Val instances:", len(im_list_val) if train: print "Training..." self.train(category, im_list_train) print "Training Done" else: print "Skipping training" print "Running on validation data" output_fn = "{}-ade20k_val.txt".format(category) fn_val = self.run(category, im_list_val, output_fn=output_fn) plot.plot(fn_val, evaluated_val, category, "ade20k_val") print "Running on training data" output_fn = "{}-ade20k.txt".format(category) fn_train = self.run(category, im_list_train, output_fn=output_fn) plot.plot(fn_train, evaluated_train, category, "ade20k") plot.vis(fn_val, evaluated_val, category, "ade20k_val", self.datasource_val)
def __init__(self, urm, pop=None, binary=False, verbose=True, mode='offline', datareader=None, verbose_evaluation=True): assert (mode in ('offline', 'online')) if binary: urm.data = np.ones(urm.data.shape[0]) if pop is None: self.pop = urm.sum(axis=0).A1 else: self.pop = pop self.urm = urm self.m_ui = pre.bm25_row(urm.copy()).tocsr() self.m_iu = pre.bm25_row(urm.T.copy()).tocsr() self.binary = binary self.verbose = verbose self.verbose_ev = verbose_evaluation self.dr = datareader self.mode = mode if mode == 'offline': self.ev = Evaluator(self.dr)
def grid_tailboost(): datareader = Datareader(mode='offline', only_load=True, verbose=False) ev = Evaluator(datareader) # LOAD AND COMBINE eurm_lele = sparse.load_npz( ROOT_DIR + '/data/lele/ensembled_CLUSTERARTISTScat4-5-6-8-10_offline.npz') eurm_std = sparse.load_npz(ROOT_DIR + '/data/lele/ensembled_SUBCREATIVA_offline.npz') eurm_ens = combine_two_eurms(eurm_lele, eurm_std, cat_first=[4, 5, 6, 8, 10]) sim = sparse.load_npz(ROOT_DIR + '/data/sim_offline.npz') # TAILBOOST for lt in [2, 3, 5, 6, 10]: for k in [20, 50, 80, 100, 150]: for g in [0.005, 0.01, 0.02, 0.05]: tb = TailBoost(similarity=sim, eurm=eurm_ens, datareader=datareader, norm=norm_l2_row) eurm_ens = tb.boost_eurm(categories=[9, 7, 6, 5], last_tracks=[lt, lt, lt, lt], k=[k, k, k, k], gamma=[g, g, g, g]) rec_list = eurm_to_recommendation_list(eurm_ens, datareader=datareader) print( '--------------------------------------------------------------------------' ) print('LT =', lt) print('K =', k) print('G =', g) ev.evaluate(rec_list, name='tb', save=False)
def __init__(self, settings: dict, settings_to_log: list): self.settings = settings self.settings_to_log = settings_to_log self.threshold = self.settings['threshold'] self.start_epoch = self.settings['start_epoch'] self.dataset = self.settings['dataset'] self.batch_size = self.settings['batch_size'] self.workers = self.settings['workers'] self.cuda = self.settings['cuda'] self.fp16 = self.settings['fp16'] self.epochs = self.settings['epochs'] self.ignore_index = self.settings['ignore_index'] self.loss_reduction = self.settings['loss_reduction'] # -------------------- Define Data loader ------------------------------ self.loaders, self.nclass, self.plotter = make_data_loader(settings) self.train_loader, self.val_loader, self.test_loader = [self.loaders[key] for key in ['train', 'val', 'test']] # -------------------- Define model ------------------------------------ self.model = get_model(self.settings) # -------------------- Define optimizer and its options ---------------- self.optimizer = define_optimizer(self.model, self.settings['optimizer'], self.settings['optimizer_params']) if self.settings['lr_scheduler']: self.lr_scheduler = LRScheduler(self.settings['lr_scheduler'], self.optimizer, self.batch_size) # -------------------- Define loss ------------------------------------- input_size = (self.batch_size, self.nclass, *self.settings['target_size']) self.criterion = CustomLoss(input_size=input_size, ignore_index=self.ignore_index, reduction=self.loss_reduction) self.evaluator = Evaluator(metrics=self.settings['metrics'], num_class=self.nclass, threshold=self.settings['threshold']) self.logger = MainLogger(loggers=self.settings['loggers'], settings=settings, settings_to_log=settings_to_log) if self.settings['resume']: self.resume_checkpoint(self.settings['resume']) self.metric_to_watch = 0.0
def __init__(self, urm, pop=None, binary=False, K1=1.2, B=0.75, verbose=True, mode='offline', datareader=None, verbose_evaluation=True, mode_t=False, trick=False): assert (mode in ('offline', 'online')) if binary: urm.data = np.ones(urm.data.shape[0]) if pop is None: self.pop = urm.sum(axis=0).A1 else: self.pop = pop self.dr = datareader self.urm = urm urm_aux = urm.copy() ut.inplace_set_rows_zero(X=urm_aux, target_rows=self.dr.get_test_pids() ) #don't learn from challange set urm_aux.eliminate_zeros() if mode_t: self.m_ui = urm_aux.copy().tocsr() else: self.m_ui = pre.bm25_row(urm_aux.copy(), K1=K1, B=B).tocsr() if mode_t: self.m_iu = urm_aux.T.copy().tocsr() else: self.m_iu = pre.bm25_row(urm_aux.T.copy(), K1=K1, B=B).tocsr() self.binary = binary self.verbose = verbose self.verbose_ev = verbose_evaluation self.mode = mode self.mode_t = mode_t if trick: self.urm = pre.bm25_row( urm).tocsr() #high click, high ndcg, better no use if mode == 'offline': self.ev = Evaluator(self.dr)
def __init__(self, urm, pop=None, binary=False, verbose=True, mode='offline', datareader=None, verbose_evaluation=True, similarity='tversky'): assert (mode in ('offline', 'online')) if binary: urm.data = np.ones(urm.data.shape[0]) if pop is None: self.pop = urm.sum(axis=1).A1 #####maybe better use followers else: self.pop = pop self.urm = urm self.binary = binary self.verbose = verbose self.verbose_ev = verbose_evaluation self.dr = datareader self.mode = mode self.similarity = similarity urm_aux = urm.copy() ut.inplace_set_rows_zero(X=urm_aux, target_rows=self.dr.get_test_pids() ) #don't learn from challange set urm_aux.eliminate_zeros() if self.similarity == 'p3alpha': self.m_ui = pre.bm25_col(urm.copy()).tocsr() else: self.m_ui = urm.copy().tocsr() if self.similarity == 'p3alpha': self.m_iu = pre.bm25_col(urm_aux.T.copy()).tocsr() else: self.m_iu = urm_aux.T.copy().tocsr() if mode == 'offline': self.ev = Evaluator(self.dr)
def main(config: ConfigParser): device = torch.device("cuda:0" if config["n_gpu"] > 0 else "cpu") train_iters, test_iters, vocab = load_dataset(config, device) print("Vocab size:", len(vocab)) print("prepare Evaluator...") evaluator_path = config["evaluator"]["save_path"] if evaluator_path is None or not Path(evaluator_path).exists(): evaluator = Evaluator.create(**config["evaluator"]["args"]) config["evaluator"]["save_path"] = (config.save_dir / "evaluator").as_posix() evaluator.save(config["evaluator"]["save_path"]) else: evaluator = Evaluator.load(evaluator_path) config.save() model_F = config.initialize("arch_generator", module_arch, vocab=vocab).to(device) model_D = config.initialize("arch_discriminator", module_arch, vocab=vocab).to(device) opt_F = config.initialize("generator_optimizer", module_optim) opt_D = config.initialize("discriminator_optimizer", module_optim) opt_F.set_parameters(model_F.parameters()) opt_D.set_parameters(model_D.parameters()) train(config=config, vocab=vocab, model_F=model_F, model_D=model_D, opt_F=opt_F, opt_D=opt_D, train_iters=train_iters, test_iters=test_iters, evaluator=evaluator)
def new(): datareader = Datareader(mode='offline', only_load=True) evaluator = Evaluator(datareader) print('NLP...') stopwords = STOP_WORDS token_weights = np.array(TOKEN_WEIGHTS) test_playlists = datareader.get_test_pids() nlp = NLP(datareader=datareader, stopwords=[], mode='both') print('Getting ucm and icm...') ucm = nlp.get_ucm() ucm = bm25_row(ucm) icm = nlp.get_icm() icm = bm25_row(icm) icm_T = icm.T #ucm = bm25_row(ucm) #urm = datareader.get_urm() print('Computing eurm...') start = time.time() eurm_nlp = dot_product(ucm[test_playlists, :], icm_T, k=500) print(time.time() - start) print('Converting to csr...') eurm_nlp = eurm_nlp.tocsr() print(eurm_nlp.shape) #eurm_nlp = eurm_nlp[test_playlists:, :] sparse.save_npz(ROOT_DIR + '/data/eurm_nlp_new_method_offline.npz', eurm_nlp) evaluator.evaluate(eurm_to_recommendation_list(eurm_nlp), name='nlp_new_method', show_plot=False)
def evaluate_from_pred_folder(pred_folder, refvg_split, analyze_subset=True, exp_name_in_summary=None, save_result_to_path=None, verbose=False): print(exp_name_in_summary) evaluator = Evaluator(refvg_split=refvg_split, analyze_subset=analyze_subset) fnames = os.listdir(pred_folder) fnames.sort() img_preds = dict() cur_img_id = '' for fname in fnames: if not fname.endswith('.png'): continue task_id = fname.split('.')[0] img_id = task_id.split('__')[0] pred_mask = png_to_pred_mask(os.path.join(pred_folder, fname)) if img_id == cur_img_id: img_preds[task_id] = {'pred_mask': pred_mask} else: if len(img_preds) > 0: # print(img_preds) evaluator.eval_single_img(img_id=int(cur_img_id), im_pred_dict=img_preds, pred_mask_tag='pred_mask', verbose=verbose) cur_img_id = img_id img_preds = dict() img_preds[task_id] = {'pred_mask': pred_mask} if len(img_preds) > 0: evaluator.eval_single_img(img_id=int(cur_img_id), im_pred_dict=img_preds, pred_mask_tag='pred_mask', verbose=verbose) evaluator.analyze_stats(mask_box=['mask'], exp_name_in_summary=exp_name_in_summary, save_result_to_path=save_result_to_path) return
def inference( images_path: str, sentences_path: str, test_imgs_file_path: str, batch_size: int, load_model_path: str, joint_space: int, ): # Check for CUDA device = torch.device("cuda" if torch.cuda.is_available() else "cpu") dataset_test = FlickrDatasetValTest(images_path, sentences_path, test_imgs_file_path) test_loader = DataLoader( dataset_test, batch_size=batch_size, num_workers=4, collate_fn=collate_pad_batch, pin_memory=True, ) # Create the model model = nn.DataParallel(ImageTextMatchingModel(joint_space)).to(device) # Load model model.load_state_dict(torch.load(load_model_path)) # Set model in evaluation mode model.train(False) # Create evaluator evaluator = Evaluator(len(dataset_test), joint_space) with torch.no_grad(): evaluator.reset_all_vars() for images, sentences in tqdm(test_loader): images, sentences = images.to(device), sentences.to(device) embedded_images, embedded_sentences = model(images, sentences) evaluator.update_embeddings( embedded_images.cpu().numpy().copy(), embedded_sentences.cpu().numpy().copy(), ) print("=============================") print(f"Image-text recall at 1, 5, 10: " f"{evaluator.image2text_recall_at_k()} \n" f"Text-image recall at 1, 5, 10: " f"{evaluator.text2image_recall_at_k()}") print("=============================")
def hseq_eval(): with open(FLAGS.config, 'r') as f: test_config = yaml.load(f, Loader=yaml.FullLoader) # Configure dataset hseq_utils = HSeqUtils(test_config['hseq']) # Configure evaluation evaluator = Evaluator(test_config['eval']) # Construct inference networks. model = get_model('feat_model')(test_config['model_path'], **(test_config['net'])) # Create the initializier. config = tf.compat.v1.ConfigProto() config.gpu_options.allow_growth = True producer_queue = Queue(maxsize=18) consumer_queue = Queue() producer0 = Thread(target=loader, args=(hseq_utils, producer_queue)) producer0.daemon = True producer0.start() producer1 = Thread(target=extractor, args=(producer_queue, model, consumer_queue)) producer1.daemon = True producer1.start() consumer = Thread(target=matcher, args=(consumer_queue, model.sess, evaluator, test_config['eval'])) consumer.daemon = True consumer.start() producer0.join() producer1.join() consumer.join() evaluator.print_stats('i_eval_stats') evaluator.print_stats('v_eval_stats') evaluator.print_stats('all_eval_stats')
import sys from scipy import sparse import numpy as np import utils.pre_processing as pre from utils.definitions import * from utils.datareader import Datareader from utils.evaluator import Evaluator from utils.pre_processing import * from utils.post_processing import * dr = Datareader(mode='offline', only_load=True, verbose=False) ev = Evaluator(dr) urm = dr.get_urm(binary=True) urm_csc = urm.tocsc(copy=True) sim_nlp = sparse.load_npz(ROOT_DIR + '/data/sim_nlp_lele.npz') for k in [1, 2, 3, 4, 5]: eurm_top = dr.get_eurm_top_pop_filter_cat_1(sim_nlp, k, topk=500) eurm_top = norm_l1_row(eurm_top) eurm_nlp = sparse.load_npz(ROOT_DIR + '/data/nlp_fusion_tuned_offline.npz') eurm_nlp = norm_l1_row(eurm_nlp) for a in [0.05, 0.10, 0.15, 0.20]: eurm = eurm_nlp * (1.0 - a) + eurm_top * a rec_list = eurm_to_recommendation_list(eurm, datareader=dr) ev.evaluate(rec_list, name='pop_first_k=' + str(k) + '_a=' + str(a))
import sys from scipy import sparse import numpy as np import utils.pre_processing as pre from utils.definitions import * from utils.datareader import Datareader from utils.evaluator import Evaluator from utils.pre_processing import * from utils.post_processing import * dr = Datareader(mode='offline', only_load=True, verbose=False) ev = Evaluator(dr) urm = dr.get_urm(binary=True) pos_matrix = dr.get_position_matrix(position_type='last') rows = [] cols = [] data = [] for p in tqdm(range(pos_matrix.shape[0])): start = pos_matrix.indptr[p] end = pos_matrix.indptr[p + 1] tracks = pos_matrix.indices[start:end] positions = pos_matrix.indices[start:end] for idx in range(len(tracks)): if positions[idx] <= 250: rows.append(p) cols.append((tracks[idx] * positions[idx]) + tracks[idx])