def __init__(self,
urm,
ucm,
binary=False,
verbose=True,
mode='offline',
datareader=None,
verbose_evaluation=True,
bm25=False,
similarity='tversky'):
assert (mode in ('offline', 'online'))
if binary: ucm.data = np.ones(ucm.data.shape[0])
self.urm = urm
self.binary = binary
self.verbose = verbose
self.verbose_ev = verbose_evaluation
self.dr = datareader
self.mode = mode
self.similarity = similarity
self.bm25 = bm25
ucm_aux = ucm.copy()
ut.inplace_set_rows_zero(X=ucm_aux,
target_rows=self.dr.get_test_pids()
) #don't learn from challange set
ucm_aux.eliminate_zeros()
if self.bm25: self.m_ui = bm25_row(ucm.copy()).tocsr()
else: self.m_ui = ucm.copy().tocsr()
if self.bm25: self.m_iu = bm25_col(ucm_aux.T.copy()).tocsr()
else: self.m_iu = ucm_aux.T.copy().tocsr()
if mode == 'offline':
self.ev = Evaluator(self.dr)
def __init__(self, urm, ucm=None, stopwords=[], load_ucm=False, save_ucm=False, binary=False, verbose=True, mode='offline', datareader=None, verbose_evaluation=True):
assert(mode in ('offline', 'online'))
if binary: urm.data=np.ones(urm.data.shape[0])
# best: norm, wor, split, skipw, porter2, lanca2
norm = True
work = True
split = True
skip_words = True
date = False
porter = False
porter2 = True
lanca = False
lanca2 = True
data1 = False
self.ucm=ucm
if self.ucm is None and not load_ucm:
nlp = NLP(datareader, stopwords=stopwords, norm=norm, work=work, split=split, date=date, skip_words=skip_words,
porter=porter, porter2=porter2, lanca=lanca, lanca2=lanca2)
self.ucm = nlp.get_UCM(data1=data1)
elif self.ucm is None and load_ucm:
self.load_ucm('ucm_nlp.npz')
if save_ucm:
self.save_ucm('ucm_nlp.npz')
self.m_uc = pre.bm25_row(self.ucm.copy()).tocsr()
self.m_cu = pre.bm25_row(self.ucm.copy()).T.tocsr()
self.urm = urm
self.binary = binary
self.verbose = verbose
self.verbose_ev = verbose_evaluation
self.dr = datareader
self.mode = mode
if mode == 'offline':
self.ev = Evaluator(self.dr)
def grid_holeboost():
datareader = Datareader(mode='offline', only_load=True, verbose=False)
ev = Evaluator(datareader)
# LOAD AND COMBINE
eurm_lele = sparse.load_npz(
ROOT_DIR +
'/data/lele/ensembled_CLUSTERARTISTScat4-5-6-8-10_offline.npz')
eurm_std = sparse.load_npz(ROOT_DIR +
'/data/lele/ensembled_SUBCREATIVA_offline.npz')
eurm_ens = combine_two_eurms(eurm_lele,
eurm_std,
cat_first=[4, 5, 6, 8, 10])
sim_offline = sparse.load_npz(ROOT_DIR + '/data/sim_offline.npz')
for k in [50, 100, 150, 200, 250, 300, 350, 400]:
for gamma in [1, 2, 5, 10]:
h = HoleBoost(similarity=sim_offline,
eurm=eurm_ens,
datareader=datareader,
norm=norm_l1_row)
eurm_ens_boosted = h.boost_eurm(categories=[8, 10],
k=k,
gamma=gamma)
rec_list = eurm_to_recommendation_list(eurm_ens_boosted,
datareader=datareader)
print(
'--------------------------------------------------------------------------'
)
print('K =', k)
print('G =', gamma)
ev.evaluate(rec_list, name='hb', save=False)
def offline():
# INIT
dr = Datareader(mode='offline', only_load=True, verbose=False)
ev = Evaluator(dr)
# LOAD AND COMBINE
eurm_lele = sparse.load_npz(
ROOT_DIR +
'/data/lele/ensembled_CLUSTERARTISTScat4-5-6-8-10_offline.npz')
eurm_std = sparse.load_npz(ROOT_DIR +
'/data/lele/ensembled_SUBCREATIVA_offline.npz')
eurm_ens = combine_two_eurms(eurm_lele,
eurm_std,
cat_first=[4, 5, 6, 8, 10])
# LOAD
# eurm_ens = sparse.load_npz(ROOT_DIR + '/data/ensembled_creativeFIRE_offline.npz')
sim = sparse.load_npz(ROOT_DIR + '/data/sim_offline.npz')
# TOPBOOST
# topb = TopBoost(dr, eurm_ens, sim)
# eurm_ens = topb.boost_eurm(categories=[9], top_k=100, gamma=0.01)
# HOLEBOOST
hb = HoleBoost(similarity=sim,
eurm=eurm_ens,
datareader=dr,
norm=norm_l1_row)
eurm_ens = hb.boost_eurm(categories=[8], k=300, gamma=1)
hb = HoleBoost(similarity=sim,
eurm=eurm_ens,
datareader=dr,
norm=norm_l1_row)
eurm_ens = hb.boost_eurm(categories=[10], k=150, gamma=1)
# TAILBOOST
tb = TailBoost(similarity=sim,
eurm=eurm_ens,
datareader=dr,
norm=norm_l2_row)
eurm_ens = tb.boost_eurm(categories=[9, 7, 6, 5],
last_tracks=[10, 3, 3, 3],
k=[100, 80, 100, 100],
gamma=[0.01, 0.01, 0.01, 0.01])
# ALBUMBOOST
ab = AlbumBoost(dr, eurm_ens)
eurm_ens = ab.boost_eurm(categories=[3, 4, 7, 9],
gamma=2,
top_k=[3, 3, 10, 40])
# MATCHBOOST
# mb = MatchBoost(datareader=dr, eurm=eurm_ens, top_k_alb=5000, top_k_art=10000)
# eurm_ens, pids = mb.boost_eurm(categories='all', k_art=300, k_alb=300, gamma_art=0.1, gamma_alb=0.1)
# EVALUATION
rec_list = eurm_to_recommendation_list(eurm_ens, datareader=dr)
sparse.save_npz('FINAL.npz', eurm_ens)
ev.evaluate(rec_list, name='LELE_boosts.csv')
def __init__(self, UCM, URM_train, test_playlists_indices, logFile, bestIndividualFile, mode="selection",
numGenerations=30, populationSize=30, initialRandomDistribution=np.random.uniform(0, 1),
verbose=True):
self.UCM = UCM
self.URM_train = URM_train
self.test_playlists_indices = test_playlists_indices.astype(np.int)
self.logFile = open(logFile, "a")
self.bestIndividualFile = open(bestIndividualFile, "a")
self.initialRandomDistribution = initialRandomDistribution
self.verbose = verbose
self.top = 0
self.current = 0
self.evaluator = Evaluator(Datareader(mode='offline', only_load=True, verbose=False))
self.NUM_VARIABLES = UCM.shape[1]
if (mode == "weighting" or mode == "selection"):
self.mode = mode
# Crossover probability
self.CXPB = 0.5
# Mutation probability
self.MUTPB = 0.2
# Number of generations for which the evolution runs
self.NGEN = numGenerations
self.POPULATION_SIZE = populationSize
def icm():
datareader = Datareader(mode='offline', only_load=True)
evaluator = Evaluator(datareader)
print('NLP...')
stopwords = STOP_WORDS
token_weights = np.array(TOKEN_WEIGHTS)
test_playlists = datareader.get_test_pids()
nlp = NLP(datareader=datareader, stopwords=[], mode='tracks')
print('Getting ucm and icm...')
icm = nlp.get_icm()
icm = bm25_row(icm)
print('Computing similarity...')
start = time.time()
# Compute similarity
similarity = tversky_similarity(icm, shrink=200, alpha=0.1, beta=1)
similarity = similarity.tocsr()
print(time.time() - start)
urm = datareader.get_urm()
print('Computing eurm...')
start = time.time()
# Compute eurm
eurm_nlp = dot_product(urm[test_playlists, :], similarity, k=500)
eurm_nlp = eurm_nlp.tocsr()
# sparse.save_npz(ROOT_DIR + '/data/eurm_nlp_weighted_offline.npz', eurm_nlp)
evaluator.evaluate(eurm_to_recommendation_list(eurm_nlp),
name='nlp_enriched')
def __init__(self,
matrices_names,
matrices_array,
dr,
cat,
start,
end,
n_calls=1000,
n_random_starts=0.1,
n_points=50,
step=0.001,
verbose=True):
self.target_metric = 'ndcg'
self.best_score = 0
self.best_params = 0
self.norm = norm_max_row
self.verbose = verbose
self.n_cpu = int(multiprocessing.cpu_count() / 10)
if self.n_cpu == 0:
self.n_cpu = 1
# Do not edit
self.start = start
self.end = end
self.cat = cat
self.global_counter = 0
self.start_index = (cat - 1) * 1000
self.end_index = cat * 1000
self.matrices_array = list()
self.matrices_names = matrices_names
self.n_calls = n_calls
self.global_counter = 0
self.x0 = None
self.y0 = None
self.n_random_starts = int(n_calls * n_random_starts)
self.n_points = n_points
self.step = step
# memory_on_disk= False
self.memory_on_notebook = True
self.dr = dr
self.ev = Evaluator(self.dr)
for matrix in matrices_array:
self.matrices_array.append(
self.norm(
eurm_remove_seed(
matrix,
datareader=dr)[self.start_index:self.end_index]))
del self.dr, matrices_array
def classification():
"""
Classification.
"""
# create data
X, y = DataCreator().make_classification(sklearn=True, n_classes=2)
# train kNN classifier
# clf = kNN(n_neighbors=3)
# clf.fit(X, y)
# train SVM classifier
# clf = SVM(kernel="polynomial", C=1.0, p=2, s=5.0)
# y[np.where(y == 0)] = -1
# clf.fit(X, y)
# train logistic regression classifier
# clf = LogisticRegression(poly=False)
# clf.fit(X, y, batch_size=X.shape[0])
# train one-vs-one logistic regression classifier
# clf = LogRegOneVsOne(poly=True)
# clf.fit(X, y)
# train a neural network
# clf = NeuralNetwork(layers=[10, 5])
# clf.fit(X, y)
# train tensorflow mlp
clf = MLP()
clf.fit(X, y)
# train Fisher's linear discriminant
# clf = LDA(n_dims=1)
# y[np.where(y == 0)] = -1
# clf.fit(X, y)
# Expectation Maximization
# em = EM()
# em.fit(X, n_comp=3, n_iter=30)
# plot boundary
BoundaryPlotter(X, y).plot_boundary(clf, step_size=0.005)
# evaluation
evaluator = Evaluator()
acc = evaluator.accuracy(clf, X, y)
print("Accuracy: {} %".format(acc))
evaluator.conf_mat(clf, X, y)
def prova():
dr = Datareader(mode='offline', only_load=True)
print(dr.get_artist_to_tracks_dict())
exit()
dr = Datareader(mode='offline', only_load=True, verbose=False)
test_playlists = dr.get_test_pids()
stopwords = STOP_WORDS
token_weights = np.array(TOKEN_WEIGHTS)
nlp = NLP(mode='playlists', datareader=dr, stopwords=STOP_WORDS)
s = nlp.get_ucm()
print(s.shape)
evaluator = Evaluator(dr)
ucm = nlp.get_ucm()
sim = sparse.load_npz(ROOT_DIR + '/data/cf_user_similarity.npz')
print('Computing dot...')
ucm = dot_product(sim, ucm, k=200)
print('NNZ', ucm.nnz)
exit()
urm = dr.get_urm()
# ucm = ucm.astype(np.float64)
# inplace_csr_column_scale(ucm, token_weights)
print('Computing similarity...')
start = time.time()
# Compute similarity
similarity = tversky_similarity(ucm, shrink=200, alpha=0.1, beta=1)
similarity = similarity.tocsr()
print(time.time() - start)
print('Computing eurm...')
start = time.time()
# Compute eurm
eurm_nlp = dot_product(similarity, urm, k=500)
eurm_nlp = eurm_nlp.tocsr()
eurm_nlp = eurm_nlp[test_playlists, :]
#sparse.save_npz(ROOT_DIR + '/data/eurm_nlp_weighted_offline.npz', eurm_nlp)
evaluator.evaluate(eurm_to_recommendation_list(eurm_nlp),
name='nlp_enriched')
def _load_evaluator(self):
print('loading evaluator...')
from utils.evaluator import Evaluator
self.evaluator = Evaluator()
print('evaluator load finished!')
def configuration(cls,
plm=None,
method='lgesql',
table_path='data/tables.json',
tables='data/tables.bin',
db_dir='data/database'):
cls.plm, cls.method = plm, method
cls.grammar = ASDLGrammar.from_filepath(GRAMMAR_FILEPATH)
cls.trans = TransitionSystem.get_class_by_lang('sql')(cls.grammar)
cls.tables = pickle.load(open(tables,
'rb')) if type(tables) == str else tables
cls.evaluator = Evaluator(cls.trans, table_path, db_dir)
if plm is None:
cls.word2vec = Word2vecUtils()
cls.tokenizer = lambda x: x
cls.word_vocab = Vocab(
padding=True,
unk=True,
boundary=True,
default=UNK,
filepath='./pretrained_models/glove.42b.300d/vocab.txt',
specials=SCHEMA_TYPES) # word vocab for glove.42B.300d
else:
cls.tokenizer = AutoTokenizer.from_pretrained(
os.path.join('./pretrained_models', plm))
cls.word_vocab = cls.tokenizer.get_vocab()
cls.relation_vocab = Vocab(padding=False,
unk=False,
boundary=False,
iterable=RELATIONS,
default=None)
cls.graph_factory = GraphFactory(cls.method, cls.relation_vocab)
def evaluate_from_pred_dict(predictions,
refvg_split,
analyze_subset=True,
exp_name_in_summary=None,
save_result_to_path=None,
update_predictions=False,
verbose=False):
evaluator = Evaluator(refvg_split=refvg_split,
analyze_subset=analyze_subset)
mb = list()
for img_id, im_preds in predictions.items():
pred_mask_tag = None
pred_boxes_tag = None
correct_tag = None
for task_id, task_pred in im_preds.items():
if 'pred_mask' in task_pred:
pred_mask_tag = 'pred_mask'
if 'pred_boxlist' in task_pred:
pred_boxes_tag = 'pred_boxlist'
if 'correct' in task_pred:
correct_tag = 'correct'
break
if len(mb) == 0:
if pred_mask_tag is not None:
mb.append('mask')
if pred_boxes_tag is not None:
mb.append('box')
im, ib = evaluator.eval_single_img(img_id=img_id,
im_pred_dict=im_preds,
pred_mask_tag=pred_mask_tag,
pred_boxes_tag=pred_boxes_tag,
correct_tag=correct_tag,
verbose=verbose)
if update_predictions:
for task_id, task_pred in im_preds.items():
if task_id in im:
task_pred['mask_iou'] = im[task_id]
if task_id in ib:
task_pred['box_iou'] = ib[task_id]
evaluator.analyze_stats(mask_box=mb,
exp_name_in_summary=exp_name_in_summary,
save_result_to_path=save_result_to_path)
return predictions
def main(args):
sys.stdout = Logger(args.log_dir)
train_loader, test_loader = get_data(args)
model = BaseModel(args)
evaluator = Evaluator(model=model, data_loader=test_loader)
best_acc = evaluator.evaluate()
accuracies = [best_acc]
losses = []
for e in range(1, args.epochs + 1):
epoch_loss = 0
print("Epoch", e)
for data in tqdm(train_loader):
model.set_input(data)
model.optimize_parameters()
epoch_loss += model.get_loss()
print("Epoch finished with loss", epoch_loss)
losses.append(epoch_loss)
if e % args.eval_step == 0:
acc = evaluator.evaluate()
accuracies.append(acc)
best_acc = max(acc, best_acc)
print("[Epoch {}] Accuracy:{:.2f}, Best Accuracy:{:.2f}".format(
e, acc, best_acc))
if e % args.save_step == 0:
model.save_model(e)
model.update_lr()
plt.figure()
plt.plot(range(len(losses)), losses)
plt.xlabel('Epochs')
plt.ylabel('Training Loss')
plt.savefig(os.path.join(args.exp_dir, 'losses.png'))
plt.figure()
plt.plot(range(len(accuracies)), accuracies)
plt.xlabel('Epochs')
plt.ylabel('Test Accuracy')
plt.savefig(os.path.join(args.exp_dir, 'accuracies.png'))
def polyfit_upperbound(dataset, degree):
evaluator = Evaluator(dataset, '/tmp', degree)
print('Predicting with upperbound...')
for i, anno in enumerate(progressbar(dataset.annotations)):
label = anno['label']
pred = np.zeros((label.shape[0], 1 + 2 + degree + 1))
pred[:, :3] = label[:, :3]
for j, lane in enumerate(label):
if lane[0] == 0:
continue
xy = lane[3:]
x = xy[:(len(xy) // 2)]
y = xy[(len(xy) // 2):]
ind = x > 0
pred[j, -(degree + 1):] = np.polyfit(y[ind], x[ind], degree)
evaluator.add_prediction([i], pred, 0.0005) # 0.0005 = dummy runtime
_, result = evaluator.eval(label='upperbound', only_metrics=True)
return result
def __init__(self, args):
self.args = args
# 初始化tensorboard summary
self.summary = TensorboardSummary(directory=args.save_path)
self.writer = self.summary.create_summary()
# 初始化dataloader
kwargs = {'num_workers': args.workers, 'pin_memory': True}
self.train_dataset = Apolloscapes('train_dataset.csv', '/home/aistudio/data/data1919/Image_Data', '/home/aistudio/data/data1919/Gray_Label',
args.crop_size, type='train')
self.dataloader = DataLoader(self.train_dataset, batch_size=args.batch_size, shuffle=True, drop_last=True, **kwargs)
self.val_dataset = Apolloscapes('val_dataset.csv', '/home/aistudio/data/data1919/Image_Data', '/home/aistudio/data/data1919/Gray_Label',
args.crop_size, type='val')
self.val_loader = DataLoader(self.val_dataset, batch_size=args.batch_size, shuffle=False, drop_last=False, **kwargs)
# 初始化model
self.model = DeeplabV3Plus(backbone=args.backbone,
output_stride=args.out_stride,
batch_norm=args.batch_norm,
num_classes=args.num_classes,
pretrain=True)
# 初始化优化器
self.optimizer = torch.optim.SGD(self.model.parameters(),
momentum=args.momentum,
nesterov=args.nesterov,
weight_decay=args.weight_decay,
lr=args.lr)
# 定义损失函数
self.loss = CELoss(num_class=args.num_classes, cuda=args.cuda)
# 定义验证器
self.evaluator = Evaluator(args.num_classes)
# 定义学习率
self.scheduler = LR_Scheduler('poly', args.lr, args.epochs, len(self.dataloader))
# 使用cuda
if args.cuda:
self.model = self.model.cuda(device=args.gpus[0])
self.model = torch.nn.DataParallel(self.model, device_ids=args.gpus)
def __init__(self,
icm,
urm,
binary=False,
verbose=True,
mode='offline',
datareader=None,
verbose_evaluation=True):
assert (mode in ('offline', 'online'))
if binary: urm.data = np.ones(urm.data.shape[0])
self.urm = urm
self.m_ic = pre.bm25_col(icm.copy()).tocsr()
self.m_ci = pre.bm25_col(icm.T.copy()).tocsr()
self.binary = binary
self.verbose = verbose
self.verbose_ev = verbose_evaluation
self.dr = datareader
self.mode = mode
if mode == 'offline':
self.ev = Evaluator(self.dr)
def hseq_eval():
with open(FLAGS.config, 'r') as f:
test_config = yaml.load(f, Loader=yaml.FullLoader)
# Configure dataset
hseq_utils = HSeqUtils(test_config['hseq'])
prepare_reg_feat(
hseq_utils,
os.path.join(test_config['eval']['reg_model'], 'model.ckpt-550000'))
# Configure evaluation
evaluator = Evaluator(test_config['eval'])
# Construct inference networks.
output_tensors = inference(test_config['network'])
# Create the initializier.
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
with tf.compat.v1.Session(config=config) as sess:
# Restore pre-trained model.
recoverer(
sess,
os.path.join(test_config['eval']['loc_model'],
'model.ckpt-400000'))
producer_queue = Queue(maxsize=18)
consumer_queue = Queue()
producer0 = Thread(target=loader,
args=(hseq_utils, test_config['network']['ori_est'],
test_config['network']['dense_desc'],
producer_queue))
producer0.daemon = True
producer0.start()
producer1 = Thread(target=extractor,
args=(producer_queue, sess, output_tensors,
test_config['network'], consumer_queue))
producer1.daemon = True
producer1.start()
consumer = Thread(target=matcher,
args=(consumer_queue, sess, evaluator))
consumer.daemon = True
consumer.start()
producer0.join()
producer1.join()
consumer.join()
evaluator.stats['all_avg_recall'] /= max(hseq_utils.seq_num, 1)
evaluator.stats['i_avg_recall'] /= max(hseq_utils.seq_i_num, 1)
evaluator.stats['v_avg_recall'] /= max(hseq_utils.seq_v_num, 1)
print(evaluator.stats)
def evaluate(self, category, train=True):
print "Evaluting discriminator pipeline for category", category
evaluated_train = Evaluator("sigmoid_normal3", "ade20k")
evaluated_val = Evaluator("sigmoid_normal3", "ade20k_val")
im_list_train = evaluated_train.get_im_list_by_category(category)
im_list_val = evaluated_val.get_im_list_by_category(category)
print "Train instances:", len(im_list_train)
print "Val instances:", len(im_list_val)
if train:
print "Training..."
self.train(category, im_list_train)
print "Training Done"
else:
print "Skipping training"
print "Running on validation data"
output_fn = "{}-ade20k_val.txt".format(category)
fn_val = self.run(category, im_list_val, output_fn=output_fn)
plot.plot(fn_val, evaluated_val, category, "ade20k_val")
print "Running on training data"
output_fn = "{}-ade20k.txt".format(category)
fn_train = self.run(category, im_list_train, output_fn=output_fn)
plot.plot(fn_train, evaluated_train, category, "ade20k")
plot.vis(fn_val, evaluated_val, category, "ade20k_val",
self.datasource_val)
def __init__(self,
urm,
pop=None,
binary=False,
verbose=True,
mode='offline',
datareader=None,
verbose_evaluation=True):
assert (mode in ('offline', 'online'))
if binary: urm.data = np.ones(urm.data.shape[0])
if pop is None: self.pop = urm.sum(axis=0).A1
else: self.pop = pop
self.urm = urm
self.m_ui = pre.bm25_row(urm.copy()).tocsr()
self.m_iu = pre.bm25_row(urm.T.copy()).tocsr()
self.binary = binary
self.verbose = verbose
self.verbose_ev = verbose_evaluation
self.dr = datareader
self.mode = mode
if mode == 'offline':
self.ev = Evaluator(self.dr)
def grid_tailboost():
datareader = Datareader(mode='offline', only_load=True, verbose=False)
ev = Evaluator(datareader)
# LOAD AND COMBINE
eurm_lele = sparse.load_npz(
ROOT_DIR +
'/data/lele/ensembled_CLUSTERARTISTScat4-5-6-8-10_offline.npz')
eurm_std = sparse.load_npz(ROOT_DIR +
'/data/lele/ensembled_SUBCREATIVA_offline.npz')
eurm_ens = combine_two_eurms(eurm_lele,
eurm_std,
cat_first=[4, 5, 6, 8, 10])
sim = sparse.load_npz(ROOT_DIR + '/data/sim_offline.npz')
# TAILBOOST
for lt in [2, 3, 5, 6, 10]:
for k in [20, 50, 80, 100, 150]:
for g in [0.005, 0.01, 0.02, 0.05]:
tb = TailBoost(similarity=sim,
eurm=eurm_ens,
datareader=datareader,
norm=norm_l2_row)
eurm_ens = tb.boost_eurm(categories=[9, 7, 6, 5],
last_tracks=[lt, lt, lt, lt],
k=[k, k, k, k],
gamma=[g, g, g, g])
rec_list = eurm_to_recommendation_list(eurm_ens,
datareader=datareader)
print(
'--------------------------------------------------------------------------'
)
print('LT =', lt)
print('K =', k)
print('G =', g)
ev.evaluate(rec_list, name='tb', save=False)
def __init__(self, settings: dict, settings_to_log: list):
self.settings = settings
self.settings_to_log = settings_to_log
self.threshold = self.settings['threshold']
self.start_epoch = self.settings['start_epoch']
self.dataset = self.settings['dataset']
self.batch_size = self.settings['batch_size']
self.workers = self.settings['workers']
self.cuda = self.settings['cuda']
self.fp16 = self.settings['fp16']
self.epochs = self.settings['epochs']
self.ignore_index = self.settings['ignore_index']
self.loss_reduction = self.settings['loss_reduction']
# -------------------- Define Data loader ------------------------------
self.loaders, self.nclass, self.plotter = make_data_loader(settings)
self.train_loader, self.val_loader, self.test_loader = [self.loaders[key] for key in ['train', 'val', 'test']]
# -------------------- Define model ------------------------------------
self.model = get_model(self.settings)
# -------------------- Define optimizer and its options ----------------
self.optimizer = define_optimizer(self.model, self.settings['optimizer'], self.settings['optimizer_params'])
if self.settings['lr_scheduler']:
self.lr_scheduler = LRScheduler(self.settings['lr_scheduler'], self.optimizer, self.batch_size)
# -------------------- Define loss -------------------------------------
input_size = (self.batch_size, self.nclass, *self.settings['target_size'])
self.criterion = CustomLoss(input_size=input_size, ignore_index=self.ignore_index, reduction=self.loss_reduction)
self.evaluator = Evaluator(metrics=self.settings['metrics'], num_class=self.nclass, threshold=self.settings['threshold'])
self.logger = MainLogger(loggers=self.settings['loggers'], settings=settings, settings_to_log=settings_to_log)
if self.settings['resume']:
self.resume_checkpoint(self.settings['resume'])
self.metric_to_watch = 0.0
def __init__(self,
urm,
pop=None,
binary=False,
K1=1.2,
B=0.75,
verbose=True,
mode='offline',
datareader=None,
verbose_evaluation=True,
mode_t=False,
trick=False):
assert (mode in ('offline', 'online'))
if binary: urm.data = np.ones(urm.data.shape[0])
if pop is None: self.pop = urm.sum(axis=0).A1
else: self.pop = pop
self.dr = datareader
self.urm = urm
urm_aux = urm.copy()
ut.inplace_set_rows_zero(X=urm_aux,
target_rows=self.dr.get_test_pids()
) #don't learn from challange set
urm_aux.eliminate_zeros()
if mode_t: self.m_ui = urm_aux.copy().tocsr()
else: self.m_ui = pre.bm25_row(urm_aux.copy(), K1=K1, B=B).tocsr()
if mode_t: self.m_iu = urm_aux.T.copy().tocsr()
else: self.m_iu = pre.bm25_row(urm_aux.T.copy(), K1=K1, B=B).tocsr()
self.binary = binary
self.verbose = verbose
self.verbose_ev = verbose_evaluation
self.mode = mode
self.mode_t = mode_t
if trick:
self.urm = pre.bm25_row(
urm).tocsr() #high click, high ndcg, better no use
if mode == 'offline':
self.ev = Evaluator(self.dr)
def __init__(self,
urm,
pop=None,
binary=False,
verbose=True,
mode='offline',
datareader=None,
verbose_evaluation=True,
similarity='tversky'):
assert (mode in ('offline', 'online'))
if binary: urm.data = np.ones(urm.data.shape[0])
if pop is None:
self.pop = urm.sum(axis=1).A1 #####maybe better use followers
else:
self.pop = pop
self.urm = urm
self.binary = binary
self.verbose = verbose
self.verbose_ev = verbose_evaluation
self.dr = datareader
self.mode = mode
self.similarity = similarity
urm_aux = urm.copy()
ut.inplace_set_rows_zero(X=urm_aux,
target_rows=self.dr.get_test_pids()
) #don't learn from challange set
urm_aux.eliminate_zeros()
if self.similarity == 'p3alpha':
self.m_ui = pre.bm25_col(urm.copy()).tocsr()
else:
self.m_ui = urm.copy().tocsr()
if self.similarity == 'p3alpha':
self.m_iu = pre.bm25_col(urm_aux.T.copy()).tocsr()
else:
self.m_iu = urm_aux.T.copy().tocsr()
if mode == 'offline':
self.ev = Evaluator(self.dr)
def main(config: ConfigParser):
device = torch.device("cuda:0" if config["n_gpu"] > 0 else "cpu")
train_iters, test_iters, vocab = load_dataset(config, device)
print("Vocab size:", len(vocab))
print("prepare Evaluator...")
evaluator_path = config["evaluator"]["save_path"]
if evaluator_path is None or not Path(evaluator_path).exists():
evaluator = Evaluator.create(**config["evaluator"]["args"])
config["evaluator"]["save_path"] = (config.save_dir /
"evaluator").as_posix()
evaluator.save(config["evaluator"]["save_path"])
else:
evaluator = Evaluator.load(evaluator_path)
config.save()
model_F = config.initialize("arch_generator", module_arch,
vocab=vocab).to(device)
model_D = config.initialize("arch_discriminator", module_arch,
vocab=vocab).to(device)
opt_F = config.initialize("generator_optimizer", module_optim)
opt_D = config.initialize("discriminator_optimizer", module_optim)
opt_F.set_parameters(model_F.parameters())
opt_D.set_parameters(model_D.parameters())
train(config=config,
vocab=vocab,
model_F=model_F,
model_D=model_D,
opt_F=opt_F,
opt_D=opt_D,
train_iters=train_iters,
test_iters=test_iters,
evaluator=evaluator)
def new():
datareader = Datareader(mode='offline', only_load=True)
evaluator = Evaluator(datareader)
print('NLP...')
stopwords = STOP_WORDS
token_weights = np.array(TOKEN_WEIGHTS)
test_playlists = datareader.get_test_pids()
nlp = NLP(datareader=datareader, stopwords=[], mode='both')
print('Getting ucm and icm...')
ucm = nlp.get_ucm()
ucm = bm25_row(ucm)
icm = nlp.get_icm()
icm = bm25_row(icm)
icm_T = icm.T
#ucm = bm25_row(ucm)
#urm = datareader.get_urm()
print('Computing eurm...')
start = time.time()
eurm_nlp = dot_product(ucm[test_playlists, :], icm_T, k=500)
print(time.time() - start)
print('Converting to csr...')
eurm_nlp = eurm_nlp.tocsr()
print(eurm_nlp.shape)
#eurm_nlp = eurm_nlp[test_playlists:, :]
sparse.save_npz(ROOT_DIR + '/data/eurm_nlp_new_method_offline.npz',
eurm_nlp)
evaluator.evaluate(eurm_to_recommendation_list(eurm_nlp),
name='nlp_new_method',
show_plot=False)
def evaluate_from_pred_folder(pred_folder,
refvg_split,
analyze_subset=True,
exp_name_in_summary=None,
save_result_to_path=None,
verbose=False):
print(exp_name_in_summary)
evaluator = Evaluator(refvg_split=refvg_split,
analyze_subset=analyze_subset)
fnames = os.listdir(pred_folder)
fnames.sort()
img_preds = dict()
cur_img_id = ''
for fname in fnames:
if not fname.endswith('.png'):
continue
task_id = fname.split('.')[0]
img_id = task_id.split('__')[0]
pred_mask = png_to_pred_mask(os.path.join(pred_folder, fname))
if img_id == cur_img_id:
img_preds[task_id] = {'pred_mask': pred_mask}
else:
if len(img_preds) > 0:
# print(img_preds)
evaluator.eval_single_img(img_id=int(cur_img_id),
im_pred_dict=img_preds,
pred_mask_tag='pred_mask',
verbose=verbose)
cur_img_id = img_id
img_preds = dict()
img_preds[task_id] = {'pred_mask': pred_mask}
if len(img_preds) > 0:
evaluator.eval_single_img(img_id=int(cur_img_id),
im_pred_dict=img_preds,
pred_mask_tag='pred_mask',
verbose=verbose)
evaluator.analyze_stats(mask_box=['mask'],
exp_name_in_summary=exp_name_in_summary,
save_result_to_path=save_result_to_path)
return
def inference(
images_path: str,
sentences_path: str,
test_imgs_file_path: str,
batch_size: int,
load_model_path: str,
joint_space: int,
):
# Check for CUDA
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
dataset_test = FlickrDatasetValTest(images_path, sentences_path,
test_imgs_file_path)
test_loader = DataLoader(
dataset_test,
batch_size=batch_size,
num_workers=4,
collate_fn=collate_pad_batch,
pin_memory=True,
)
# Create the model
model = nn.DataParallel(ImageTextMatchingModel(joint_space)).to(device)
# Load model
model.load_state_dict(torch.load(load_model_path))
# Set model in evaluation mode
model.train(False)
# Create evaluator
evaluator = Evaluator(len(dataset_test), joint_space)
with torch.no_grad():
evaluator.reset_all_vars()
for images, sentences in tqdm(test_loader):
images, sentences = images.to(device), sentences.to(device)
embedded_images, embedded_sentences = model(images, sentences)
evaluator.update_embeddings(
embedded_images.cpu().numpy().copy(),
embedded_sentences.cpu().numpy().copy(),
)
print("=============================")
print(f"Image-text recall at 1, 5, 10: "
f"{evaluator.image2text_recall_at_k()} \n"
f"Text-image recall at 1, 5, 10: "
f"{evaluator.text2image_recall_at_k()}")
print("=============================")
def hseq_eval():
with open(FLAGS.config, 'r') as f:
test_config = yaml.load(f, Loader=yaml.FullLoader)
# Configure dataset
hseq_utils = HSeqUtils(test_config['hseq'])
# Configure evaluation
evaluator = Evaluator(test_config['eval'])
# Construct inference networks.
model = get_model('feat_model')(test_config['model_path'],
**(test_config['net']))
# Create the initializier.
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
producer_queue = Queue(maxsize=18)
consumer_queue = Queue()
producer0 = Thread(target=loader, args=(hseq_utils, producer_queue))
producer0.daemon = True
producer0.start()
producer1 = Thread(target=extractor,
args=(producer_queue, model, consumer_queue))
producer1.daemon = True
producer1.start()
consumer = Thread(target=matcher,
args=(consumer_queue, model.sess, evaluator,
test_config['eval']))
consumer.daemon = True
consumer.start()
producer0.join()
producer1.join()
consumer.join()
evaluator.print_stats('i_eval_stats')
evaluator.print_stats('v_eval_stats')
evaluator.print_stats('all_eval_stats')
import sys
from scipy import sparse
import numpy as np
import utils.pre_processing as pre
from utils.definitions import *
from utils.datareader import Datareader
from utils.evaluator import Evaluator
from utils.pre_processing import *
from utils.post_processing import *
dr = Datareader(mode='offline', only_load=True, verbose=False)
ev = Evaluator(dr)
urm = dr.get_urm(binary=True)
urm_csc = urm.tocsc(copy=True)
sim_nlp = sparse.load_npz(ROOT_DIR + '/data/sim_nlp_lele.npz')
for k in [1, 2, 3, 4, 5]:
eurm_top = dr.get_eurm_top_pop_filter_cat_1(sim_nlp, k, topk=500)
eurm_top = norm_l1_row(eurm_top)
eurm_nlp = sparse.load_npz(ROOT_DIR + '/data/nlp_fusion_tuned_offline.npz')
eurm_nlp = norm_l1_row(eurm_nlp)
for a in [0.05, 0.10, 0.15, 0.20]:
eurm = eurm_nlp * (1.0 - a) + eurm_top * a
rec_list = eurm_to_recommendation_list(eurm, datareader=dr)
ev.evaluate(rec_list, name='pop_first_k=' + str(k) + '_a=' + str(a))
import sys
from scipy import sparse
import numpy as np
import utils.pre_processing as pre
from utils.definitions import *
from utils.datareader import Datareader
from utils.evaluator import Evaluator
from utils.pre_processing import *
from utils.post_processing import *
dr = Datareader(mode='offline', only_load=True, verbose=False)
ev = Evaluator(dr)
urm = dr.get_urm(binary=True)
pos_matrix = dr.get_position_matrix(position_type='last')
rows = []
cols = []
data = []
for p in tqdm(range(pos_matrix.shape[0])):
start = pos_matrix.indptr[p]
end = pos_matrix.indptr[p + 1]
tracks = pos_matrix.indices[start:end]
positions = pos_matrix.indices[start:end]
for idx in range(len(tracks)):
if positions[idx] <= 250:
rows.append(p)
cols.append((tracks[idx] * positions[idx]) + tracks[idx])
