def validate(valid_loader, model):
model.eval()
recalls10 = []
mrrs10 = []
recalls20 = []
mrrs20 = []
recalls50 = []
mrrs50 = []
with torch.no_grad():
for seq, target, lens in tqdm(valid_loader,
total=len(valid_loader),
miniters=1000):
seq = seq.to(device)
target = target.to(device)
outputs = model(seq, lens)
logits = F.softmax(outputs, dim=1)
recall10, mrr10 = metric.evaluate(logits, target, k=10)
recall20, mrr20 = metric.evaluate(logits, target, k=20)
recall50, mrr50 = metric.evaluate(logits, target, k=50)
recalls10.append(recall10)
mrrs10.append(mrr10)
recalls20.append(recall20)
mrrs20.append(mrr20)
recalls50.append(recall50)
mrrs50.append(mrr50)
mean_recall10 = np.mean(recalls10)
mean_mrr10 = np.mean(mrrs10)
mean_recall20 = np.mean(recalls20)
mean_mrr20 = np.mean(mrrs20)
mean_recall50 = np.mean(recalls50)
mean_mrr50 = np.mean(mrrs50)
return mean_recall10, mean_mrr10, mean_recall20, mean_mrr20, mean_recall50, mean_mrr50
def evaluat(self):
if self.test_file2:
in_file = open(self.test_file2)
gold_data = [int(line.strip().split()[0]) for line in in_file]
in_file.close()
in_file = open(self.result_file2)
classifier_data = [
int(line.strip().split()[0])
for line in in_file.readlines()[1:]
]
in_file.close()
p, r, f, auc = evaluate(np.asarray(gold_data),
np.asarray(classifier_data))
return (p, r, f, auc)
else:
in_file = open(self.test_file)
gold_data = [int(line.strip().split()[0]) for line in in_file]
in_file.close()
in_file = open(self.result_file)
classifier_data = [
int(line.strip().split()[0])
for line in in_file.readlines()[1:]
]
in_file.close()
p, r, f, auc = evaluate(np.asarray(gold_data),
np.asarray(classifier_data))
return (p, r, f, auc)
def predict(config):
pprint.pprint(config)
if config.model == 'bs.ngram_beam_search':
model_func = bs.ngram_beam_search
elif config.model == 's2s':
model_func = s2s.query
else:
raise NotImplementedError()
source = load_data(config.test_data_source)
target = load_data(config.test_data_target)
predictions = []
ground_truth = []
index = 0
for truth, data in zip(target, source):
if index % 10000 == 0:
print(index)
pprint.pprint(metric.evaluate(ground_truth, predictions, config.k))
index += 1
data = data.split('|')
(context, pinyin) = data
context = "".join(context.split())
pinyin = pu.segment_with_hint(metric.normalize_text(pinyin))
context = "".join(context.split())
prediction = model_func(context.strip(), pinyin)
#print(prediction)
#print(truth)
predictions.append(prediction)
ground_truth.append(truth)
pprint.pprint(metric.evaluate(target, predictions, config.k))
def validate(self):
'''
validate by auc value
:return: auc
'''
y_, y_pred = self.predict(self.dataloader["val"])
rocprc, rocauc, best_th, best_f1 = evaluate(y_, y_pred)
return rocauc, best_th, best_f1
def test_type(self):
self.G.eval()
self.D.eval()
res_th = self.opt.threshold
save_dir = os.path.join(self.outf, self.model, self.dataset, "test",
str(self.opt.folder))
if not os.path.exists(save_dir):
os.makedirs(save_dir)
y_N, y_pred_N = self.predict(self.dataloader["test_N"], scale=False)
y_S, y_pred_S = self.predict(self.dataloader["test_S"], scale=False)
y_V, y_pred_V = self.predict(self.dataloader["test_V"], scale=False)
y_F, y_pred_F = self.predict(self.dataloader["test_F"], scale=False)
y_Q, y_pred_Q = self.predict(self.dataloader["test_Q"], scale=False)
over_all = np.concatenate(
[y_pred_N, y_pred_S, y_pred_V, y_pred_F, y_pred_Q])
over_all_gt = np.concatenate([y_N, y_S, y_V, y_F, y_Q])
min_score, max_score = np.min(over_all), np.max(over_all)
A_res = {"S": y_pred_S, "V": y_pred_V, "F": y_pred_F, "Q": y_pred_Q}
self.analysisRes(y_pred_N, A_res, min_score, max_score, res_th,
save_dir)
#save fig for Interpretable
# self.predictForRight(self.dataloader["test_N"], save_dir=os.path.join(save_dir, "N"))
self.predict_for_right(self.dataloader["test_S"],
min_score,
max_score,
res_th,
save_dir=os.path.join(save_dir, "S"))
self.predict_for_right(self.dataloader["test_V"],
min_score,
max_score,
res_th,
save_dir=os.path.join(save_dir, "V"))
self.predict_for_right(self.dataloader["test_F"],
min_score,
max_score,
res_th,
save_dir=os.path.join(save_dir, "F"))
self.predict_for_right(self.dataloader["test_Q"],
min_score,
max_score,
res_th,
save_dir=os.path.join(save_dir, "Q"))
aucprc, aucroc, best_th, best_f1 = evaluate(
over_all_gt, (over_all - min_score) / (max_score - min_score))
print("#############################")
print("######## Result ###########")
print("ap:{}".format(aucprc))
print("auc:{}".format(aucroc))
print("best th:{} --> best f1:{}".format(best_th, best_f1))
with open(os.path.join(save_dir, "res-record.txt"), 'w') as f:
f.write("auc_prc:{}\n".format(aucprc))
f.write("auc_roc:{}\n".format(aucroc))
f.write("best th:{} --> best f1:{}".format(best_th, best_f1))
def validate(valid_loader, model):
model.eval()
recalls = []
mrrs = []
with torch.no_grad():
for seq, target, lens in tqdm(valid_loader):
seq = seq.to(device)
target = target.to(device)
outputs = model(seq, lens)
logits = F.softmax(outputs, dim=1)
recall, mrr = metric.evaluate(logits, target, k=args.topk)
recalls.append(recall)
mrrs.append(mrr)
mean_recall = np.mean(recalls)
mean_mrr = np.mean(mrrs)
return mean_recall, mean_mrr
def validate(valid_loader, model):
model.eval()
recalls = []
mrrs = []
with torch.no_grad():
for seq, target, lens in (valid_loader):
seq = seq.to(device)
target = target.to(device)
lens = torch.tensor(lens).to(device)
scores = model.full_sort_predict(seq, lens)
logits = F.softmax(scores, dim=1)
recall, mrr = metric.evaluate(logits, target, k=topk)
recalls.append(recall)
mrrs.append(mrr)
mean_recall = np.mean(recalls)
mean_mrr = np.mean(mrrs)
return mean_recall, mean_mrr
def test_score_dist(self):
self.load()
self.G.eval()
self.D.eval()
y_, y_pred = self.predict(self.dataloader["test"])
f = plt.figure()
ax = f.add_subplot(111)
X1 = []
X2 = []
for gt, sc in zip(y_, y_pred):
if gt == 0:
X1.append(sc)
else:
X2.append(sc)
_, bins, _ = ax.hist(X1,
bins=55,
range=[0, 1],
density=True,
alpha=0.3,
color='r',
label="walk")
_ = ax.hist(X2,
bins=bins,
alpha=0.3,
density=True,
color='b',
label="others")
ax.set_yticks([])
ax.set_xticks(np.arange(0, 1.2, 0.2))
ax.legend()
f.savefig(os.path.join("./output", "dist.pdf"))
auc_prc, roc_auc, best_threshold, best_f1 = evaluate(y_, y_pred)
print("ap:{}".format(auc_prc))
print("auc:{}".format(roc_auc))
print("best threshold:{} ==> F1:{}".format(best_threshold, best_f1))
def learnSBPR_FPMC(self,
log_path_txt,
tr_data,
te_data=None,
n_epoch=10,
neg_batch_size=10,
eval_per_epoch=True,
Ks=[10, 20, 50],
Ks_auc=[100]):
best_result_recall = [0] * len(Ks)
best_result_mrr = [0] * len(Ks)
best_result_ndcg = [0] * len(Ks)
best_result_auc = [0] * len(Ks_auc)
best_epoch_auc = [0] * len(Ks_auc)
best_epoch_ndcg = [0] * len(Ks)
best_epoch_recall = [0] * len(Ks)
best_epoch_mrr = [0] * len(Ks)
for epoch in range(n_epoch):
self.learn_epoch(tr_data, neg_batch_size=neg_batch_size)
if eval_per_epoch == True:
scores, ground_truth = self.predict(tr_data)
recall, mrr, ndcg, auc = evaluate(scores=scores,
ground_truth=ground_truth,
Ks=Ks,
Ks_auc=Ks_auc)
# print train data result
file_write(log_path_txt,
f'Current Train Data Result Epoch {epoch}:')
print_result(log_path_txt, 'Recall', recall, Ks)
print_result(log_path_txt, 'MRR', mrr, Ks)
print_result(log_path_txt, 'NDCG', ndcg, Ks)
print_auc_result(log_path_txt, 'AUC', auc, Ks_auc)
# start test
scores, ground_truth = self.predict(te_data)
recall, mrr, ndcg, auc = evaluate(scores=scores,
ground_truth=ground_truth,
Ks=Ks,
Ks_auc=Ks_auc)
for i, topk in enumerate(Ks):
if recall[i] >= best_result_recall[i]:
best_result_recall[i] = recall[i]
best_epoch_recall[i] = epoch
if mrr[i] >= best_result_mrr[i]:
best_result_mrr[i] = mrr[i]
best_epoch_mrr[i] = epoch
if ndcg[i] >= best_result_ndcg[i]:
best_result_ndcg[i] = ndcg[i]
best_epoch_ndcg[i] = epoch
for i, k in enumerate(Ks_auc):
if auc[i] >= best_result_auc[i]:
best_result_auc[i] = auc[i]
best_epoch_auc[i] = epoch
# print test result
cprint(log_path_txt, f'Current Test Data Result Epoch {epoch}:')
print_result(log_path_txt, 'Recall', recall, Ks)
print_result(log_path_txt, 'MRR', mrr, Ks)
print_result(log_path_txt, 'NDCG', ndcg, Ks)
print_auc_result(log_path_txt, 'AUC', auc, Ks_auc)
file_write(log_path_txt, 'Best Result: ')
file_write(log_path_txt, f'best_epoch_recall: {best_epoch_recall}')
file_write(log_path_txt, f'best_epoch_mrr: {best_epoch_mrr}')
file_write(log_path_txt, f'best_epoch_ndcg: {best_epoch_ndcg}')
file_write(log_path_txt, f'best_epoch_auc: {best_epoch_auc}')
print_result(log_path_txt, 'Recall', best_result_recall, Ks)
print_result(log_path_txt, 'MRR', best_result_mrr, Ks)
print_result(log_path_txt, 'NDCG', best_result_ndcg, Ks)
print_auc_result(log_path_txt, 'AUC', best_result_auc, Ks_auc)
return None
def train_test():
'''
load config
'''
rel_level = 2
max_q_len_consider = 10
max_d_len_consider = 1000
if args.config != None:
model_config = json.load(open(args.config))
print('model config: {}'.format(model_config))
'''
load word vector
'''
w2v_file = os.path.join(args.data_dir, 'w2v')
vocab_file = os.path.join(args.data_dir, 'vocab')
print('loading word vector ...')
wv = WordVector(filepath=w2v_file)
vocab = Vocab(filepath=vocab_file, file_format=args.format)
print('vocab size: {}, word vector dim: {}'.format(wv.vocab_size, wv.dim))
if not args.tfrecord:
'''
load data (placeholder)
'''
train_file = os.path.join(args.data_dir,
'train.prep.{}'.format(args.paradigm))
test_file = os.path.join(args.data_dir,
'test.prep.{}'.format(args.paradigm))
test_file_judge = os.path.join(args.data_dir, 'test.prep.pointwise')
doc_file = os.path.join(args.data_dir, 'docs.prep')
if args.format == 'ir':
query_file = os.path.join(args.data_dir, 'query.prep')
print('loading query doc content ...')
doc_raw = load_prep_file(doc_file, file_format=args.format)
if args.format == 'ir':
query_raw = load_prep_file(query_file, file_format=args.format)
else:
query_raw = doc_raw
print('truncate long document')
d_long_count = 0
avg_doc_len, avg_truncate_doc_len = 0, 0
truncate_len = max(max_q_len_consider, max_d_len_consider)
for d in doc_raw:
avg_doc_len += len(doc_raw[d])
if len(doc_raw[d]) > truncate_len:
d_long_count += 1
doc_raw[d] = doc_raw[d][:truncate_len]
avg_truncate_doc_len += truncate_len
else:
avg_truncate_doc_len += len(doc_raw[d])
avg_doc_len = avg_doc_len / len(doc_raw)
avg_truncate_doc_len = avg_truncate_doc_len / len(doc_raw)
print(
'total doc: {}, long doc: {}, average len: {}, average truncate len: {}'
.format(len(doc_raw), d_long_count, avg_doc_len,
avg_truncate_doc_len))
max_q_len = min(max_q_len_consider,
max([len(query_raw[q]) for q in query_raw]))
max_d_len = min(max_d_len_consider,
max([len(doc_raw[d]) for d in doc_raw]))
print('data assemble with max_q_len: {}, max_d_len: {} ...'.format(
max_q_len, max_d_len))
def relevance_mapper(r):
if r < 0:
return 0
if r >= rel_level:
return rel_level - 1
return r
train_X, train_y, batcher = data_assemble(
train_file,
query_raw,
doc_raw,
max_q_len,
max_d_len,
relevance_mapper=relevance_mapper)
'''
doc_len_list = []
for q_x in train_X:
for d in q_x['qd_size']:
doc_len_list.append(d[1])
doc_len_list = np.array(doc_len_list, dtype=np.int32)
doc_len_list = [min(max_jump_offset ** 2 / d, max_jump_offset) for d in doc_len_list]
plt.hist(doc_len_list, bins=max_jump_offset)
plt.xlim(xmin=0, xmax=max_jump_offset)
plt.xlabel('preserve number')
plt.ylabel('number')
plt.show()
'''
test_X, test_y, _ = data_assemble(test_file,
query_raw,
doc_raw,
max_q_len,
max_d_len,
relevance_mapper=relevance_mapper)
if args.paradigm == 'pairwise':
test_X_judge, test_y_judge, _ = data_assemble(
test_file_judge,
query_raw,
doc_raw,
max_q_len,
max_d_len,
relevance_mapper=relevance_mapper)
else:
text_X_judge, test_y_judge = test_X, test_y
print('number of training samples: {}'.format(
sum([len(x['query']) for x in train_X])))
'''
load judge file
'''
test_qd_judge = load_judge_file(test_file_judge,
file_format=args.format,
reverse=args.reverse)
for q in test_qd_judge:
for d in test_qd_judge[q]:
test_qd_judge[q][d] = relevance_mapper(test_qd_judge[q][d])
else:
'''
load data (tfrecord)
'''
max_q_len = max_q_len_consider
max_d_len = max_d_len_consider
batcher = None
'''
train and test the model
'''
model_config_ = {
'max_q_len': max_q_len,
'max_d_len': max_d_len,
'max_jump_step': 100,
'word_vector': wv.get_vectors(normalize=True),
'oov_word_vector': None,
'vocab': vocab,
'word_vector_trainable': False,
'use_pad_word': True,
'interaction': 'dot',
'glimpse': 'all_next_hard',
'glimpse_fix_size': 10,
'min_density': -1,
'use_ratio': False,
'min_jump_offset': 3,
'jump': 'min_density_hard',
'represent': 'interaction_cnn_hard',
'separate': False,
'aggregate': 'max',
'rnn_size': 16,
'max_jump_offset': 50,
'max_jump_offset2': max_q_len,
'rel_level': rel_level,
'loss_func': 'classification',
'keep_prob': 1.0,
'paradigm': args.paradigm,
'learning_rate': 0.0002,
'random_seed': SEED,
'n_epochs': 30,
'batch_size': 256,
'batch_num': 400,
'batcher': batcher,
'verbose': 1,
'save_epochs': 1,
'reuse_model': args.reuse_model_path,
'save_model': args.save_model_path,
'summary_path': args.tf_summary_path,
'tfrecord': args.tfrecord,
}
if args.config != None:
model_config_.update(model_config)
rri = RRI(**model_config_)
if not args.tfrecord:
#train_X, train_y, test_X, test_y = train_X[:2560], train_y[:2560], test_X[:2560], test_y[:2560]
print('train query: {}, test query: {}'.format(len(train_X),
len(test_X)))
for e in rri.fit_iterable(train_X, train_y):
start = time.time()
loss, acc = rri.test(test_X, test_y)
if args.format == 'ir':
ranks, _ = rri.decision_function(test_X_judge)
scores = evaluate(ranks, test_qd_judge, metric=ndcg, top_k=20)
avg_score = np.mean(list(scores.values()))
elif args.format == 'text':
avg_score = None
print('\t{:>7}:{:>5.3f}:{:>5.3f}:{:>5.3f}'.format(
'test_{:>3.1f}'.format((time.time() - start) / 60), loss, acc,
avg_score),
end='',
flush=True)
else:
for e in rri.fit_iterable_tfrecord(
'data/bing/test.prep.pairwise.tfrecord-???-of-???'):
print(e)
input()
R_test5 = np.array(dataset5[1])
#print(R_train1,R_test1)
K = 19 # number of latent features
# initialize P and Q with random values
P = np.random.rand(R_train1.shape[0], K)
Q = np.random.rand(R_train1.shape[1], K)
print(R_test1.shape[0],R_test1.shape[1])
print ('***************************************')
print ('NMF1 started!!')
print ('***************************************')
P1, Q1 = nmf1.matrix_factorization(R_train1, P, Q, K)
R_pred1 = np.dot(P1, Q1.T)
#print(R_pred1)
precision1, recall1, fvalue1, rmse1 = evaluate(R_test1, R_pred1)
P2, Q2 = nmf1.matrix_factorization(R_train2, P, Q, K)
R_pred2 = np.dot(P2, Q2.T)
precision2, recall2, fvalue2, rmse2 = evaluate(R_test2, R_pred2)
P3, Q3 = nmf1.matrix_factorization(R_train3, P, Q, K)
R_pred3 = np.dot(P3, Q3.T)
precision3, recall3, fvalue3, rmse3 = evaluate(R_test3, R_pred3)
P4, Q4 = nmf1.matrix_factorization(R_train4, P, Q, K)
R_pred4 = np.dot(P4, Q4.T)
precision4, recall4, fvalue4, rmse4 = evaluate(R_test4, R_pred4)
P5, Q5 = nmf1.matrix_factorization(R_train5, P, Q, K)
R_pred5 = np.dot(P5, Q5.T)
precision5, recall5, fvalue5, rmse5 = evaluate(R_test5, R_pred5)
# 5 cross validation
print ('**************precision***************')
def main():
import csv
import time
from scipy.stats import rankdata
from dataloader import Data
# ds = Data(root_dir="./data/ta_feng/")
# # train, test, valid = ds.get_data(neg=0)
# train, test, valid = ds.get_data(neg=2)
# # Load network
# n_usr = len(ds.usrset)
# n_itm = len(ds.itemset)
# k = 32
# d=k
# h=2
# gamma=[1,1,1,1]
# alpha=0.0
# lr=0.0001
# momentum=0
# weight_decay=0.01
# epochs=21
# neg=2
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# model = ABFM(n_usr, n_itm, k).to(device=device)
# model = FABFM(n_usr, n_itm, k, d, h, gamma, alpha).to(device=device)
with open("./path_for_metric") as f:
paths = [row[0] for row in csv.reader(f, delimiter="\n")]
# paths = paths[35:36]
paths = paths[41:]
cnt = 0
for i, path in enumerate(paths):
print(f"{path:-^60}")
# choice= "train"
choice = "test"
metric = True
name = get_name(path)
model, train, test, l_train, l_test, n_usr, n_itm = load_model(
path, device, model_name=name, choice=choice)
if metric:
from metric import evaluate
if choice == "train":
result, r_result, diversity, auc = evaluate(train, l_train, model, n_usr, n_itm, \
device, C=100, beta=5, n_rank=10, fAUC=True)
print(f"Data : {choice}\n" \
f"HLU : {result}\n" \
f"R@10 : {r_result}\n" \
f"Diversity : {diversity}\n" \
f"AUC : {auc}\n" \
"{:-^60}".format(""), flush=True)
elif choice == "test":
result, r_result, diversity, auc = evaluate(test, l_test, model, n_usr, n_itm, \
device, C=100, beta=5, n_rank=10, fAUC=True)
print(f"Data : {choice}\n" \
f"HLU : {result}\n" \
f"R@10 : {r_result}\n" \
f"Diversity : {diversity}\n" \
f"AUC : {auc}\n" \
"{:-^60}".format(""), flush=True)
else:
criterion = nn.BCEWithLogitsLoss()
if choice == "train":
data = train
elif choice == "test":
data = test
cnt = 0
for i in data:
if cnt == 50: break
with torch.no_grad():
x, label = i[0].to(device), i[1].to(device)
print(f"\nLabel : {label.item()}")
# print(f"fm output : {model.fm(x, debug=True).item():>8.5f}")
print(f"fm output : {model.fm(x, debug=True).item()}")
print(
f"sig out : {torch.sigmoid(model(x, label, pmi=1)).item()}"
)
if label == -1:
# print(f"Loss : {criterion(torch.sigmoid(model(x, label, pmi=1)), label+1).item():>8.5f}")
print(
f"Loss : {criterion(torch.sigmoid(model(x, label, pmi=1)), label+1).item()}"
)
else:
# print(f"Loss : {criterion(torch.sigmoid(model(x, label, pmi=1)), label).item():>8.5f}")
print(
f"Loss : {criterion(torch.sigmoid(model(x, label, pmi=1)), label).item()}"
)
# target_idx = (x[n:n+m]==1).nonzero()
# y = model.rank_list(x).cpu().numpy()
# rank = rankdata(-y, method="min")[target_idx]
# print("Rank :", rank)
cnt += 1
R_train5 = np.array(dataset5[0])
R_test5 = np.array(dataset5[1])
K = 19 # number of latent features
# initialize P and Q with random values
P = np.random.rand(R_train1.shape[0], K)
Q = np.random.rand(R_train1.shape[1], K)
print '***************************************'
print 'NMF1 started!!'
print '***************************************'
P1, Q1 = nmf1.matrix_factorization(R_train1, P, Q, K)
R_pred1 = np.dot(P1, Q1.T)
precision1, recall1, fvalue1, rmse1 = evaluate(R_test1, R_pred1)
P2, Q2 = nmf1.matrix_factorization(R_train2, P, Q, K)
R_pred2 = np.dot(P2, Q2.T)
precision2, recall2, fvalue2, rmse2 = evaluate(R_test2, R_pred2)
P3, Q3 = nmf1.matrix_factorization(R_train3, P, Q, K)
R_pred3 = np.dot(P3, Q3.T)
precision3, recall3, fvalue3, rmse3 = evaluate(R_test3, R_pred3)
P4, Q4 = nmf1.matrix_factorization(R_train4, P, Q, K)
R_pred4 = np.dot(P4, Q4.T)
precision4, recall4, fvalue4, rmse4 = evaluate(R_test4, R_pred4)
P5, Q5 = nmf1.matrix_factorization(R_train5, P, Q, K)
R_pred5 = np.dot(P5, Q5.T)
precision5, recall5, fvalue5, rmse5 = evaluate(R_test5, R_pred5)
# 5 cross validation
print '**************precision***************'
def validate(self):
y_, y_pred = self.predict(self.dataloader["test"])
rocprc, rocauc, best_th, best_f1 = evaluate(y_, y_pred)
return rocauc, best_th, best_f1
def combined_result(gold_file,
classifier_result1,
classifier_result2,
method='sum'):
infile = open(classifier_result1)
data = infile.readlines()
label1 = data[0].strip().split()
result1 = [line.strip().split() for line in data[1:]]
infile.close()
infile = open(classifier_result2)
data = infile.readlines()
label2 = data[0].strip().split()
result2 = [line.strip().split() for line in data[1:]]
infile.close()
infile = open(gold_file)
data = infile.readlines()
gold_data = [int(line.strip().split()[0]) for line in data]
infile.close()
if len(result1) != len(result2):
raise ValueError('two list should be equal!')
if label1[1] != label2[1]:
raise ValueError('train data not consistent')
result = []
if method == 'max' or method == None:
for i in range(len(result1)):
temp = [
float(result1[i][1]),
float(result1[i][2]),
float(result2[i][1]),
float(result2[i][2])
]
index = temp.index(max(temp))
if label1[1] == '-1':
if index == 0 or index == 2:
result.append(-1)
else:
result.append(1)
else:
if index == 0 or index == 2:
result.append(1)
else:
result.append(-1)
elif method == 'product':
for i in range(len(result1)):
if float(result1[i][1]) * float(result2[i][1]) >= float(
result1[i][2]) * float(result2[i][2]):
result.append(1)
else:
result.append(-1)
elif method == 'min':
for i in range(len(result1)):
temp = [
min([float(result1[i][1]),
float(result2[i][1])]),
min([float(result1[i][2]),
float(result2[i][2])])
]
index = temp.index(max(temp))
if index == 0:
result.append(1)
else:
result.append(-1)
elif method == 'sum':
for i in range(len(result1)):
temp = [
float(result1[i][1]) + float(result2[i][1]),
float(result1[i][2]) + float(result2[i][2])
]
index = temp.index(max(temp))
if index == 0:
result.append(1)
else:
result.append(-1)
p, r, f, auc = evaluate(np.asarray(gold_data), np.asarray(result))
return (p, r, f, auc)
def main():
# tensorboard writer
writer = SummaryWriter(settings.TENSORBOARD_DIR)
# makedir snapshot
makedir(settings.CHECKPOINT_DIR)
# enable cudnn
torch.backends.cudnn.enabled = True
# create segmentor network
model = Segmentor(pretrained=settings.PRETRAINED,
num_classes=settings.NUM_CLASSES,
modality=settings.MODALITY)
model.train()
model.cuda()
torch.backends.cudnn.benchmark = True
# dataset and dataloader
dataset = TrainDataset()
dataloader = data.DataLoader(dataset,
batch_size=settings.BATCH_SIZE,
shuffle=True,
num_workers=settings.NUM_WORKERS,
pin_memory=True,
drop_last=True)
dataloader_iter = enumerate(dataloader)
# optimizer for generator network (segmentor)
optim = SGD(model.optim_parameters(settings.LR),
lr=settings.LR,
momentum=settings.LR_MOMENTUM,
weight_decay=settings.WEIGHT_DECAY)
# losses
ce_loss = CrossEntropyLoss2d(
ignore_index=settings.IGNORE_LABEL) # to use for segmentor
# upsampling for the network output
upsample = nn.Upsample(size=(settings.CROP_SIZE, settings.CROP_SIZE),
mode='bilinear',
align_corners=True)
# confusion matrix ; to track metrics such as mIoU during training
conf_mat = np.zeros((settings.NUM_CLASSES, settings.NUM_CLASSES))
for i_iter in range(settings.MAX_ITER):
# initialize losses
loss_G_seg_value = 0
# clear optim gradients and adjust learning rates
optim.zero_grad()
lr_poly_scheduler(optim, settings.LR, settings.LR_DECAY_ITER, i_iter,
settings.MAX_ITER, settings.LR_POLY_POWER)
####### train generator #######
# get the batch of data
try:
_, batch = next(dataloader_iter)
except:
dataloader_iter = enumerate(dataloader)
_, batch = next(dataloader_iter)
images, depths, labels = batch
images = images.cuda()
depths = depths.cuda()
labels = labels.cuda()
# get a mask where an elemnt is True for every pixel with ignore_label value
ignore_mask = (labels == settings.IGNORE_LABEL)
target_mask = torch.logical_not(ignore_mask)
target_mask = target_mask.unsqueeze(dim=1)
# get the output of generator
if settings.MODALITY == 'rgb':
predict = upsample(model(images))
elif settings.MODALITY == 'middle':
predict = upsample(model(images, depths))
# calculate cross-entropy loss
loss_G_seg = ce_loss(predict, labels)
# accumulate loss, backward and store value
loss_G_seg.backward()
loss_G_seg_value += loss_G_seg.data.cpu().numpy()
####### end of train generator #######
optim.step()
# get pred and gt to compute confusion matrix
seg_pred = np.argmax(predict.detach().cpu().numpy(), axis=1)
seg_gt = labels.cpu().numpy().copy()
seg_pred = seg_pred[target_mask.squeeze(dim=1).cpu().numpy()]
seg_gt = seg_gt[target_mask.squeeze(dim=1).cpu().numpy()]
conf_mat += confusion_matrix(seg_gt,
seg_pred,
labels=np.arange(settings.NUM_CLASSES))
####### log ########
if i_iter % (
(settings.TRAIN_SIZE // settings.BATCH_SIZE)) == 0 and i_iter != 0:
metrics = evaluate(conf_mat)
writer.add_scalar('Pixel Accuracy/Train', metrics['pAcc'], i_iter)
writer.add_scalar('Mean Accuracy/Train', metrics['mAcc'], i_iter)
writer.add_scalar('mIoU/Train', metrics['mIoU'], i_iter)
writer.add_scalar('fwavacc/Train', metrics['fIoU'], i_iter)
conf_mat = np.zeros_like(conf_mat)
writer.add_scalar('Loss_G_SEG/Train', loss_G_seg_value, i_iter)
writer.add_scalar('learning_rate_G/Train', optim.param_groups[0]['lr'],
i_iter)
print("iter = {:6d}/{:6d},\t loss_seg = {:.3f}".format(
i_iter, settings.MAX_ITER, loss_G_seg_value))
with open(settings.LOG_FILE, "a") as f:
output_log = '{:6d},\t {:.8f}\n'.format(i_iter, loss_G_seg_value)
f.write(output_log)
# taking snapshot
if i_iter >= settings.MAX_ITER:
print('saving the final model ...')
torch.save(
model.state_dict(),
osp.join(settings.CHECKPOINT_DIR,
'CHECKPOINT_' + str(settings.MAX_ITER) + '.pt'))
break
if i_iter % settings.SAVE_EVERY == 0 and i_iter != 0:
print('taking snapshot ...')
torch.save(
model.state_dict(),
osp.join(settings.CHECKPOINT_DIR,
'CHECKPOINT_' + str(i_iter) + '.pt'))
def main():
print('Loading data...')
train, valid, test = load_data(args.dataset_path, valid_portion=args.valid_portion)
train_data = RecSysDataset(train)
valid_data = RecSysDataset(valid)
test_data = RecSysDataset(test)
train_loader = DataLoader(train_data, batch_size = args.batch_size, shuffle = True, collate_fn = collate_fn)
valid_loader = DataLoader(valid_data, batch_size = args.batch_size, shuffle = False, collate_fn = collate_fn)
test_loader = DataLoader(test_data, batch_size = args.batch_size, shuffle = False, collate_fn = collate_fn)
n_items = 37484
if args.test:
for i in range(5):
results=np.zeros((3,3))
model = NARM(n_items, args.hidden_size, args.embed_dim, args.batch_size).to(device)
optimizer = optim.Adam(model.parameters(), args.lr)
criterion = nn.CrossEntropyLoss()
scheduler = StepLR(optimizer, step_size=args.lr_dc_step, gamma=args.lr_dc)
for epoch in tqdm(range(args.epoch)):
# train for one epoch
scheduler.step(epoch=epoch)
trainForEpoch(train_loader, model, optimizer, epoch, args.epoch, criterion, log_aggr=1000)
model.eval()
recalls5,recalls10,recalls20 = [],[],[]
mrrs5,mrrs10,mrrs20 = [],[],[]
ndcgs5,ndcgs10,ndcgs20 = [],[],[]
with torch.no_grad():
for seq, target, lens in tqdm(valid_loader):
seq = seq.to(device)
target = target.to(device)
outputs = model(seq, lens)
logits = F.softmax(outputs, dim=1)
recall5, mrr5, ndcg5 = metric.evaluate(logits, target, k=5)
recall10, mrr10, ndcg10 = metric.evaluate(logits, target, k=10)
recall20, mrr20, ndcg20 = metric.evaluate(logits, target, k=args.topk)
recalls5.append(recall5)
mrrs5.append(mrr5)
ndcgs5.append(ndcg5)
recalls10.append(recall10)
mrrs10.append(mrr10)
ndcgs10.append(ndcg10)
recalls20.append(recall20)
mrrs20.append(mrr20)
ndcgs20.append(ndcg20)
results[0,0]=np.mean(recalls5)
results[0,1]=np.mean(mrrs5)
results[0,2]=np.mean(ndcgs5)
results[1, 0] = np.mean(recalls10)
results[1, 1] = np.mean(mrrs10)
results[1, 2] = np.mean(ndcgs10)
results[2, 0] = np.mean(recalls20)
results[2, 1] = np.mean(mrrs20)
results[2, 2] = np.mean(ndcgs20)
with open('recsys19/test_performances_on.txt', 'a') as f:
f.write( str(results) + '\n')
model = NARM(n_items, args.hidden_size, args.embed_dim, args.batch_size).to(device)
optimizer = optim.Adam(model.parameters(), args.lr)
criterion = nn.CrossEntropyLoss()
scheduler = StepLR(optimizer, step_size = args.lr_dc_step, gamma = args.lr_dc)
for epoch in tqdm(range(args.epoch)):
# train for one epoch
scheduler.step(epoch = epoch)
trainForEpoch(train_loader, model, optimizer, epoch, args.epoch, criterion, log_aggr = 1000)
recall, mrr ,ndcg= validate(test_loader, model)
print('Epoch {} validation: Recall@{}: {:.4f}, MRR@{}: {:.4f} \n'.format(epoch, args.topk, recall, args.topk, mrr))
# store best loss and save a model checkpoint
ckpt_dict = {
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict()
}
torch.save(ckpt_dict, 'latest_checkpoint.pth.tar')
