def get_product_scores(args):
# Prepare data.
print("Reading data in %s" % args.data_dir)
ranklist_dir = "%s/ranklist" % (args.save_dir)
if not os.path.isdir(ranklist_dir):
os.makedirs(ranklist_dir)
data_set = data_util.upsearch_data(args.data_dir,
args.input_train_dir,
'test',
args.batch_size,
args.model_net_struct,
args.threshold,
is_av_embed_shared=args.share_av_emb)
# Create model.
orig_init_learning_rate = args.init_learning_rate
model, optimizer = create_model(args, data_set)
model.eval()
#for dropout
current_step = 0
user_ranklist_map = colls.defaultdict(list)
user_ranklist_score_map = colls.defaultdict(list)
print('Start Testing')
data_set.setup_data_set(0)
data_set.initialize_test_iter()
data_set.prepare_test_epoch()
data_set.read_train_product_ids(args.input_train_dir)
has_next = True
while has_next:
user_idxs, query_idxs, query_word_idxs, has_next = data_set.get_test_batch(
)
if len(user_idxs) == 0:
break
#product_idxs can be refined to a candidate set
user_product_scores = model.get_product_scores(user_idxs,
query_word_idxs,
product_idxs=None)
current_step += 1
#batch_size, product_size
# record the results
store_sorted_ranklist(data_set, args.rank_cutoff, \
user_ranklist_map, user_ranklist_score_map,
user_product_scores, user_idxs, query_idxs,
args.keep_first_rel_qu, product_idxs = None)
if current_step % args.steps_per_checkpoint == 0:
print("Finish test qu %d/%d\r" %
(data_set.cur_uqr_i, len(data_set.test_seq)),
end="")
fname_arr = [args.model_net_struct, args.similarity_func]
fname_arr += ["bz%d" % args.batch_size, args.optimizer]
if args.L2_lambda > 0.:
fname_arr.append("%.0E" % Decimal(args.L2_lambda))
fname_arr += [
"lr%.4f" % orig_init_learning_rate,
"subs%.0E" % Decimal(args.subsampling_rate)
]
fname_arr += ["qw%.1f" % args.query_weight, "emb%d" % args.embed_size]
if args.scale_grad:
fname_arr += ["scale%s" % args.scale_grad]
fname_arr += [
"sparse%s" % args.sparse_emb,
"firstkept%s.ckpt" % args.keep_first_rel_qu
]
fname = '_'.join(fname_arr)
data_set.output_ranklist(user_ranklist_map, user_ranklist_score_map,
ranklist_dir, fname)
print("first relevant skipped qu:", data_set.skip_store_rank_count)
def get_product_scores_av(args):
# Prepare data.
print("Reading data in %s" % args.data_dir)
data_set = data_util.upsearch_data(
args.data_dir,
args.input_train_dir,
'test',
args.batch_size,
args.model_net_struct,
args.threshold,
is_av_embed_shared=args.share_av_emb,
is_feedback_same_user=args.is_feedback_same_user,
neg_per_pos=args.neg_per_pos,
keep_feedback_type=args.keep_feedback_type, #"pos", "neg"
feedback_user_model=args.feedback_user_model, #ra, fa, rv, fv
max_av_count=args.av_count_per_iter,
info_level=args.info_level)
#get q-u pair both with av and without av, the total number of which is args.batch_size
# Create model.
ranklist_dir = "%s/ranklist" % (args.save_dir)
if not os.path.isdir(ranklist_dir):
os.makedirs(ranklist_dir)
orig_init_learning_rate = args.init_learning_rate
model, optimizer = create_model(args, data_set)
model.eval()
model.init_aspect_vecs_for_test()
#for dropout
current_step = 0
user_ranklist_map = colls.defaultdict(list)
user_ranklist_score_map = colls.defaultdict(list)
print('Start Testing')
start_time = time.time()
#data_set.setup_data_set(0)
data_set.prepare_test_epoch()
data_set.read_train_product_ids(args.input_train_dir)
torch.set_grad_enabled(False) #do not require grad
for cur_iter_i in xrange(1, args.iter_count + 1):
data_set.initialize_test_iter()
has_next = True
qu_count = 0
apaired_qu_count = 0
while has_next:
product_idxs = None
av_product_idxs = None
user_idxs, query_idxs, query_word_idxs, \
av_user_idxs, av_query_idxs, av_query_word_idxs, query_av_pairs, has_next \
= data_set.get_av_test_batch(user_ranklist_map, cur_iter_i)
qu_count += len(user_idxs)
apaired_qu_count += len(av_user_idxs)
if len(user_idxs) == 0 and len(av_user_idxs) == 0:
break
if len(user_idxs) > 0:
user_product_scores = model.get_product_scores(
user_idxs, query_word_idxs, product_idxs=None)
store_sorted_ranklist(data_set, args.rank_cutoff, \
user_ranklist_map, user_ranklist_score_map,
user_product_scores, user_idxs, query_idxs,
args.keep_first_rel_qu,
cur_iter = cur_iter_i, product_idxs = None)
if len(av_user_idxs) > 0:
user_av_product_scores = model.get_product_scores_av(
av_user_idxs,
av_query_word_idxs,
query_av_pairs,
product_idxs=av_product_idxs)
store_sorted_ranklist(data_set, args.rank_cutoff, \
user_ranklist_map, user_ranklist_score_map,
user_av_product_scores, av_user_idxs, av_query_idxs,
args.keep_first_rel_qu,
cur_iter = cur_iter_i, product_idxs = None)
#print("len(av_user_idx)", len(av_user_idxs))
current_step += 1
#batch_size, product_size
# record the results
#if current_step % args.steps_per_checkpoint == 0:
if current_step % 10 == 0:
print("Finish test qu %d/%d" %
(data_set.cur_uqr_i, len(data_set.test_seq)),
end="")
print("Iter%d qu_count:%d apaired_qu_count:%d" %
(cur_iter_i, qu_count, apaired_qu_count))
fname_arr = [
args.model_net_struct, args.similarity_func, args.comb_net_struct
]
fname_arr += ["bz%d" % args.batch_size, args.optimizer]
if args.L2_lambda > 0.:
fname_arr.append("%.0E" % Decimal(args.L2_lambda))
fname_arr += [
"lr%.4f" % orig_init_learning_rate,
"subs%.0E" % Decimal(args.subsampling_rate)
]
fname_arr += ["a%s" % args.aspect_prob_type, args.value_loss_func]
fname_arr += ["qw%.1f" % args.query_weight, "emb%d" % args.embed_size]
if args.scale_grad:
fname_arr += ["scale%s" % args.scale_grad]
fname_arr += [
"lh%s" % args.likelihood_way,
"sparse%s%s" % (args.sparse_emb, args.loss_ablation)
]
fname_arr += [
"info%d" % args.info_level, "sameu",
str(args.is_feedback_same_user)
]
fname_arr += [
"firstkept%s" % args.keep_first_rel_qu, args.keep_feedback_type
]
fname_arr += ["avcount%d" % args.av_count_per_iter]
fname_arr += ["%siter%d.ckpt" % (args.feedback_user_model, cur_iter_i)]
fname = '_'.join(fname_arr)
data_set.output_ranklist(user_ranklist_map, user_ranklist_score_map,
ranklist_dir, fname)
print("first relevant skipped qu:", data_set.skip_store_rank_count)
print("QU updated in this iter:", len(data_set.test_user_query_set))
def train(args):
print("Reading data in %s" % args.data_dir)
args.start_epoch = 0
data_set = data_util.upsearch_data(
args.data_dir,
args.input_train_dir,
'train',
args.batch_size,
args.model_net_struct,
args.threshold,
is_av_embed_shared=args.share_av_emb,
is_feedback_same_user=args.is_feedback_same_user,
neg_per_pos=args.neg_per_pos)
data_set.sub_sampling(args.subsampling_rate)
orig_init_learning_rate = args.init_learning_rate
learning_rate = orig_init_learning_rate
model, optimizer = create_model(args, data_set)
#args.init_learning_rate as current initial learning rate has been reset from the model loaded
model.train()
model_dir = "%s/model" % (args.save_dir)
if not os.path.isdir(model_dir):
os.makedirs(model_dir)
words_to_train = float(args.max_train_epoch * data_set.word_count) + 1
data_set.setup_data_set(words_to_train)
previous_words, step_time, loss = 0., 0., 0.
get_batch_time = 0.0
start_time = time.time()
current_epoch = args.start_epoch
current_step = 0
total_norm = 0.
is_best = False
while current_epoch < args.max_train_epoch:
print("Initialize epoch:%d" % current_epoch)
data_set.initialize_epoch()
word_has_next = True
while word_has_next:
time_flag = time.time()
batch_data = None
#only when it's its turn and not empty
#or when the other one is empty
if word_has_next:
#print("train word")
if args.model_net_struct == 'AVHEM':
batch_data = data_set.get_av_train_batch()
else:
batch_data = data_set.get_train_batch()
word_has_next = batch_data[-1]
get_batch_time += time.time() - time_flag
pivot_time = time.time()
learning_rate = args.init_learning_rate * max(
0.0001, 1.0 - data_set.finished_word_num / words_to_train)
if args.optimizer == 'sgd':
adjust_learning_rate(optimizer, learning_rate)
if len(batch_data[0]) > 0:
time_flag = time.time()
step_loss = model(batch_data[:-1])
optimizer.zero_grad()
step_loss.backward()
cur_batch_norm = torch.nn.utils.clip_grad_norm_(
model.parameters(), args.max_gradient_norm)
total_norm += cur_batch_norm / args.steps_per_checkpoint
optimizer.step()
loss += step_loss.item(
) / args.steps_per_checkpoint #convert an tensor with dim 0 to value
current_step += 1
step_time += time.time() - time_flag
# Once in a while, we print statistics.
if current_step % args.steps_per_checkpoint == 0:
print(
"Epoch %d Words %d/%d: lr = %5.4f loss = %6.2f words/sec = %5.2f prepare_time %.2f step_time %.2f"
% (current_epoch, data_set.finished_word_num,
words_to_train, learning_rate, loss,
(data_set.finished_word_num - previous_words) /
(time.time() - start_time), get_batch_time,
step_time)) #, end=""
print(
"av_loss:%.2f ia_loss:%.2f iav_loss:%.2f iav_pos_loss:%.2f iav_neg_loss:%.2f up_loss1:%.2f"
% (model.group_av_loss / args.steps_per_checkpoint,
model.group_ia_loss / args.steps_per_checkpoint,
model.group_iav_loss / args.steps_per_checkpoint,
model.group_iav_pos_loss / args.steps_per_checkpoint,
model.group_iav_neg_loss / args.steps_per_checkpoint,
model.group_up_loss1 / args.steps_per_checkpoint))
print(
"total_norm:%.2f word_loss:%.2f uw_loss:%.2f pw_loss:%.2f up_loss2:%.2f"
% (total_norm,
model.group_word_loss / args.steps_per_checkpoint,
model.group_uw_loss / args.steps_per_checkpoint,
model.group_pw_loss / args.steps_per_checkpoint,
model.group_up_loss2 / args.steps_per_checkpoint))
model.group_ia_loss, model.group_iav_loss, model.group_up_loss1 = 0., 0., 0.
model.group_uw_loss, model.group_pw_loss, model.group_up_loss2 = 0., 0., 0.
model.group_av_loss, model.group_word_loss, model.group_iav_pos_loss = 0., 0., 0.
model.group_iav_neg_loss, step_time, get_batch_time = 0., 0., 0.
total_norm, loss = 0., 0.
current_step = 1
sys.stdout.flush()
previous_words = data_set.finished_word_num
start_time = time.time()
#save model after each epoch
save_checkpoint(
{
'epoch': current_epoch + 1,
'learning_rate': learning_rate,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict()
}, is_best, model_path)
current_epoch += 1
save_checkpoint(
{
'epoch': current_epoch,
'learning_rate': learning_rate,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict()
}, is_best, model_path)