def eval_from_saved_model(model, args):
global hr_recover, ndcg_recover
import tensorflow as tf
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
if args.model == 'Multi_GMF':
EvalDict = EvalUser.init_evaluate_model(model, dataset[0])
else:
EvalDict = EvalUser.init_evaluate_model(model, dataset)
hits, ndcgs = [], []
loader = None
eval_begin = time()
eval_graph = tf.Graph()
with eval_graph.as_default():
model_load_path = '/data/stu/gandahua/MBR/model_save/model_' + filename
loader = tf.train.import_meta_graph(model_load_path + '.meta',
clear_devices=True)
with tf.Session(graph=eval_graph, config=config) as sess:
loader.restore(sess, model_load_path)
for idx in xrange(len(EvalDict)):
if args.model == 'Multi_GMF':
gtItem = dataset[0].testRatings[idx][1]
predictions = sess.run('loss/inference/score_buy:0',
feed_dict=EvalDict[idx])
else:
gtItem = dataset.testRatings[idx][1]
predictions = sess.run('loss/inference/output:0',
feed_dict=EvalDict[idx])
rank = 0
rank_score = predictions[gtItem]
for i in predictions:
if i > rank_score:
rank += 1
if rank < args.topK:
hr_tmp = 1
ndcg_tmp = math.log(2) / math.log(rank + 2)
else:
hr_tmp = 0
ndcg_tmp = 0
hits.append(hr_tmp)
ndcgs.append(ndcg_tmp)
hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean()
print "Final HR = %.4f, NDCG = %.4f" % (hr, ndcg)
# save info to the server
hr_recover, ndcg_recover = hr, ndcg
def training(model, args, behave_type=None, base_epoch=0, save=True):
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
global pool, dataset, eval_queue, job_num, loss_list, hr_list, ndcg_list
# initialize for Evaluate
EvalDict = EvalUser.init_evaluate_model(model, dataset, args)
with model.g.as_default():
with tf.name_scope('optimizer'):
if args.optimizer == 'Adam':
optimizer = tf.train.AdamOptimizer(learning_rate=args.lr).minimize(model.loss)
elif args.optimizer == 'Adagrad':
optimizer = tf.train.AdagradOptimizer(learning_rate=args.lr, initial_accumulator_value=1e-8).minimize(
model.loss)
else: # TODO(wgcn96): SGD
optimizer = tf.train.GradientDescentOptimizer(learning_rate=args.lr).minimize(loss=model.loss)
with tf.Session(graph=model.g, config=config) as sess:
# initial training
sess.run(tf.global_variables_initializer())
logging.info("--- Start training ---")
print("--- Start training ---")
# plot network then exit
print('plot_network:', args.plot_network)
if args.plot_network:
print("writing network to TensorBoard/graphs/network")
writer = tf.summary.FileWriter('TensorBoard/graphs/network')
writer.add_graph(sess.graph)
# return 0
# dict for printing behavior type
b_dict = {'vb': 'view and buy', 'cb': 'cart and buy', 'vc': 'view and cart'}
samples = BatchUser.sampling(args, dataset, args.num_neg) # TODO(wgcn96): 生成数据集不需要放在for循环里面
print('all training number: %d' % len(samples[0]))
# print('first label: %s' % samples[2][:20])
# print('first user: %s' % samples[0][:20])
# print('first item: %s' % samples[1][:20])
if behave_type == 'ipv':
bs = args.batch_size_ipv
elif behave_type == 'cart':
bs = args.batch_size_cart
else:
bs = args.batch_size_buy
best_hr, best_ndcg, best_epoch, best_loss = 0, 0, 0, 0 # TODO(wgcn96): 记录所有循环中的最佳结果
# train by epoch
for epoch_count in range(args.epochs):
# TODO(wgcn96): 单行为优化
batches = BatchUser.shuffle(samples, bs, args)
print('Already generate batch, batch size is %d' % bs)
train_begin = time()
train_loss = training_batch(model, sess, batches, args, optimizer)
train_time = time() - train_begin
# print('train time: %d' % train_time)
if epoch_count % args.verbose == 0 and args.evaluate == 'yes':
eval_begin = time()
hits, ndcgs = EvalUser.eval(model, sess, dataset, EvalDict, args)
hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean()
eval_time = time() - eval_begin
logging.info(
"Epoch %d [train %.1fs]: train_loss = %.4f [test %.1fs] HR = %.4f, NDCG = %.4f"
% (epoch_count + 1, train_time, train_loss, eval_time, hr, ndcg))
print("Epoch %d [train %.1fs]: train_loss = %.4f [test %.1fs] HR = %.4f, NDCG = %.4f" % (
epoch_count + 1, train_time, train_loss, eval_time, hr, ndcg))
# print('eval time : {}'.format(eval_time))
if hr >= best_hr and ndcg >= best_ndcg:
best_hr = hr
best_ndcg = ndcg
best_epoch = epoch_count + 1
best_loss = train_loss
# save results, save model
(hr_list[base_epoch + epoch_count], ndcg_list[base_epoch + epoch_count],
loss_list[base_epoch + epoch_count]) = (hr, ndcg, train_loss)
if epoch_count == (args.epochs - 1):
print("All finish, best result hr: {}, ndcg: {}, epoch: {}, train loss: {}".format(best_hr, best_ndcg,
best_epoch,
best_loss))
hr_list.append(best_hr)
ndcg_list.append(best_ndcg)
loss_list.append(best_epoch)
def training(model, args, behave_type=None, base_epoch=0, save=True):
# import tensorflow as tf
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# config.gpu_options.per_process_gpu_memory_fraction = 0.1
global pool, dataset, eval_queue, job_num, loss_list, hr_list, ndcg_list
# initialize for Evaluate
if args.model in ['Multi_GMF', 'Multi_MLP', 'Multi_NCF', 'CMF']:
EvalDict = EvalUser.init_evaluate_model(model, dataset[0], args)
else:
EvalDict = EvalUser.init_evaluate_model(model, dataset, args)
with model.g.as_default():
saver = tf.train.Saver()
with tf.name_scope('optimizer'):
if args.optimizer == 'Adam':
optimizer = tf.train.AdamOptimizer(
learning_rate=args.lr).minimize(model.loss)
elif args.optimizer == 'Adagrad':
optimizer = tf.train.AdagradOptimizer(
learning_rate=args.lr,
initial_accumulator_value=1e-8).minimize(model.loss)
else: # TODO(wgcn96): SGD
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=args.lr).minimize(loss=model.loss)
with tf.Session(graph=model.g, config=config) as sess:
# initial training
sess.run(tf.global_variables_initializer())
logging.info("--- Start training ---")
print("--- Start training ---")
# plot network then exit
print('plot_network:', args.plot_network)
if args.plot_network:
print("writing network to TensorBoard/graphs/network")
writer = tf.summary.FileWriter('TensorBoard/graphs/network')
writer.add_graph(sess.graph)
# return 0
# dict for printing behavior type
b_dict = {
'vb': 'view and buy',
'cb': 'cart and buy',
'vc': 'view and cart'
}
samples = BatchUser.sampling_3(
args, dataset, args.num_neg) # TODO(wgcn96): 生成数据集不需要放在for循环里面
print('all training number: %d' % len(samples[0]))
if args.b_num == 3:
bs = args.batch_size
elif args.b_2_type == 'cb':
bs = args.batch_size_cart
else:
bs = args.batch_size_ipv
best_hr, best_ndcg, best_epoch, best_loss = 0, 0, 0, 0 # TODO(wgcn96): 记录所有循环中的最佳结果
# train by epoch
for epoch_count in range(args.epochs):
# if 'Multi' in args.model or 'BPR' in args.model: # TODO(wgcn96): 这说明什么，BPR本身就用了多行为的？
batches = BatchUser.shuffle_3(samples, bs, args)
# print("check list {}, arg1: {}, arg2: {}, arg3: {}".format(len(batches), len(batches[0]), len(batches[1]),
# len(batches[2])))
train_begin = time()
train_loss = training_batch_3(model, sess, batches, args,
optimizer)
train_time = time() - train_begin
if epoch_count % args.verbose == 0 and args.evaluate == 'yes':
eval_begin = time()
hits, ndcgs = EvalUser.eval(model, sess, dataset, EvalDict,
args)
hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean()
eval_time = time() - eval_begin
logging.info(
"Epoch %d [train %.1fs]: train_loss = %.4f [test %.1fs] HR = %.4f, NDCG = %.4f"
% (epoch_count + 1, train_time, train_loss, eval_time, hr,
ndcg))
print(
"Epoch %d [train %.1fs]: train_loss = %.4f [test %.1fs] HR = %.4f, NDCG = %.4f"
% (epoch_count + 1, train_time, train_loss, eval_time, hr,
ndcg))
# print('eval time : {}'.format(eval_time))
if hr >= best_hr and ndcg >= best_ndcg:
best_hr = hr
best_ndcg = ndcg
best_epoch = epoch_count + 1
best_loss = train_loss
# save results, save model
(hr_list[base_epoch + epoch_count],
ndcg_list[base_epoch + epoch_count],
loss_list[base_epoch + epoch_count]) = (hr, ndcg, train_loss)
model_save_path = 'Model/'
if args.add_datetime == 'yes':
model_save_path += datetime.datetime.now().strftime("%m%d")
model_save_path += '/'
if not os.path.exists(model_save_path):
os.makedirs(model_save_path)
model_save_path += 'model_' + filename
saver.save(sess, model_save_path)
if epoch_count == (args.epochs - 1):
print(
"All finish, best result hr: {}, ndcg: {}, epoch: {}, train loss: {}"
.format(best_hr, best_ndcg, best_epoch, best_loss))
hr_list.append(best_hr)
ndcg_list.append(best_ndcg)
loss_list.append(best_epoch)
def eval_from_saved_model(model, args):
global hr_recover, ndcg_recover
# import tensorflow as tf
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# EvalDict = EvalUser.init_evaluate_model(model, dataset[0], args) # TODO(wgcn96): why this?
EvalDict = EvalUser.gen_feed_dict(dataset[0])
hits, ndcgs = [], []
loader = None
eval_begin = time()
eval_graph = tf.Graph()
with eval_graph.as_default():
# model_load_path = 'Model/'
# if args.add_datetime == 'yes':
# model_load_path += datetime.datetime.now().strftime("%m%d")
# model_load_path += '/'
# if not os.path.exists(model_load_path):
# raise Exception('file not exist error!!!')
# model_load_path += 'model_' + filename + '-' + str(args.epochs-1)
loader = tf.train.import_meta_graph(args.recover_path + '.meta',
clear_devices=True)
with tf.Session(graph=eval_graph, config=config) as sess:
loader.restore(sess, args.recover_path)
for idx in range(len(EvalDict)):
if 'Multi' in args.model or 'Our' in args.model: # TODO(wgcn96): for multi-behavior
gtItem = dataset[0].testRatings[idx][1]
predictions = sess.run('loss/inference/score_buy:0',
feed_dict=EvalDict[idx])
else: # TODO(wgcn96): for single-behavior
gtItem = dataset.testRatings[idx][1]
predictions = sess.run('loss/inference/output:0',
feed_dict=EvalDict[idx])
sort_indexes = np.argsort(
-predictions.reshape(predictions.shape[0]))
top_K = sort_indexes[:args.topK]
hr_tmp = 0
ndcg_tmp = 0
# 计算打分
if gtItem in top_K:
hr_tmp = 1
pos = np.where(top_K == gtItem)[0]
ndcg_tmp = math.log(2) / math.log(pos + 2)
hits.append(hr_tmp)
ndcgs.append(ndcg_tmp)
"""
rank = 0
rank_score = predictions[gtItem]
for i in predictions:
if i > rank_score:
rank += 1
if rank < args.topK:
hr_tmp = 1
ndcg_tmp = math.log(2) / math.log(rank + 2)
else:
hr_tmp = 0
ndcg_tmp = 0
hits.append(hr_tmp)
ndcgs.append(ndcg_tmp)
"""
hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean()
print("Final HR = %.4f, NDCG = %.4f , total time %.f" %
(hr, ndcg, time() - eval_begin))
# save info to the server
hr_recover, ndcg_recover = hr, ndcg
def training(model, args, behave_type=None, base_epoch=0, save=True):
# import tensorflow as tf
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# config.gpu_options.per_process_gpu_memory_fraction = 0.1
global pool, dataset, eval_queue, job_num, loss_list, hr_list, ndcg_list
# initialize for Evaluate
EvalDict = EvalUser.init_evaluate_model(model, dataset[0], args)
with model.g.as_default():
saver = tf.train.Saver()
with tf.name_scope('optimizer'):
if args.optimizer == 'Adam':
optimizer = tf.train.AdamOptimizer(
learning_rate=args.lr).minimize(model.loss)
elif args.optimizer == 'Adagrad':
optimizer = tf.train.AdagradOptimizer(
learning_rate=args.lr,
initial_accumulator_value=1e-8).minimize(model.loss)
else: # TODO(wgcn96): SGD
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=args.lr).minimize(loss=model.loss)
with tf.Session(graph=model.g, config=config) as sess:
# initial training
sess.run(tf.global_variables_initializer())
logging.info("--- Start training ---")
print("--- Start training ---")
# plot network then exit
print('plot_network:', args.plot_network)
if args.plot_network:
print("writing network to TensorBoard/graphs/network")
writer = tf.summary.FileWriter('TensorBoard/graphs/network')
writer.add_graph(sess.graph)
return 0
# dict for printing behavior type
b_dict = {
'vb': 'view and buy',
'cb': 'cart and buy',
'vc': 'view and cart'
}
samples = BatchUser.sampling_3(
args, dataset, args.num_neg) # TODO(wgcn96): 生成数据集不需要放在for循环里面
print('all training number: %d' % len(samples[0]))
if args.b_num == 3:
bs = args.batch_size
elif args.b_2_type == 'cb':
bs = args.batch_size_cart
else:
bs = args.batch_size_ipv
best_hr, best_ndcg, best_epoch, best_loss = 0, 0, 0, 0 # TODO(wgcn96): 记录所有循环中的最佳结果
# train by epoch
for epoch_count in range(args.epochs):
batches = BatchUser.shuffle_3(samples, bs, args)
# print('Already generate batch, behavior is %d(%s), \n\
# batch size is %d, all training entries: %d' % (
# args.b_num, b_dict[args.b_2_type], bs, len(samples[0])))
train_begin = time()
train_loss = training_batch_3(model, sess, batches, args,
optimizer)
train_time = time() - train_begin
if epoch_count % args.verbose == 0 and args.evaluate == 'yes':
eval_begin = time()
hits, ndcgs = EvalUser.eval(model, sess, dataset, EvalDict,
args)
hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean()
eval_time = time() - eval_begin
logging.info(
"Epoch %d [train %.1fs]: train_loss = %.4f [test %.1fs] HR = %.4f, NDCG = %.4f"
% (epoch_count + 1, train_time, train_loss, eval_time, hr,
ndcg))
print(
"Epoch %d [train %.1fs]: train_loss = %.4f [test %.1fs] HR = %.4f, NDCG = %.4f"
% (epoch_count + 1, train_time, train_loss, eval_time, hr,
ndcg))
# print('eval time : {}'.format(eval_time))
if hr >= best_hr and ndcg >= best_ndcg:
best_hr = hr
best_ndcg = ndcg
best_epoch = epoch_count + 1
best_loss = train_loss
model_save_path = 'Model/' + args.process_name + '_best' # save best model
saver.save(sess, model_save_path)
# save best result
(hr_list[base_epoch + epoch_count],
ndcg_list[base_epoch + epoch_count],
loss_list[base_epoch + epoch_count]) = (hr, ndcg, train_loss)
if epoch_count == (args.epochs - 1):
print(
"All finish, best result hr: {}, ndcg: {}, epoch: {}, train loss: {}"
.format(best_hr, best_ndcg, best_epoch, best_loss))
hr_list.append(best_hr)
ndcg_list.append(best_ndcg)
loss_list.append(best_epoch)
prefix = ''
if args.add_datetime == 'yes':
prefix += datetime.datetime.now().strftime("%m%d")
prefix += '/'
if not os.path.exists(prefix):
os.makedirs(prefix)
with open(prefix + args.save_file, 'a+') as fo:
fo.write("---------------\n")
# fo.write("loss_func: %s\n" % args.loss_func)
fo.write("batch_size: %s\n" % args.batch_size)
fo.write("layer_num: %s\n" % args.layer_num)
fo.write("lr: %s\n" % args.lr)
fo.write("epoch: %s\n" % args.epochs)
fo.write("optimizer: %s\n" % args.optimizer)
fo.write("reg: %s\n" % args.regs)
fo.write(
"All finish, best result hr: {}, ndcg: {}, epoch: {}, train loss: {}\n"
.format(best_hr, best_ndcg, best_epoch, best_loss))
def eval_from_saved_model(model, args):
global hr_recover, ndcg_recover
# import tensorflow as tf
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# EvalDict = EvalUser.init_evaluate_model(model, dataset[0], args) # TODO(wgcn96): why this?
EvalDict = EvalUser.gen_feed_dict(dataset[0])
hits, ndcgs = [], []
loader = None
eval_graph = tf.Graph()
with eval_graph.as_default():
# model_load_path = 'Model/model_' + filename
loader = tf.train.import_meta_graph(args.recover_path + '.meta',
clear_devices=True)
with tf.Session(graph=eval_graph, config=config) as sess:
loader.restore(sess, args.recover_path)
for idx in range(len(EvalDict)):
gtItem = dataset[0].testRatings[idx][1]
predictions = sess.run('loss/errors/inference/score_buy:0',
feed_dict=EvalDict[idx])
sort_indexes = np.argsort(
-predictions.reshape(predictions.shape[0]))
top_K = sort_indexes[:args.topK]
hr_tmp = 0
ndcg_tmp = 0
# 计算打分
if gtItem in top_K:
hr_tmp = 1
pos = np.where(top_K == gtItem)[0]
ndcg_tmp = math.log(2) / math.log(pos + 2)
hits.append(hr_tmp)
ndcgs.append(ndcg_tmp)
"""
rank = 0
rank_score = predictions[gtItem]
for i in predictions:
if i > rank_score:
rank += 1
if rank < args.topK:
hr_tmp = 1
ndcg_tmp = math.log(2) / math.log(rank + 2)
else:
hr_tmp = 0
ndcg_tmp = 0
hits.append(hr_tmp)
ndcgs.append(ndcg_tmp)
"""
hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean()
print("Final HR = %.4f, NDCG = %.4f" % (hr, ndcg))
# save info to the server
hr_recover, ndcg_recover = hr, ndcg
def training(model, dataset, args, saver = None): # saver is an object to save pq
with tf.Session() as sess:
# pretrain nor not
sess.run(tf.global_variables_initializer())
logging.info("initialized")
print "initialized"
writer = tf.summary.FileWriter('./graphs', sess.graph)
# initialize for training batches
if args.batch_gen == "fixed":
if args.model == "FISM":
samples = BatchItem.sampling(args, dataset, args.num_neg)
else:
samples = BatchUser.sampling(args, dataset, args.num_neg)
# initialize for Evaluate
if args.model == "FISM":
EvalDict = EvalItem.init_evaluate_model(model, dataset)
else:
EvalDict = EvalUser.init_evaluate_model(model, dataset)
# train by epoch
for epoch_count in range(args.epochs):
batch_begin = time()
if args.model == "FISM":
if args.batch_gen == "unfixed":
samples = BatchItem.sampling(args, dataset, args.num_neg)
batches = BatchItem.shuffle(samples, args.batch_size, dataset)
else :
if args.batch_gen == "unfixed":
samples = BatchUser.sampling(args, dataset, args.num_neg)
batches = BatchUser.shuffle(samples, args.batch_size)
batch_time = time() - batch_begin
train_begin = time()
training_batch(model, sess, batches, args)
train_time = time() - train_begin
if epoch_count % args.verbose == 0:
if args.train_loss:
loss_begin = time()
train_loss = training_loss(model, sess, batches, args)
loss_time = time() - loss_begin
else:
loss_time, train_loss = 0, 0
eval_begin = time()
if args.model == "FISM":
hits, ndcgs, losses = EvalItem.eval(model, sess, dataset, EvalDict)
else:
hits, ndcgs, losses = EvalUser.eval(model, sess, dataset, EvalDict)
hr, ndcg, test_loss = np.array(hits).mean(), np.array(ndcgs).mean(), np.array(losses).mean()
eval_time = time() - eval_begin
logging.info(
"Epoch %d [%.1fs + %.1fs]: HR = %.4f, NDCG = %.4f, loss = %.4f [%.1fs] train_loss = %.4f [%.1fs]" % (
epoch_count, batch_time, train_time, hr, ndcg, test_loss, eval_time, train_loss, loss_time))
print "Epoch %d [%.1fs + %.1fs]: HR = %.4f, NDCG = %.4f, loss = %.4f [%.1fs] train_loss = %.4f [%.1fs]" % (
epoch_count, batch_time, train_time, hr, ndcg, test_loss, eval_time, train_loss, loss_time)
if saver != None:
saver.save(model, sess)
def training(model, args, behave_type=None, base_epoch=0, save=True):
cascade = args.model == 'NCF_FC' or args.model == 'GMF_FC'
if cascade:
cascade_path = None
import tensorflow as tf
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.gpu_options.per_process_gpu_memory_fraction = 0.1
global pool, dataset, eval_queue, job_num, loss_list, hr_list, ndcg_list
# initialize for Evaluate
if args.multiprocess == 'no':
if args.model in ['Multi_GMF', 'Multi_MLP', 'Multi_NCF']:
EvalDict = EvalUser.init_evaluate_model(model, dataset[0], args)
else:
EvalDict = EvalUser.init_evaluate_model(model, dataset, args)
with model.g.as_default():
saver = tf.train.Saver()
if cascade:
cascade_saver = tf.train.Saver()
with tf.name_scope('optimizer'):
if args.optimizer == 'Adam':
if cascade == False:
optimizer = tf.train.AdamOptimizer(
learning_rate=args.lr).minimize(model.loss)
else:
if behave_type == 'ipv':
optimizer = tf.train.AdamOptimizer(
learning_rate=args.lr).minimize(
model.loss1,
var_list=[
model.embedding_P, model.embedding_Q,
model.h_1
])
elif behave_type == 'cart':
optimizer = tf.train.AdamOptimizer(
learning_rate=args.lr).minimize(
model.loss2,
var_list=[model.W1, model.b1, model.h_2])
else:
optimizer = tf.train.AdamOptimizer(
learning_rate=args.lr).minimize(
model.loss3,
var_list=[model.W2, model.b2, model.h_3])
elif args.optimizer == 'Adagrad':
if cascade == False:
optimizer = tf.train.AdagradOptimizer(
learning_rate=args.lr,
initial_accumulator_value=1e-8).minimize(model.loss)
else:
if behave_type == 'ipv':
optimizer = tf.train.AdagradOptimizer(
learning_rate=args.lr,
initial_accumulator_value=1e-8).minimize(
model.loss1,
var_list=[
model.embedding_P, model.embedding_Q,
model.h_1
])
elif behave_type == 'cart':
optimizer = tf.train.AdagradOptimizer(
learning_rate=args.lr,
initial_accumulator_value=1e-8).minimize(
model.loss2,
var_list=[model.W1, model.b1, model.h_2])
else:
optimizer = tf.train.AdagradOptimizer(
learning_rate=args.lr,
initial_accumulator_value=1e-8).minimize(
model.loss3,
var_list=[model.W2, model.b2, model.h_3])
with tf.Session(graph=model.g, config=config) as sess:
# initial training
sess.run(tf.global_variables_initializer())
if cascade and (behave_type == 'cart' or behave_type == 'buy'):
cascade_path = '/data/stu/gandahua/MBR/model_save/cascade/' + args.model + '_'
if behave_type == 'cart':
cascade_path += 'ipv'
elif behave_type == 'buy':
cascade_path += 'cart'
cascade_saver.restore(sess, cascade_path)
logging.info("--- Start training ---")
print("--- Start training ---")
# plot network
print('plot_network:', args.plot_network)
if args.plot_network:
print "writing network to TensorBoard/graphs/network"
writer = tf.summary.FileWriter('./TensorBoard/graphs/network')
writer.add_graph(sess.graph)
return 0
# dict for printing behavior type
b_dict = {}
b_dict['vb'], b_dict['cb'], b_dict['vc'] = [
'view and buy', 'cart and buy', 'view and cart'
]
# train by epoch
for epoch_count in range(args.epochs):
batch_begin = time()
if 'Multi' in args.model or 'BPR' in args.model:
samples = BatchUser.sampling_3(args, dataset, args.num_neg)
if args.dataset == 'bb2':
bs = int(args.batch_size / 4)
# samples = pkl.load(open('/data/stu/gandahua/MBR/batch_sample/multi_data_ep_%d_cs.pkl' %(epoch_count), 'rb'))
batches = BatchUser.shuffle_3(samples, bs, args)
else:
if args.b_num == 3:
bs = args.batch_size
else:
if args.b_2_type == 'cb':
bs = args.batch_size_cart
else:
bs = args.batch_size_ipv
# samples = pkl.load(open('/data/stu/gandahua/MBR/batch_sample/multi_data_ep_%d.pkl' %(epoch_count), 'rb'))
batches = BatchUser.shuffle_3(samples, bs, args)
print(
'Already generate batch, behavior is %d(%s), \n\
batch size is %d, all training entries: %d' %
(args.b_num, b_dict[args.b_2_type], bs, len(samples[0])))
batch_time = time() - batch_begin
train_begin = time()
train_loss = training_batch_3(model, sess, batches, args,
optimizer)
train_time = time() - train_begin
else:
samples = BatchUser.sampling(args, dataset, args.num_neg)
print('all training number: %d' % len(samples[0]))
# print('first label: %s' % samples[2][:20])
# print('first user: %s' % samples[0][:20])
# print('first item: %s' % samples[1][:20])
if args.dataset == 'bb2':
# samples = pkl.load(open('/data/stu/gandahua/MBR/batch_sample/single_data_ep_%d_cs.pkl' %(epoch_count), 'rb'))
if behave_type == 'ipv':
bs = int(args.batch_size_ipv / 4)
batches = BatchUser.shuffle(samples, bs, args)
elif behave_type == 'cart':
bs = int(args.batch_size_cart / 4)
batches = BatchUser.shuffle(samples, bs, args)
else:
bs = int(args.batch_size_buy / 4)
batches = BatchUser.shuffle(samples, bs, args)
else:
# samples = pkl.load(open('/data/stu/gandahua/MBR/batch_sample/single_data_ep_%d.pkl' %(epoch_count), 'rb'))
if behave_type == 'ipv':
bs = args.batch_size_ipv
batches = BatchUser.shuffle(samples, bs, args)
elif behave_type == 'cart':
bs = args.batch_size_cart
batches = BatchUser.shuffle(samples, bs, args)
else:
bs = args.batch_size_buy
batches = BatchUser.shuffle(samples, bs, args)
print('Already generate batch, batch size is %d' % bs)
batch_time = time() - batch_begin
train_begin = time()
if cascade == True:
train_loss = training_batch(model, sess, batches, args,
optimizer, behave_type)
else:
train_loss = training_batch(model, sess, batches, args,
optimizer)
train_time = time() - train_begin
print('train time: %d' % train_time)
if epoch_count % args.verbose == 0 and args.evaluate == 'yes':
if args.multiprocess == 'yes':
# save model
eval_path = '/data/stu/gandahua/MBR/model_for_eval/model_' + str(
epoch_count) + '_' + filename
# saver.save(sess, eval_path)
if epoch_count == (args.epochs - 1):
model_save_path = '/data/stu/gandahua/MBR/model_save/model_' + filename
# saver.save(sess, model_save_path)
eval_info = [
eval_path, epoch_count, batch_time, train_time,
train_loss
]
eval_queue.put(eval_info)
job_num.release()
else:
eval_begin = time()
if cascade == True:
hits, ndcgs = EvalUser.eval(model, sess, dataset,
EvalDict, args,
behave_type)
else:
hits, ndcgs = EvalUser.eval(model, sess, dataset,
EvalDict, args)
hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean()
eval_time = time() - eval_begin
logging.info(
"Epoch %d [%.1fs + %.1fs]: train_loss = %.4f [%.1fs] HR = %.4f, NDCG = %.4f"
% (epoch_count + 1, batch_time, train_time, train_loss,
eval_time, hr, ndcg))
print "Epoch %d [%.1fs + %.1fs]: train_loss = %.4f [%.1fs] HR = %.4f, NDCG = %.4f" % (
epoch_count + 1, batch_time, train_time, train_loss,
eval_time, hr, ndcg)
# save results, save model
(hr_list[base_epoch + epoch_count],
ndcg_list[base_epoch + epoch_count],
loss_list[base_epoch + epoch_count]) = (hr, ndcg,
train_loss)
model_save_path = '/data/stu/gandahua/MBR/model_save/model_' + filename
# saver.save(sess, model_save_path)
if cascade and (behave_type == 'ipv' or behave_type == 'cart'):
cascade_path = '/data/stu/gandahua/MBR/model_save/cascade/' + args.model + '_' + behave_type
cascade_saver.save(sess, cascade_path)
(dataset_ipv.num_users, dataset_ipv.num_items))
model.build_graph()
dataset = dataset_all
# recover result or not
if args.recover == 'yes':
eval_from_saved_model(model, args)
else:
if args.multiprocess == 'yes':
print('start multiprocess')
train_process = Process(target=training, args=(model, args))
train_process.start()
# evaluate
# initialize for Evaluate
EvalDict = EvalUser.gen_feed_dict(dataset[0])
cpu_num = 3
eval_pool = Pool(cpu_num)
for _ in range(cpu_num):
eval_pool.apply_async(do_eval_job, (args, EvalDict))
train_process.join()
eval_queue.close()
eval_queue.join()
else:
print('start single process')
training(model, args)
elif args.model == 'Multi_MLP':
model = Multi_MLP(dataset_all[0].num_users, dataset_all[0].num_items,