def eval_from_saved_model(model, args): global hr_recover, ndcg_recover import tensorflow as tf config = tf.ConfigProto() config.gpu_options.allow_growth = True if args.model == 'Multi_GMF': EvalDict = EvalUser.init_evaluate_model(model, dataset[0]) else: EvalDict = EvalUser.init_evaluate_model(model, dataset) hits, ndcgs = [], [] loader = None eval_begin = time() eval_graph = tf.Graph() with eval_graph.as_default(): model_load_path = '/data/stu/gandahua/MBR/model_save/model_' + filename loader = tf.train.import_meta_graph(model_load_path + '.meta', clear_devices=True) with tf.Session(graph=eval_graph, config=config) as sess: loader.restore(sess, model_load_path) for idx in xrange(len(EvalDict)): if args.model == 'Multi_GMF': gtItem = dataset[0].testRatings[idx][1] predictions = sess.run('loss/inference/score_buy:0', feed_dict=EvalDict[idx]) else: gtItem = dataset.testRatings[idx][1] predictions = sess.run('loss/inference/output:0', feed_dict=EvalDict[idx]) rank = 0 rank_score = predictions[gtItem] for i in predictions: if i > rank_score: rank += 1 if rank < args.topK: hr_tmp = 1 ndcg_tmp = math.log(2) / math.log(rank + 2) else: hr_tmp = 0 ndcg_tmp = 0 hits.append(hr_tmp) ndcgs.append(ndcg_tmp) hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean() print "Final HR = %.4f, NDCG = %.4f" % (hr, ndcg) # save info to the server hr_recover, ndcg_recover = hr, ndcg
def training(model, args, behave_type=None, base_epoch=0, save=True): config = tf.ConfigProto(allow_soft_placement=True) config.gpu_options.allow_growth = True global pool, dataset, eval_queue, job_num, loss_list, hr_list, ndcg_list # initialize for Evaluate EvalDict = EvalUser.init_evaluate_model(model, dataset, args) with model.g.as_default(): with tf.name_scope('optimizer'): if args.optimizer == 'Adam': optimizer = tf.train.AdamOptimizer(learning_rate=args.lr).minimize(model.loss) elif args.optimizer == 'Adagrad': optimizer = tf.train.AdagradOptimizer(learning_rate=args.lr, initial_accumulator_value=1e-8).minimize( model.loss) else: # TODO(wgcn96): SGD optimizer = tf.train.GradientDescentOptimizer(learning_rate=args.lr).minimize(loss=model.loss) with tf.Session(graph=model.g, config=config) as sess: # initial training sess.run(tf.global_variables_initializer()) logging.info("--- Start training ---") print("--- Start training ---") # plot network then exit print('plot_network:', args.plot_network) if args.plot_network: print("writing network to TensorBoard/graphs/network") writer = tf.summary.FileWriter('TensorBoard/graphs/network') writer.add_graph(sess.graph) # return 0 # dict for printing behavior type b_dict = {'vb': 'view and buy', 'cb': 'cart and buy', 'vc': 'view and cart'} samples = BatchUser.sampling(args, dataset, args.num_neg) # TODO(wgcn96): 生成数据集不需要放在for循环里面 print('all training number: %d' % len(samples[0])) # print('first label: %s' % samples[2][:20]) # print('first user: %s' % samples[0][:20]) # print('first item: %s' % samples[1][:20]) if behave_type == 'ipv': bs = args.batch_size_ipv elif behave_type == 'cart': bs = args.batch_size_cart else: bs = args.batch_size_buy best_hr, best_ndcg, best_epoch, best_loss = 0, 0, 0, 0 # TODO(wgcn96): 记录所有循环中的最佳结果 # train by epoch for epoch_count in range(args.epochs): # TODO(wgcn96): 单行为优化 batches = BatchUser.shuffle(samples, bs, args) print('Already generate batch, batch size is %d' % bs) train_begin = time() train_loss = training_batch(model, sess, batches, args, optimizer) train_time = time() - train_begin # print('train time: %d' % train_time) if epoch_count % args.verbose == 0 and args.evaluate == 'yes': eval_begin = time() hits, ndcgs = EvalUser.eval(model, sess, dataset, EvalDict, args) hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean() eval_time = time() - eval_begin logging.info( "Epoch %d [train %.1fs]: train_loss = %.4f [test %.1fs] HR = %.4f, NDCG = %.4f" % (epoch_count + 1, train_time, train_loss, eval_time, hr, ndcg)) print("Epoch %d [train %.1fs]: train_loss = %.4f [test %.1fs] HR = %.4f, NDCG = %.4f" % ( epoch_count + 1, train_time, train_loss, eval_time, hr, ndcg)) # print('eval time : {}'.format(eval_time)) if hr >= best_hr and ndcg >= best_ndcg: best_hr = hr best_ndcg = ndcg best_epoch = epoch_count + 1 best_loss = train_loss # save results, save model (hr_list[base_epoch + epoch_count], ndcg_list[base_epoch + epoch_count], loss_list[base_epoch + epoch_count]) = (hr, ndcg, train_loss) if epoch_count == (args.epochs - 1): print("All finish, best result hr: {}, ndcg: {}, epoch: {}, train loss: {}".format(best_hr, best_ndcg, best_epoch, best_loss)) hr_list.append(best_hr) ndcg_list.append(best_ndcg) loss_list.append(best_epoch)
def training(model, args, behave_type=None, base_epoch=0, save=True): # import tensorflow as tf config = tf.ConfigProto() config.gpu_options.allow_growth = True # config.gpu_options.per_process_gpu_memory_fraction = 0.1 global pool, dataset, eval_queue, job_num, loss_list, hr_list, ndcg_list # initialize for Evaluate if args.model in ['Multi_GMF', 'Multi_MLP', 'Multi_NCF', 'CMF']: EvalDict = EvalUser.init_evaluate_model(model, dataset[0], args) else: EvalDict = EvalUser.init_evaluate_model(model, dataset, args) with model.g.as_default(): saver = tf.train.Saver() with tf.name_scope('optimizer'): if args.optimizer == 'Adam': optimizer = tf.train.AdamOptimizer( learning_rate=args.lr).minimize(model.loss) elif args.optimizer == 'Adagrad': optimizer = tf.train.AdagradOptimizer( learning_rate=args.lr, initial_accumulator_value=1e-8).minimize(model.loss) else: # TODO(wgcn96): SGD optimizer = tf.train.GradientDescentOptimizer( learning_rate=args.lr).minimize(loss=model.loss) with tf.Session(graph=model.g, config=config) as sess: # initial training sess.run(tf.global_variables_initializer()) logging.info("--- Start training ---") print("--- Start training ---") # plot network then exit print('plot_network:', args.plot_network) if args.plot_network: print("writing network to TensorBoard/graphs/network") writer = tf.summary.FileWriter('TensorBoard/graphs/network') writer.add_graph(sess.graph) # return 0 # dict for printing behavior type b_dict = { 'vb': 'view and buy', 'cb': 'cart and buy', 'vc': 'view and cart' } samples = BatchUser.sampling_3( args, dataset, args.num_neg) # TODO(wgcn96): 生成数据集不需要放在for循环里面 print('all training number: %d' % len(samples[0])) if args.b_num == 3: bs = args.batch_size elif args.b_2_type == 'cb': bs = args.batch_size_cart else: bs = args.batch_size_ipv best_hr, best_ndcg, best_epoch, best_loss = 0, 0, 0, 0 # TODO(wgcn96): 记录所有循环中的最佳结果 # train by epoch for epoch_count in range(args.epochs): # if 'Multi' in args.model or 'BPR' in args.model: # TODO(wgcn96): 这说明什么,BPR本身就用了多行为的? batches = BatchUser.shuffle_3(samples, bs, args) # print("check list {}, arg1: {}, arg2: {}, arg3: {}".format(len(batches), len(batches[0]), len(batches[1]), # len(batches[2]))) train_begin = time() train_loss = training_batch_3(model, sess, batches, args, optimizer) train_time = time() - train_begin if epoch_count % args.verbose == 0 and args.evaluate == 'yes': eval_begin = time() hits, ndcgs = EvalUser.eval(model, sess, dataset, EvalDict, args) hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean() eval_time = time() - eval_begin logging.info( "Epoch %d [train %.1fs]: train_loss = %.4f [test %.1fs] HR = %.4f, NDCG = %.4f" % (epoch_count + 1, train_time, train_loss, eval_time, hr, ndcg)) print( "Epoch %d [train %.1fs]: train_loss = %.4f [test %.1fs] HR = %.4f, NDCG = %.4f" % (epoch_count + 1, train_time, train_loss, eval_time, hr, ndcg)) # print('eval time : {}'.format(eval_time)) if hr >= best_hr and ndcg >= best_ndcg: best_hr = hr best_ndcg = ndcg best_epoch = epoch_count + 1 best_loss = train_loss # save results, save model (hr_list[base_epoch + epoch_count], ndcg_list[base_epoch + epoch_count], loss_list[base_epoch + epoch_count]) = (hr, ndcg, train_loss) model_save_path = 'Model/' if args.add_datetime == 'yes': model_save_path += datetime.datetime.now().strftime("%m%d") model_save_path += '/' if not os.path.exists(model_save_path): os.makedirs(model_save_path) model_save_path += 'model_' + filename saver.save(sess, model_save_path) if epoch_count == (args.epochs - 1): print( "All finish, best result hr: {}, ndcg: {}, epoch: {}, train loss: {}" .format(best_hr, best_ndcg, best_epoch, best_loss)) hr_list.append(best_hr) ndcg_list.append(best_ndcg) loss_list.append(best_epoch)
def eval_from_saved_model(model, args): global hr_recover, ndcg_recover # import tensorflow as tf config = tf.ConfigProto() config.gpu_options.allow_growth = True # EvalDict = EvalUser.init_evaluate_model(model, dataset[0], args) # TODO(wgcn96): why this? EvalDict = EvalUser.gen_feed_dict(dataset[0]) hits, ndcgs = [], [] loader = None eval_begin = time() eval_graph = tf.Graph() with eval_graph.as_default(): # model_load_path = 'Model/' # if args.add_datetime == 'yes': # model_load_path += datetime.datetime.now().strftime("%m%d") # model_load_path += '/' # if not os.path.exists(model_load_path): # raise Exception('file not exist error!!!') # model_load_path += 'model_' + filename + '-' + str(args.epochs-1) loader = tf.train.import_meta_graph(args.recover_path + '.meta', clear_devices=True) with tf.Session(graph=eval_graph, config=config) as sess: loader.restore(sess, args.recover_path) for idx in range(len(EvalDict)): if 'Multi' in args.model or 'Our' in args.model: # TODO(wgcn96): for multi-behavior gtItem = dataset[0].testRatings[idx][1] predictions = sess.run('loss/inference/score_buy:0', feed_dict=EvalDict[idx]) else: # TODO(wgcn96): for single-behavior gtItem = dataset.testRatings[idx][1] predictions = sess.run('loss/inference/output:0', feed_dict=EvalDict[idx]) sort_indexes = np.argsort( -predictions.reshape(predictions.shape[0])) top_K = sort_indexes[:args.topK] hr_tmp = 0 ndcg_tmp = 0 # 计算打分 if gtItem in top_K: hr_tmp = 1 pos = np.where(top_K == gtItem)[0] ndcg_tmp = math.log(2) / math.log(pos + 2) hits.append(hr_tmp) ndcgs.append(ndcg_tmp) """ rank = 0 rank_score = predictions[gtItem] for i in predictions: if i > rank_score: rank += 1 if rank < args.topK: hr_tmp = 1 ndcg_tmp = math.log(2) / math.log(rank + 2) else: hr_tmp = 0 ndcg_tmp = 0 hits.append(hr_tmp) ndcgs.append(ndcg_tmp) """ hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean() print("Final HR = %.4f, NDCG = %.4f , total time %.f" % (hr, ndcg, time() - eval_begin)) # save info to the server hr_recover, ndcg_recover = hr, ndcg
def training(model, args, behave_type=None, base_epoch=0, save=True): # import tensorflow as tf config = tf.ConfigProto() config.gpu_options.allow_growth = True # config.gpu_options.per_process_gpu_memory_fraction = 0.1 global pool, dataset, eval_queue, job_num, loss_list, hr_list, ndcg_list # initialize for Evaluate EvalDict = EvalUser.init_evaluate_model(model, dataset[0], args) with model.g.as_default(): saver = tf.train.Saver() with tf.name_scope('optimizer'): if args.optimizer == 'Adam': optimizer = tf.train.AdamOptimizer( learning_rate=args.lr).minimize(model.loss) elif args.optimizer == 'Adagrad': optimizer = tf.train.AdagradOptimizer( learning_rate=args.lr, initial_accumulator_value=1e-8).minimize(model.loss) else: # TODO(wgcn96): SGD optimizer = tf.train.GradientDescentOptimizer( learning_rate=args.lr).minimize(loss=model.loss) with tf.Session(graph=model.g, config=config) as sess: # initial training sess.run(tf.global_variables_initializer()) logging.info("--- Start training ---") print("--- Start training ---") # plot network then exit print('plot_network:', args.plot_network) if args.plot_network: print("writing network to TensorBoard/graphs/network") writer = tf.summary.FileWriter('TensorBoard/graphs/network') writer.add_graph(sess.graph) return 0 # dict for printing behavior type b_dict = { 'vb': 'view and buy', 'cb': 'cart and buy', 'vc': 'view and cart' } samples = BatchUser.sampling_3( args, dataset, args.num_neg) # TODO(wgcn96): 生成数据集不需要放在for循环里面 print('all training number: %d' % len(samples[0])) if args.b_num == 3: bs = args.batch_size elif args.b_2_type == 'cb': bs = args.batch_size_cart else: bs = args.batch_size_ipv best_hr, best_ndcg, best_epoch, best_loss = 0, 0, 0, 0 # TODO(wgcn96): 记录所有循环中的最佳结果 # train by epoch for epoch_count in range(args.epochs): batches = BatchUser.shuffle_3(samples, bs, args) # print('Already generate batch, behavior is %d(%s), \n\ # batch size is %d, all training entries: %d' % ( # args.b_num, b_dict[args.b_2_type], bs, len(samples[0]))) train_begin = time() train_loss = training_batch_3(model, sess, batches, args, optimizer) train_time = time() - train_begin if epoch_count % args.verbose == 0 and args.evaluate == 'yes': eval_begin = time() hits, ndcgs = EvalUser.eval(model, sess, dataset, EvalDict, args) hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean() eval_time = time() - eval_begin logging.info( "Epoch %d [train %.1fs]: train_loss = %.4f [test %.1fs] HR = %.4f, NDCG = %.4f" % (epoch_count + 1, train_time, train_loss, eval_time, hr, ndcg)) print( "Epoch %d [train %.1fs]: train_loss = %.4f [test %.1fs] HR = %.4f, NDCG = %.4f" % (epoch_count + 1, train_time, train_loss, eval_time, hr, ndcg)) # print('eval time : {}'.format(eval_time)) if hr >= best_hr and ndcg >= best_ndcg: best_hr = hr best_ndcg = ndcg best_epoch = epoch_count + 1 best_loss = train_loss model_save_path = 'Model/' + args.process_name + '_best' # save best model saver.save(sess, model_save_path) # save best result (hr_list[base_epoch + epoch_count], ndcg_list[base_epoch + epoch_count], loss_list[base_epoch + epoch_count]) = (hr, ndcg, train_loss) if epoch_count == (args.epochs - 1): print( "All finish, best result hr: {}, ndcg: {}, epoch: {}, train loss: {}" .format(best_hr, best_ndcg, best_epoch, best_loss)) hr_list.append(best_hr) ndcg_list.append(best_ndcg) loss_list.append(best_epoch) prefix = '' if args.add_datetime == 'yes': prefix += datetime.datetime.now().strftime("%m%d") prefix += '/' if not os.path.exists(prefix): os.makedirs(prefix) with open(prefix + args.save_file, 'a+') as fo: fo.write("---------------\n") # fo.write("loss_func: %s\n" % args.loss_func) fo.write("batch_size: %s\n" % args.batch_size) fo.write("layer_num: %s\n" % args.layer_num) fo.write("lr: %s\n" % args.lr) fo.write("epoch: %s\n" % args.epochs) fo.write("optimizer: %s\n" % args.optimizer) fo.write("reg: %s\n" % args.regs) fo.write( "All finish, best result hr: {}, ndcg: {}, epoch: {}, train loss: {}\n" .format(best_hr, best_ndcg, best_epoch, best_loss))
def eval_from_saved_model(model, args): global hr_recover, ndcg_recover # import tensorflow as tf config = tf.ConfigProto() config.gpu_options.allow_growth = True # EvalDict = EvalUser.init_evaluate_model(model, dataset[0], args) # TODO(wgcn96): why this? EvalDict = EvalUser.gen_feed_dict(dataset[0]) hits, ndcgs = [], [] loader = None eval_graph = tf.Graph() with eval_graph.as_default(): # model_load_path = 'Model/model_' + filename loader = tf.train.import_meta_graph(args.recover_path + '.meta', clear_devices=True) with tf.Session(graph=eval_graph, config=config) as sess: loader.restore(sess, args.recover_path) for idx in range(len(EvalDict)): gtItem = dataset[0].testRatings[idx][1] predictions = sess.run('loss/errors/inference/score_buy:0', feed_dict=EvalDict[idx]) sort_indexes = np.argsort( -predictions.reshape(predictions.shape[0])) top_K = sort_indexes[:args.topK] hr_tmp = 0 ndcg_tmp = 0 # 计算打分 if gtItem in top_K: hr_tmp = 1 pos = np.where(top_K == gtItem)[0] ndcg_tmp = math.log(2) / math.log(pos + 2) hits.append(hr_tmp) ndcgs.append(ndcg_tmp) """ rank = 0 rank_score = predictions[gtItem] for i in predictions: if i > rank_score: rank += 1 if rank < args.topK: hr_tmp = 1 ndcg_tmp = math.log(2) / math.log(rank + 2) else: hr_tmp = 0 ndcg_tmp = 0 hits.append(hr_tmp) ndcgs.append(ndcg_tmp) """ hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean() print("Final HR = %.4f, NDCG = %.4f" % (hr, ndcg)) # save info to the server hr_recover, ndcg_recover = hr, ndcg
def training(model, dataset, args, saver = None): # saver is an object to save pq with tf.Session() as sess: # pretrain nor not sess.run(tf.global_variables_initializer()) logging.info("initialized") print "initialized" writer = tf.summary.FileWriter('./graphs', sess.graph) # initialize for training batches if args.batch_gen == "fixed": if args.model == "FISM": samples = BatchItem.sampling(args, dataset, args.num_neg) else: samples = BatchUser.sampling(args, dataset, args.num_neg) # initialize for Evaluate if args.model == "FISM": EvalDict = EvalItem.init_evaluate_model(model, dataset) else: EvalDict = EvalUser.init_evaluate_model(model, dataset) # train by epoch for epoch_count in range(args.epochs): batch_begin = time() if args.model == "FISM": if args.batch_gen == "unfixed": samples = BatchItem.sampling(args, dataset, args.num_neg) batches = BatchItem.shuffle(samples, args.batch_size, dataset) else : if args.batch_gen == "unfixed": samples = BatchUser.sampling(args, dataset, args.num_neg) batches = BatchUser.shuffle(samples, args.batch_size) batch_time = time() - batch_begin train_begin = time() training_batch(model, sess, batches, args) train_time = time() - train_begin if epoch_count % args.verbose == 0: if args.train_loss: loss_begin = time() train_loss = training_loss(model, sess, batches, args) loss_time = time() - loss_begin else: loss_time, train_loss = 0, 0 eval_begin = time() if args.model == "FISM": hits, ndcgs, losses = EvalItem.eval(model, sess, dataset, EvalDict) else: hits, ndcgs, losses = EvalUser.eval(model, sess, dataset, EvalDict) hr, ndcg, test_loss = np.array(hits).mean(), np.array(ndcgs).mean(), np.array(losses).mean() eval_time = time() - eval_begin logging.info( "Epoch %d [%.1fs + %.1fs]: HR = %.4f, NDCG = %.4f, loss = %.4f [%.1fs] train_loss = %.4f [%.1fs]" % ( epoch_count, batch_time, train_time, hr, ndcg, test_loss, eval_time, train_loss, loss_time)) print "Epoch %d [%.1fs + %.1fs]: HR = %.4f, NDCG = %.4f, loss = %.4f [%.1fs] train_loss = %.4f [%.1fs]" % ( epoch_count, batch_time, train_time, hr, ndcg, test_loss, eval_time, train_loss, loss_time) if saver != None: saver.save(model, sess)
def training(model, args, behave_type=None, base_epoch=0, save=True): cascade = args.model == 'NCF_FC' or args.model == 'GMF_FC' if cascade: cascade_path = None import tensorflow as tf config = tf.ConfigProto() config.gpu_options.allow_growth = True config.gpu_options.per_process_gpu_memory_fraction = 0.1 global pool, dataset, eval_queue, job_num, loss_list, hr_list, ndcg_list # initialize for Evaluate if args.multiprocess == 'no': if args.model in ['Multi_GMF', 'Multi_MLP', 'Multi_NCF']: EvalDict = EvalUser.init_evaluate_model(model, dataset[0], args) else: EvalDict = EvalUser.init_evaluate_model(model, dataset, args) with model.g.as_default(): saver = tf.train.Saver() if cascade: cascade_saver = tf.train.Saver() with tf.name_scope('optimizer'): if args.optimizer == 'Adam': if cascade == False: optimizer = tf.train.AdamOptimizer( learning_rate=args.lr).minimize(model.loss) else: if behave_type == 'ipv': optimizer = tf.train.AdamOptimizer( learning_rate=args.lr).minimize( model.loss1, var_list=[ model.embedding_P, model.embedding_Q, model.h_1 ]) elif behave_type == 'cart': optimizer = tf.train.AdamOptimizer( learning_rate=args.lr).minimize( model.loss2, var_list=[model.W1, model.b1, model.h_2]) else: optimizer = tf.train.AdamOptimizer( learning_rate=args.lr).minimize( model.loss3, var_list=[model.W2, model.b2, model.h_3]) elif args.optimizer == 'Adagrad': if cascade == False: optimizer = tf.train.AdagradOptimizer( learning_rate=args.lr, initial_accumulator_value=1e-8).minimize(model.loss) else: if behave_type == 'ipv': optimizer = tf.train.AdagradOptimizer( learning_rate=args.lr, initial_accumulator_value=1e-8).minimize( model.loss1, var_list=[ model.embedding_P, model.embedding_Q, model.h_1 ]) elif behave_type == 'cart': optimizer = tf.train.AdagradOptimizer( learning_rate=args.lr, initial_accumulator_value=1e-8).minimize( model.loss2, var_list=[model.W1, model.b1, model.h_2]) else: optimizer = tf.train.AdagradOptimizer( learning_rate=args.lr, initial_accumulator_value=1e-8).minimize( model.loss3, var_list=[model.W2, model.b2, model.h_3]) with tf.Session(graph=model.g, config=config) as sess: # initial training sess.run(tf.global_variables_initializer()) if cascade and (behave_type == 'cart' or behave_type == 'buy'): cascade_path = '/data/stu/gandahua/MBR/model_save/cascade/' + args.model + '_' if behave_type == 'cart': cascade_path += 'ipv' elif behave_type == 'buy': cascade_path += 'cart' cascade_saver.restore(sess, cascade_path) logging.info("--- Start training ---") print("--- Start training ---") # plot network print('plot_network:', args.plot_network) if args.plot_network: print "writing network to TensorBoard/graphs/network" writer = tf.summary.FileWriter('./TensorBoard/graphs/network') writer.add_graph(sess.graph) return 0 # dict for printing behavior type b_dict = {} b_dict['vb'], b_dict['cb'], b_dict['vc'] = [ 'view and buy', 'cart and buy', 'view and cart' ] # train by epoch for epoch_count in range(args.epochs): batch_begin = time() if 'Multi' in args.model or 'BPR' in args.model: samples = BatchUser.sampling_3(args, dataset, args.num_neg) if args.dataset == 'bb2': bs = int(args.batch_size / 4) # samples = pkl.load(open('/data/stu/gandahua/MBR/batch_sample/multi_data_ep_%d_cs.pkl' %(epoch_count), 'rb')) batches = BatchUser.shuffle_3(samples, bs, args) else: if args.b_num == 3: bs = args.batch_size else: if args.b_2_type == 'cb': bs = args.batch_size_cart else: bs = args.batch_size_ipv # samples = pkl.load(open('/data/stu/gandahua/MBR/batch_sample/multi_data_ep_%d.pkl' %(epoch_count), 'rb')) batches = BatchUser.shuffle_3(samples, bs, args) print( 'Already generate batch, behavior is %d(%s), \n\ batch size is %d, all training entries: %d' % (args.b_num, b_dict[args.b_2_type], bs, len(samples[0]))) batch_time = time() - batch_begin train_begin = time() train_loss = training_batch_3(model, sess, batches, args, optimizer) train_time = time() - train_begin else: samples = BatchUser.sampling(args, dataset, args.num_neg) print('all training number: %d' % len(samples[0])) # print('first label: %s' % samples[2][:20]) # print('first user: %s' % samples[0][:20]) # print('first item: %s' % samples[1][:20]) if args.dataset == 'bb2': # samples = pkl.load(open('/data/stu/gandahua/MBR/batch_sample/single_data_ep_%d_cs.pkl' %(epoch_count), 'rb')) if behave_type == 'ipv': bs = int(args.batch_size_ipv / 4) batches = BatchUser.shuffle(samples, bs, args) elif behave_type == 'cart': bs = int(args.batch_size_cart / 4) batches = BatchUser.shuffle(samples, bs, args) else: bs = int(args.batch_size_buy / 4) batches = BatchUser.shuffle(samples, bs, args) else: # samples = pkl.load(open('/data/stu/gandahua/MBR/batch_sample/single_data_ep_%d.pkl' %(epoch_count), 'rb')) if behave_type == 'ipv': bs = args.batch_size_ipv batches = BatchUser.shuffle(samples, bs, args) elif behave_type == 'cart': bs = args.batch_size_cart batches = BatchUser.shuffle(samples, bs, args) else: bs = args.batch_size_buy batches = BatchUser.shuffle(samples, bs, args) print('Already generate batch, batch size is %d' % bs) batch_time = time() - batch_begin train_begin = time() if cascade == True: train_loss = training_batch(model, sess, batches, args, optimizer, behave_type) else: train_loss = training_batch(model, sess, batches, args, optimizer) train_time = time() - train_begin print('train time: %d' % train_time) if epoch_count % args.verbose == 0 and args.evaluate == 'yes': if args.multiprocess == 'yes': # save model eval_path = '/data/stu/gandahua/MBR/model_for_eval/model_' + str( epoch_count) + '_' + filename # saver.save(sess, eval_path) if epoch_count == (args.epochs - 1): model_save_path = '/data/stu/gandahua/MBR/model_save/model_' + filename # saver.save(sess, model_save_path) eval_info = [ eval_path, epoch_count, batch_time, train_time, train_loss ] eval_queue.put(eval_info) job_num.release() else: eval_begin = time() if cascade == True: hits, ndcgs = EvalUser.eval(model, sess, dataset, EvalDict, args, behave_type) else: hits, ndcgs = EvalUser.eval(model, sess, dataset, EvalDict, args) hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean() eval_time = time() - eval_begin logging.info( "Epoch %d [%.1fs + %.1fs]: train_loss = %.4f [%.1fs] HR = %.4f, NDCG = %.4f" % (epoch_count + 1, batch_time, train_time, train_loss, eval_time, hr, ndcg)) print "Epoch %d [%.1fs + %.1fs]: train_loss = %.4f [%.1fs] HR = %.4f, NDCG = %.4f" % ( epoch_count + 1, batch_time, train_time, train_loss, eval_time, hr, ndcg) # save results, save model (hr_list[base_epoch + epoch_count], ndcg_list[base_epoch + epoch_count], loss_list[base_epoch + epoch_count]) = (hr, ndcg, train_loss) model_save_path = '/data/stu/gandahua/MBR/model_save/model_' + filename # saver.save(sess, model_save_path) if cascade and (behave_type == 'ipv' or behave_type == 'cart'): cascade_path = '/data/stu/gandahua/MBR/model_save/cascade/' + args.model + '_' + behave_type cascade_saver.save(sess, cascade_path)
(dataset_ipv.num_users, dataset_ipv.num_items)) model.build_graph() dataset = dataset_all # recover result or not if args.recover == 'yes': eval_from_saved_model(model, args) else: if args.multiprocess == 'yes': print('start multiprocess') train_process = Process(target=training, args=(model, args)) train_process.start() # evaluate # initialize for Evaluate EvalDict = EvalUser.gen_feed_dict(dataset[0]) cpu_num = 3 eval_pool = Pool(cpu_num) for _ in range(cpu_num): eval_pool.apply_async(do_eval_job, (args, EvalDict)) train_process.join() eval_queue.close() eval_queue.join() else: print('start single process') training(model, args) elif args.model == 'Multi_MLP': model = Multi_MLP(dataset_all[0].num_users, dataset_all[0].num_items,