def train(self, eval_scheme=None, use_async_eval=True): model_train = self.model_train model_all_loss = self.model_all_loss conf = self.conf data_helper = self.data_helper train = data_helper.data['train'] C = data_helper.data['C'] train_time = [] for epoch in range(conf.max_epoch + 1): bb, b = 0, conf.batch_size_p cost, it = 0, 0 monitor_losses = model_all_loss.keys() costs = dict(zip(monitor_losses, np.zeros(len(monitor_losses)))) monitor_rate, monitor_it = self.monitor_rate, 0 np.random.shuffle(train) t_start = time.time() while epoch > 0 and bb < len(train): it += 1 b = bb + conf.batch_size_p if b > len(train): # get rid of uneven tail so no need to dynamically adjust batch_size_p break train_batch = train[bb:b] user_batch = train_batch[:, 0] item_batch = train_batch[:, 1] response_batch = train_batch[:, 2] cost += model_train.train_on_batch([user_batch, item_batch], [response_batch]) if random.random() < monitor_rate: monitor_it += 1 for lname, m in model_all_loss.iteritems(): costs[lname] += m.train_on_batch( [user_batch, item_batch], [response_batch]) bb = b if epoch > 0: train_time.append(time.time() - t_start) print get_cur_time(), 'epoch %d (%d it)' % (epoch, it), \ 'cost %.5f' % (cost / it if it > 0 else -1), nan_detection('cost', cost) if monitor_rate > 0: print '(', for loss in monitor_losses: print '{}={}'.format( loss, costs[loss] / monitor_it if monitor_it > 0 else -1), print ')', if eval_scheme is None: print '' else: async_eval = True \ if use_async_eval and epoch != conf.max_epoch else False try: ps[-1].join() except: pass ps = self.test(eval_scheme, use_async_eval=async_eval) print 'Training time (sec) per epoch:', np.mean(train_time)
def train(self, eval_scheme=None, use_async_eval=True): model_train = self.model_train neg_sign = self.neg_sign conf = self.conf data_helper = self.data_helper num_negatives = conf.num_negatives batch_size_p = conf.batch_size_p sample_batch_u = self.sample_batch_u train = data_helper.data['train'] C = data_helper.data['C'] train_time = [] for epoch in range(conf.max_epoch + 1): bb, b = 0, batch_size_p cost, it = 0, 0 np.random.shuffle(train) t_start = time.time() while epoch > 0 and bb < len(train): it += 1 b = bb + batch_size_p if b > len(train): # get rid of uneven tail so no need to dynamically adjust batch_size_p break train_batch_p = train[bb:b] train_batch_n = train_batch_p.repeat(num_negatives, axis=0) train_batch_n[:, 0] = sample_batch_u(num_negatives * batch_size_p) train_batch_n[:, 2] = neg_sign train_batch = np.vstack((train_batch_p, train_batch_n)) user_batch = train_batch[:, 0] item_batch = train_batch[:, 1] response_batch = train_batch[:, 2] cost += model_train.train_on_batch([user_batch, item_batch], [response_batch]) bb = b if epoch > 0: train_time.append(time.time() - t_start) print get_cur_time(), 'epoch %d (%d it)' % (epoch, it), \ 'cost %.5f' % (cost / it if it > 0 else -1), nan_detection('cost', cost) if eval_scheme is None: print '' else: async_eval = True \ if use_async_eval and epoch != conf.max_epoch else False try: ps[-1].join() except: pass ps = self.test(eval_scheme, use_async_eval=async_eval) print 'Training time (sec) per epoch:', np.mean(train_time)
def train(self, eval_scheme=None, use_async_eval=True): model_train = self.model_train conf = self.conf data_helper = self.data_helper train = data_helper.data['train'] C = data_helper.data['C'] num_negatives = conf.num_negatives sample_batch = self.sample_batch train_batch_n = np.zeros((num_negatives, 3)) response_batch = np.ones((conf.batch_size_p + num_negatives, #dummy 1 + num_negatives)) response_batch[:, 1:] = self.neg_sign train_time = [] for epoch in range(conf.max_epoch + 1): bb, b = 0, conf.batch_size_p train = group_shuffle_train(train, by='item', \ chop=conf.chop_size, iidx=self._iidx['item']) cost, it = 0, 0 t_start = time.time() while epoch > 0 and bb < len(train): it += 1 b = bb + conf.batch_size_p if b > len(train): # get rid of uneven tail so no need to dynamically adjust batch_size_p break train_batch_p = train[bb: b] train_batch_n[:, 1] = sample_batch(train_batch_n.shape[0]) train_batch = np.vstack((train_batch_p, train_batch_n)) user_batch = train_batch[:, 0] item_batch = train_batch[:, 1] cost += model_train.train_on_batch([user_batch, item_batch], [response_batch]) bb = b if epoch > 0: train_time.append(time.time() - t_start) print get_cur_time(), 'epoch %d (%d it)' % (epoch, it), \ 'cost %.5f' % (cost / it if it > 0 else -1), nan_detection('cost', cost) if eval_scheme is None: print '' else: async_eval = True \ if use_async_eval and epoch != conf.max_epoch else False try: ps[-1].join() except: pass ps = self.test(eval_scheme, use_async_eval=async_eval) print 'Training time (sec) per epoch:', np.mean(train_time)
def train(self, eval_scheme=None, use_async_eval=True): model_train = self.model_train conf = self.conf data_helper = self.data_helper batch_size_p = conf.batch_size_p train = data_helper.data['train'] C = data_helper.data['C'] group_shuffling_trick = self.group_shuffling_trick train_time = [] for epoch in range(conf.max_epoch + 1): bb, b = 0, batch_size_p cost, it = 0, 0 if group_shuffling_trick: train = group_shuffle_train(train, by='item', \ chop=conf.chop_size, iidx=self._iidx['item']) t_start = time.time() while epoch > 0 and bb < len(train): it += 1 b = bb + batch_size_p if b > len(train): # get rid of uneven tail so no need to dynamically adjust batch_size_p break if group_shuffling_trick: train_batch = train[bb: b] else: train_batch = self.group_sample(batch_size_p) user_batch = train_batch[:, 0] item_batch = train_batch[:, 1] response_batch = train_batch[:, 2] cost += model_train.train_on_batch([user_batch, item_batch], [response_batch]) bb = b if epoch > 0: train_time.append(time.time() - t_start) print get_cur_time(), 'epoch %d (%d it)' % (epoch, it), \ 'cost %.5f' % (cost / it if it > 0 else -1), nan_detection('cost', cost) if eval_scheme is None: print '' else: async_eval = True \ if use_async_eval and epoch != conf.max_epoch else False try: ps[-1].join() except: pass ps = self.test(eval_scheme, use_async_eval=async_eval) print 'Training time (sec) per epoch:', np.mean(train_time)
def train(self, eval_scheme=None, use_async_eval=True): model_train = self.model_train conf = self.conf data_helper = self.data_helper num_negatives = conf.num_negatives batch_size_p = conf.batch_size_p train = data_helper.data['train'] C = data_helper.data['C'] group_shuffling_trick = self.group_shuffling_trick if group_shuffling_trick: # Support both user and item based sampling # Only when group_shuffling_trick is True shuffle_st = conf.shuffle_st if shuffle_st.startswith('by_user'): by = 'user' elif shuffle_st.startswith('by_item'): by = 'item' else: assert False print '[INFO] sampling group based on {}'.format(by) else: print '[INFO] sampling group based on item' train_time = [] for epoch in range(conf.max_epoch + 1): bb, b = 0, batch_size_p cost, it = 0, 0 if group_shuffling_trick: train = group_shuffle_train(train, by=by, \ chop=conf.chop_size, iidx=self._iidx[by]) t_start = time.time() while epoch > 0 and bb < len(train): it += 1 b = bb + batch_size_p if b > len(train): # get rid of uneven tail so no need to dynamically adjust batch_size_p break if group_shuffling_trick: train_batch_p = train[bb: b] train_batch_n = train_batch_p.repeat(num_negatives, axis=0) if by == 'item': train_batch_n[:, 0] = self.sample_batch_u( \ num_negatives * batch_size_p) else: train_batch[:, 1] = self.sample_batch( \ num_negatives * batch_size_p) train_batch_n[:, 2] = self.neg_sign train_batch = np.vstack((train_batch_p, train_batch_n)) else: train_batch = self.group_sample_with_negs(batch_size_p, num_negatives) user_batch = train_batch[:, 0] item_batch = train_batch[:, 1] response_batch = train_batch[:, 2] cost += model_train.train_on_batch([user_batch, item_batch], [response_batch]) bb = b if epoch > 0: train_time.append(time.time() - t_start) print get_cur_time(), 'epoch %d (%d it)' % (epoch, it), \ 'cost %.5f' % (cost / it if it > 0 else -1), nan_detection('cost', cost) if eval_scheme is None: print '' else: async_eval = True \ if use_async_eval and epoch != conf.max_epoch else False try: ps[-1].join() except: pass ps = self.test(eval_scheme, use_async_eval=async_eval) print 'Training time (sec) per epoch:', np.mean(train_time)
def train(self, eval_scheme=None, use_async_eval=True): model_train = self.model_train neg_sign = self.neg_sign conf = self.conf num_negatives = conf.num_negatives batch_size = conf.batch_size_p * (1 + num_negatives) data_helper = self.data_helper sample_batch = self.sample_batch sample_batch_u = self.sample_batch_u train_p = data_helper.data['train'] C = data_helper.data['C'] chop = conf.chop_size train_time = [] for epoch in range(conf.max_epoch + 1): bb, b = 0, batch_size cost, it = 0, 0 # pre-sample if conf.shuffle_st == 'original': # sampling as in original training scheme (given link, sample neg users) np.random.shuffle(train_p) train = train_p.repeat(1 + num_negatives, axis=0) train[:, 1] = sample_batch(train.shape[0]) train[:, 2] = neg_sign train[::1 + num_negatives] = train_p elif conf.shuffle_st == 'reverse': np.random.shuffle(train_p) # sampling as in reverse training scheme (given link, sample neg users) train = train_p.repeat(1 + num_negatives, axis=0) train[:, 0] = sample_batch_u(train.shape[0]) train[:, 2] = neg_sign train[::1 + num_negatives] = train_p else: # random sampling and/or groupping train_n = train_p.repeat(num_negatives, axis=0) train_n[:, 1] = sample_batch(train_n.shape[0]) train_n[:, 2] = neg_sign train = np.vstack((train_p, train_n)) if conf.shuffle_st == 'random': np.random.shuffle(train) elif conf.shuffle_st == 'by_user': train = group_shuffle_train(train, by='user', iidx=self._iidx['user']) elif conf.shuffle_st == 'by_item': train = group_shuffle_train(train, by='item', iidx=self._iidx['item']) elif conf.shuffle_st.startswith('by_user_chop'): train = group_shuffle_train(train, by='user', chop=chop, iidx=self._iidx['user']) elif conf.shuffle_st.startswith('by_item_chop'): train = group_shuffle_train(train, by='item', chop=chop, iidx=self._iidx['item']) elif conf.shuffle_st.startswith('by_useritem_chop'): if epoch % 2 == 0: train = group_shuffle_train(train, by='user', chop=chop, iidx=self._iidx['user']) else: train = group_shuffle_train(train, by='item', chop=chop, iidx=self._iidx['item']) else: assert False, 'ERROR: unknown shuffle strategy {}'.format( conf.shuffle_st) t_start = time.time() while epoch > 0 and bb < len(train): it += 1 b = bb + batch_size if b > len(train): # get rid of uneven tail, otherwise need to dynamically adjust batch_size break train_batch = train[bb:b] user_batch = train_batch[:, 0] item_batch = train_batch[:, 1] response_batch = train_batch[:, 2] cost += model_train.train_on_batch([user_batch, item_batch], [response_batch]) bb = b if epoch > 0: train_time.append(time.time() - t_start) print get_cur_time(), 'epoch %d (%d it)' % (epoch, it), \ 'cost %.5f' % (cost / it if it > 0 else -1), nan_detection('cost', cost) if eval_scheme is None: print '' else: async_eval = True \ if use_async_eval and epoch != conf.max_epoch else False try: ps[-1].join() except: pass ps = self.test(eval_scheme, use_async_eval=async_eval) print 'Training time (sec) per epoch:', np.mean(train_time)
def run(self, model, predict_only=False, verbose=True, eval_scheme=None, batch_size=1024, use_async_eval=False): ''' predict_only: bool if True, return prediction; otherwise, evaluate normally eval_scheme: given or whole if given, evaluate with given test list if whole, predict for all items batch_size: int > 0 if want to use batch to predict <= 0 if want to use whole to predict at once use_async_eval: bool if True, will create new thread for running part of eval func, so that GPU will not be left empty ''' if eval_scheme is None: eval_scheme = self.eval_scheme self._check_eval_sheme(eval_scheme) C = self.C test_seen = self.test_seen test = self.test eval_topk = self.eval_topk user_count = self.user_count item_count = self.item_count train_true_mat = self.train_true_mat test_true_mat = self.test_true_mat train_items = self.train_items test_items = self.test_items if eval_scheme == 'given': model_dict = model test_eval_gen = test_eval([test_seen, test], eval_topk, \ model_dict, user_count, item_count, C, \ predict_only=predict_only) train_results = test_eval_gen.next() test_results = test_eval_gen.next() if verbose and not predict_only: print get_cur_time(), 'train map/auc', \ train_results['map@%s' % eval_topk], train_results['auc'], \ 'test map/auc', test_results['map@%s' % eval_topk], test_results['auc'] return train_results, test_results elif eval_scheme == 'whole': assert eval_topk > 0, \ '[ERROR] eval_top {} must > 0'.format(eval_topk) + \ 'when eval_scheme=whole' model_predict = model predict_only_local = True \ if predict_only or use_async_eval else False train_results = test_eval_mat(model_predict, eval_topk, \ train_true_mat, range(user_count), train_items, C, \ predict_only=predict_only_local, batch_size=batch_size) test_results = test_eval_mat(model_predict, eval_topk, \ test_true_mat, range(user_count), test_items, C, \ predict_only=predict_only_local, batch_size=batch_size) if verbose and not predict_only: if use_async_eval: import threading t = threading.Thread(target=evaluate_mat_thread, args=(train_results, test_results, eval_topk)) print get_cur_time(), t.start() train_results = test_results = t else: print get_cur_time(), \ 'train recall/map', train_results['recall@%s' % eval_topk], \ train_results['map@%s' % eval_topk], \ 'test recall/map', test_results['recall@%s' % eval_topk], \ test_results['map@%s' % eval_topk] return train_results, test_results
def train(self, eval_scheme=None, use_async_eval=True): model_train = self.model_train model_all_loss = self.model_all_loss neg_sign = self.neg_sign conf = self.conf num_negatives = conf.num_negatives batch_size_p = conf.batch_size_p data_helper = self.data_helper sample_batch = self.sample_batch train = data_helper.data['train'] C = data_helper.data['C'] train_time = [] for epoch in range(conf.max_epoch + 1): bb, b = 0, batch_size_p np.random.shuffle(train) cost, it = 0, 0 monitor_losses = model_all_loss.keys() costs = dict(zip(monitor_losses, np.zeros(len(monitor_losses)))) monitor_rate, monitor_it = self.monitor_rate, 0 t_start = time.time() while epoch > 0 and bb < len(train): it += 1 b = bb + batch_size_p if b > len(train): # get rid of uneven tail so no need to dynamically adjust batch_size_p break train_batch_p = train[bb:b] train_batch_n = train_batch_p.repeat(num_negatives, axis=0) train_batch_n[:, 1] = sample_batch(num_negatives * batch_size_p) train_batch_n[:, 2] = neg_sign train_batch = np.vstack((train_batch_p, train_batch_n)) user_batch = train_batch[:, 0] item_batch = train_batch[:, 1] response_batch = train_batch[:, 2] cost += model_train.train_on_batch([user_batch, item_batch], [response_batch]) if random.random() < monitor_rate: monitor_it += 1 for lname, m in model_all_loss.iteritems(): neg_sign_tmp = np.array([-1], dtype='int32') \ if lname == 'skip-gram' else 0 response_batch[batch_size_p:] = neg_sign_tmp costs[lname] += m.train_on_batch( [user_batch, item_batch], [response_batch]) bb = b if epoch > 0: train_time.append(time.time() - t_start) print get_cur_time(), 'epoch %d (%d it)' % (epoch, it), \ 'cost %.5f' % (cost / it if it > 0 else -1), nan_detection('cost', cost) if monitor_rate > 0: print '(', for loss in monitor_losses: print '{}={}'.format( loss, costs[loss] / monitor_it if monitor_it > 0 else -1), print ')', if eval_scheme is None: print '' else: async_eval = True \ if use_async_eval and epoch != conf.max_epoch else False try: ps[-1].join() except: pass ps = self.test(eval_scheme, use_async_eval=async_eval) print 'Training time (sec) per epoch:', np.mean(train_time)