def multinomial_train(self, X, y, C, w0=None, b0=None, eta=0.5, max_iterations=1000): """ Inputs: - X: training features, a N-by-D numpy array, where N is the number of training points and D is the dimensionality of features - y: multiclass training labels, a N dimensional numpy array where N is the number of training points, indicating the labels of training data - C: number of classes in the data - eta: learning rate - max_iterations: maximum number for iterations to perform Returns: - w: C-by-D weight matrix of multinomial logistic regression, where C is the number of classes and D is the dimensionality of features. - b: bias vector of length C, where C is the number of classes """ N, D = X.shape w = np.zeros((C, D)) if w0 is not None: w = w0 assert w.shape == ( C, D), f"check your w0, its dimension should be: {(C, D)}" b = np.zeros(C) if b0 is not None: b = b0 W = np.hstack( (b.reshape(-1, 1), w)).T # shape (D+1, C), you could ignore this, I use this # shape(N, D + 1), to visulize vectorization when implement it X = np.insert(X, 0, 1, axis=1) Y = one_hot(y, nb_class=C) P = softmax(X @ W) - Y tol = 1e-5 for it in range(max_iterations): idx = minibatch(X) W_prev = W W = W - eta / N * X[idx, :].T @ P[idx, :] P[idx, :] = softmax(X[idx, :] @ W) - Y[idx, :] if np.max(np.abs(W_prev - W)) < tol: print(f"Converged in {it} iters.") break w = W.T[:, 1:] b = W.T[:, 0] assert w.shape == (C, D) assert b.shape == (C, ) self.w = w self.b = b
def run_one_epoch(self, ep, train_ds, valid_ds, batch_size): losses = [] i = 0 score = 0 for xbatch, ybatch in minibatch(train_ds, batch_size): i += 1 word_seq, sequence_len = pad_sequences(xbatch) target_seq, _ = pad_sequences(ybatch) # build feed dictionary feed = { self.word_ids: word_seq, self.labels: target_seq, self.sequence_lengths: sequence_len, self.learning_rate: self.lr, self.keep_dropout_rate: self.kdr } _, train_loss = self.sess.run([self.train_op, self.loss], feed_dict=feed) losses += [train_loss] if i % 10 == 0: print('ep:', ep, 'iter:', i, 'loss:', np.mean(losses)) if i % 50 == 0: acc_score, _ = self.run_validation(valid_ds, batch_size) print('accuracy', acc_score) if acc_score == 0: acc_score, _ = self.run_validation(valid_ds, batch_size) metrics = {} metrics['acc'] = acc_score return metrics
def BPR_train_original(dataset, recommend_model, loss_class, epoch, neg_k=1, w=None): Recmodel = recommend_model Recmodel.train() bpr: utils.BPRLoss = loss_class allusers = list(range(dataset.n_users)) S, sam_time = utils.UniformSample_original(allusers, dataset) print(f"BPR[sample time][{sam_time[0]:.1f}={sam_time[1]:.2f}+{sam_time[2]:.2f}]") users = torch.Tensor(S[:, 0]).long() posItems = torch.Tensor(S[:, 1]).long() negItems = torch.Tensor(S[:, 2]).long() users = users.to(world.device) posItems = posItems.to(world.device) negItems = negItems.to(world.device) users, posItems, negItems = utils.shuffle(users, posItems, negItems) total_batch = len(users) // world.config['bpr_batch_size'] + 1 aver_loss = 0. for (batch_i, (batch_users, batch_pos, batch_neg)) in enumerate(utils.minibatch(users, posItems, negItems, batch_size=world.config['bpr_batch_size'])): cri = bpr.stageOne(batch_users, batch_pos, batch_neg) aver_loss += cri if world.tensorboard: w.add_scalar(f'BPRLoss/BPR', cri, epoch * int(len(users) / world.config['bpr_batch_size']) + batch_i) aver_loss = aver_loss / total_batch return f"[BPR[aver loss{aver_loss:.3e}]"
def BPR_train_original(cuda_loader, recommend_model, loss_class, epoch, neg_k=1, w=None): Recmodel = recommend_model Recmodel.train() bpr: utils.BPRLoss = loss_class users, posItems, negItems = cuda_loader.get_train_data_at(epoch) users, posItems, negItems = utils.shuffle(users, posItems, negItems) total_batch = len(users) // world.config['bpr_batch_size'] + 1 aver_loss = 0. for (batch_i, (batch_users, batch_pos, batch_neg)) in enumerate( utils.minibatch(users, posItems, negItems, batch_size=world.config['bpr_batch_size'])): cri = bpr.stageOne(batch_users, batch_pos, batch_neg) aver_loss += cri if world.tensorboard: w.add_scalar( f'BPRLoss/BPR', cri, epoch * int(len(users) / world.config['bpr_batch_size']) + batch_i) aver_loss = aver_loss / total_batch return f"[BPR[aver loss{aver_loss:.3e}]"
def _run_epoch_dev(self, sess, inputs, labels, shuffle): minibatches = utils.minibatch(self.config.batch_size, inputs, labels=labels, shuffle=shuffle) mean_loss = 0 for i, (inputs_batch, labels_batch) in enumerate(minibatches): loss = self._loss_on_batch(sess, inputs_batch, labels_batch) mean_loss += loss mean_loss /= (i + 1) return mean_loss
def _run_epoch_pred(self, sess, inputs): minibatches = utils.minibatch(self.config.batch_size, inputs, labels=None, shuffle=False) list_y_prob = [] for i, inputs_batch in enumerate(minibatches): prob = self._predict_on_batch(sess, inputs_batch) list_y_prob.append(prob) y_prob = np.vstack(list_y_prob) return y_prob
def Test(dataset, Recmodel, epoch, cold=False, w=None): u_batch_size = world.config['test_u_batch_size'] if cold: testDict: dict = dataset.coldTestDict else: testDict: dict = dataset.testDict Recmodel = Recmodel.eval() max_K = max(world.topks) results = { 'precision': np.zeros(len(world.topks)), 'recall': np.zeros(len(world.topks)), 'ndcg': np.zeros(len(world.topks)) } with torch.no_grad(): users = list(testDict.keys()) try: assert u_batch_size <= len(users) / 10 except AssertionError: print( f"test_u_batch_size is too big for this dataset, try a small one {len(users) // 10}" ) users_list = [] rating_list = [] groundTrue_list = [] total_batch = len(users) // u_batch_size + 1 for batch_users in utils.minibatch(users, batch_size=u_batch_size): allPos = dataset.getUserPosItems(batch_users) groundTrue = [testDict[u] for u in batch_users] batch_users_gpu = torch.Tensor(batch_users).long() batch_users_gpu = batch_users_gpu.to(world.device) rating = Recmodel.getUsersRating(batch_users_gpu) exclude_index = [] exclude_items = [] for range_i, items in enumerate(allPos): exclude_index.extend([range_i] * len(items)) exclude_items.extend(items) rating[exclude_index, exclude_items] = -(1 << 10) _, rating_K = torch.topk(rating, k=max_K) del rating users_list.append(batch_users) rating_list.append(rating_K.cpu()) groundTrue_list.append(groundTrue) assert total_batch == len(users_list) X = zip(rating_list, groundTrue_list) pre_results = [] for x in X: pre_results.append(test_one_batch(x)) for result in pre_results: results['recall'] += result['recall'] results['precision'] += result['precision'] results['ndcg'] += result['ndcg'] results['recall'] /= float(len(users)) results['precision'] /= float(len(users)) results['ndcg'] /= float(len(users)) print(results) return results
def binary_train(self, X, y, w0=None, b0=None, eta=0.5, max_iterations=1000): """ Inputs: - X: training features, a N-by-D numpy array, where N is the number of training points and D is the dimensionality of features - y: binary training labels, a N dimensional numpy array where N is the number of training points, indicating the labels of training data - eta: learning rate Returns: - w: D-dimensional vector, a numpy array which is the weight vector of logistic regression - b: scalar, which is the bias of logistic regression Find the optimal parameters w and b for inputs X and y. Use the Mini-batch gradient descent. """ N, D = X.shape assert len(np.unique(y)) == 2 y = np.array(y) w = np.zeros(D) if w0 is not None: w = w0 assert len(w) == D, f"w0 length does not match number of features: {D}" b = 0 if b0 is not None: b = b0 w = np.append(b, w) X = np.insert(X, 0, 1, axis=1) tol = 1e-7 for i in range(max_iterations): idx = minibatch(X) w_next = ( w + eta / N * np.dot(X[idx, :].T, (y[idx] - sigmoid(np.dot(X[idx, :], w))))) if sp.spatial.distance.euclidean(w, w_next) < tol: print(f"Converged in {i} iterations.") break w = w_next b = w[0] w = w[1:] assert w.shape == (D, ) self.w = w self.b = b return w, b
def BPR_train_original(dataset, recommend_model, loss_class, epoch, w=None): """Traininf procedure for uniform BPR Args: dataset (BasicDatset): defined in dataloader.BasicDataset, loaded in register.py recommend_model (PairWiseModel): recommend model with small dim loss_class (utils.BPRLoss): class to get BPR training loss, and BackPropagation epoch (int): w (SummaryWriter, optional): Tensorboard writer Returns: str: summary of aver loss and running time for one epoch """ global item_count if item_count is None: item_count = torch.zeros(dataset.m_items) Recmodel = recommend_model Recmodel.train() bpr: utils.BPRLoss = loss_class S = UniformSample_original(dataset) S = torch.from_numpy(S).long() # print(f"BPR[sample time][{sam_time[0]:.1f}={sam_time[1]:.2f}+{sam_time[2]:.2f}]") users = S[:, 0] posItems = S[:, 1] negItems = S[:, 2] users = users.to(world.DEVICE) posItems = posItems.to(world.DEVICE) negItems = negItems.to(world.DEVICE) users, posItems, negItems = utils.shuffle(users, posItems, negItems) total_batch = len(users) // world.config['bpr_batch_size'] + 1 aver_loss = 0. for (batch_i, (batch_users, batch_pos, batch_neg)) in enumerate( utils.minibatch(users, posItems, negItems, batch_size=world.config['bpr_batch_size'])): if world.ALLDATA: print(world.ALLDATA) weights = utils.getTestweight(batch_users, batch_pos, dataset) else: weights = None item_count[batch_neg] += 1 cri = bpr.stageOne(batch_users, batch_pos, batch_neg, weights=weights) aver_loss += cri if world.tensorboard: w.add_scalar( f'BPRLoss/BPR', cri, epoch * int(len(users) / world.config['bpr_batch_size']) + batch_i) aver_loss = aver_loss / total_batch return f"[BPR[aver loss{aver_loss:.3e}]"
def fit(self, X_train, n_epochs=20, batchsize=128, verbose=True): """ Fit the VAE to a training dataset. Parameters ---------- X_train : numpy array of shape (n_ex, in_rows, in_cols, in_ch) The input volume n_epochs : int (default: 20) The maximum number of training epochs to run batchsize : int (default: 128) The desired number of examples in each training batch verbose : bool (default: True) Print batch information during training """ self.verbose = verbose self.n_epochs = n_epochs self.batchsize = batchsize _, self.in_rows, self.in_cols, self.in_ch = X_train.shape self.N = self.in_rows * self.in_cols * self.in_ch prev_loss = np.inf for i in range(n_epochs): loss, estart = 0.0, time() batch_generator, nb = minibatch(X_train, batchsize, shuffle=True) # TODO: parallelize inner loop for j, b_ix in enumerate(batch_generator): bsize, bstart = len(b_ix), time() X_batch = X_train[b_ix] X_batch_col = X_train[b_ix].reshape(bsize, -1) X_recon = self.forward(X_batch) t_mean = self.derived_variables["t_mean"] t_log_var = self.derived_variables["t_log_var"] self.backward(X_batch, X_recon) batch_loss = self.loss(X_batch_col, X_recon, t_mean, t_log_var) loss += batch_loss self.update(batch_loss) if self.verbose: fstr = "\t[Batch {}/{}] Train loss: {:.3f} ({:.1f}s/batch)" print(fstr.format(j + 1, nb, batch_loss, time() - bstart)) loss /= nb fstr = "[Epoch {}] Avg. loss: {:.3f} Delta: {:.3f} ({:.2f}m/epoch)" print(fstr.format(i + 1, loss, prev_loss - loss, (time() - estart) / 60.0)) prev_loss = loss
def _run_epoch_train(self, sess, inputs, labels, shuffle): n_minibatches = len(np.arange(0, len(inputs), self.config.batch_size)) prog = tf.keras.utils.Progbar(target=n_minibatches) minibatches = utils.minibatch(self.config.batch_size, inputs, labels=labels, shuffle=shuffle) mean_loss = 0 for i, (inputs_batch, labels_batch) in enumerate(minibatches): loss = self._train_on_batch(sess, inputs_batch, labels_batch) mean_loss += loss force = (i + 1) == n_minibatches prog.update(i + 1, [('train_loss', loss)], force=force) mean_loss /= (i + 1) return mean_loss
def Distill_DNS(dataset, student, sampler, loss_class, epoch, w=None): """Training procedure for distillation methods Args: dataset (BasicDatset): defined in dataloader.BasicDataset, loaded in register.py student (PairWiseModel): recommend model with small dim sampler (DS|DL|RD|CD): tons of distill methods defined in sample.py loss_class (utils.BPRLoss): class to get BPR training loss, and BackPropagation epoch (int): w (SummaryWriter, optional): Tensorboard writer Returns: str: summary of aver loss and running time for one epoch """ sampler: DistillLogits bpr: utils.BPRLoss = loss_class student.train() aver_loss = 0 with timer(name='sampling'): S = sampler.PerSample() S = torch.from_numpy(S).long().to(world.DEVICE) users, posItems, negItems = S[:, 0], S[:, 1], S[:, 2:] users, posItems, negItems = utils.shuffle(users, posItems, negItems) total_batch = len(users) // world.config['bpr_batch_size'] + 1 for (batch_i, (batch_users, batch_pos, batch_neg)) in enumerate( utils.minibatch(users, posItems, negItems, batch_size=world.config['bpr_batch_size'])): with timer(name="KD"): batch_neg, weights, KD_loss = sampler.Sample( batch_users, batch_pos, batch_neg, epoch) with timer(name="BP"): cri = bpr.stageOne(batch_users, batch_pos, batch_neg, add_loss=KD_loss, weights=weights) aver_loss += cri # Additional section------------------------ # # ------------------------------------------ if world.tensorboard: w.add_scalar( f'BPRLoss/BPR', cri, epoch * int(len(users) / world.config['bpr_batch_size']) + batch_i) aver_loss = aver_loss / total_batch info = f"{timer.dict()}[BPR loss{aver_loss:.3e}]" timer.zero() return info
def BPR_train_DNS_neg(dataset, recommend_model, loss_class, epoch, w=None): """Traininf procedure for DNS algorithms Args: dataset (BasicDatset): defined in dataloader.BasicDataset, loaded in register.py recommend_model (PairWiseModel): recommend model with small dim loss_class (utils.BPRLoss): class to get BPR training loss, and BackPropagation epoch (int): w (SummaryWriter, optional): Tensorboard writer Returns: str: summary of aver loss and running time for one epoch """ Recmodel: PairWiseModel = recommend_model Recmodel.train() bpr: utils.BPRLoss = loss_class S = UniformSample_DNS(dataset, world.DNS_K) users = torch.Tensor(S[:, 0]).long() posItems = torch.Tensor(S[:, 1]).long() negItems = torch.Tensor(S[:, 2:]).long() users = users.to(world.DEVICE) posItems = posItems.to(world.DEVICE) negItems = negItems.to(world.DEVICE) users, posItems, negItems = utils.shuffle(users, posItems, negItems) total_batch = len(users) // world.config['bpr_batch_size'] + 1 aver_loss = 0. for (batch_i, (batch_users, batch_pos, batch_neg)) in enumerate( utils.minibatch(users, posItems, negItems, batch_size=world.config['bpr_batch_size'])): if world.ALLDATA: weights = utils.getTestweight(batch_users, batch_pos, dataset) else: weights = None batch_neg = DNS_sampling_neg(batch_users, batch_neg, dataset, Recmodel) cri = bpr.stageOne(batch_users, batch_pos, batch_neg, weights=weights) aver_loss += cri if world.tensorboard: w.add_scalar( f'BPRLoss/BPR', cri, epoch * int(len(users) / world.config['bpr_batch_size']) + batch_i) # print(f"DNS[sampling][{time()-DNS_time:.1f}={DNS_time1:.2f}+{DNS_time2:.2f}]") aver_loss = aver_loss / total_batch return f"[BPR[aver loss{aver_loss:.3e}]"
def Popularity_Bias(dataset, Recmodel, valid=True): u_batch_size = world.config['test_u_batch_size'] dataset: utils.BasicDataset testDict: dict Popularity = np.zeros(dataset.m_items, ).astype('int') if valid: testDict = dataset.validDict else: testDict = dataset.testDict Recmodel: model.LightGCN perUser = int(dataset.trainDataSize / dataset.n_users) # eval mode with no dropout Recmodel.eval() max_K = perUser user_topk = np.zeros((dataset.n_users, max_K), dtype=int) with torch.no_grad(): users = list(testDict.keys()) rating_list = [] # ratings = [] total_batch = len(users) // u_batch_size + 1 for batch_users in utils.minibatch(users, batch_size=u_batch_size): allPos = dataset.getUserPosItems(batch_users) batch_users_gpu = torch.Tensor(batch_users).long() batch_users_gpu = batch_users_gpu.to(world.DEVICE) rating = Recmodel.getUsersRating(batch_users_gpu) rating = rating.cpu() exclude_index = [] exclude_items = [] if not world.TESTDATA: for range_i, items in enumerate(allPos): exclude_index.extend([range_i] * len(items)) exclude_items.extend(items) rating[exclude_index, exclude_items] = -1e5 _, rating_K = torch.topk(rating, k=max_K) del rating rating_K = rating_K.numpy().astype('int') user_topk[batch_users] = rating_K for i in range(len(batch_users)): Popularity[rating_K[i]] += 1 return Popularity.astype('int'), user_topk.astype("int")
def BPR_train_original(dataset, Recmodel, bpr: utils.BPRLoss, epoch, w=None, val=-1): if not args.simutaneously: # train alternatively Recmodel.train() for m in Recmodel.modules(): if isinstance(m, nn.Embedding): m.requires_grad_(val <= 0) elif hasattr(m, 'weight'): m.requires_grad_(not (val <= 0)) print('mapping fixed') if val <= 0 else print('embedding fixed') # for m in Recmodel.modules(): # test separate training # if hasattr(m, 'weight'): # if hasattr(m, 'bias') and m.bias is not None: # print(m, 'weight:', m.weight.requires_grad, ' bias:', m.bias.requires_grad) # else: # print(m, 'weight', m.weight.requires_grad) S, sam_time = utils.UniformSample_original(dataset, val) # bpr sample print(f"sample time:{sam_time[0]:.1f}={sam_time[1]:.2f}+{sam_time[2]:.2f}") users = torch.Tensor(S[:, 0]).long().to(args.device) posItems = torch.Tensor(S[:, 1]).long().to(args.device) negItems = torch.Tensor(S[:, 2]).long().to(args.device) users, posItems, negItems = utils.shuffle(users, posItems, negItems) total_batch = len(users) // args.bpr_batch_size + 1 aver_loss = 0. for (batch_i, (batch_users, batch_pos, batch_neg)) \ in enumerate(utils.minibatch(users, posItems, negItems, batch_size=args.bpr_batch_size)): # train on different graph cri = bpr.stageOne(dataset.graph[val + 1], batch_users, batch_pos, batch_neg) aver_loss += cri if args.tensorboard: w.add_scalar( f'BPRLoss/BPR', cri, epoch * int(len(users) / args.bpr_batch_size) + batch_i) aver_loss = aver_loss / total_batch return aver_loss
def _run_epoch_train(self, sess, inputs, masks, labels, shuffle): n_minibatches = len(np.arange(0, len(inputs), self.config.batch_size)) prog = tf.keras.utils.Progbar(target=n_minibatches) minibatches = utils.minibatch(self.config.batch_size, inputs, masks=masks, labels=labels, shuffle=shuffle) mean_loss = 0 mean_grad_norm = 0 for i, curr_batch in enumerate(minibatches): loss, grad_norm = self._train_on_batch(sess, *curr_batch) mean_loss += loss mean_grad_norm += grad_norm force = (i + 1) == n_minibatches prog.update(i + 1, [('train_loss', loss), ('grad_norm', grad_norm)], force=force) mean_loss /= (i + 1) mean_grad_norm /= (i + 1) return mean_loss, mean_grad_norm
def _get_predictions(model): model._net.eval() sequences = model.test_sequence.sequences users = np.arange(model._num_users) n_train = sequences.shape[0] indices = np.arange(n_train) # convert from numpy to PyTorch tensor all_item_ids = np.arange(model._num_items).reshape(1, -1) all_item_ids = torch.from_numpy(all_item_ids.astype(np.int64)).clone() all_item_var = Variable(gpu(all_item_ids, model._use_cuda)) sequences_tensor = gpu(torch.from_numpy(sequences), model._use_cuda) user_tensor = gpu(torch.from_numpy(users), model._use_cuda) top = 1000 all_predictions = np.zeros((model._num_users, top), dtype=np.int64) for (batch_indices, batch_sequence, batch_user) in minibatch(indices, sequences_tensor, user_tensor, batch_size=256): batch_size = batch_user.shape[0] sequence_var = Variable(batch_sequence) user_var = Variable(batch_user) batch_all_item_var = all_item_var.repeat(batch_size, 1) prediction = model._net(sequence_var, user_var, batch_all_item_var) _, tops = prediction.topk(top, dim=1) all_predictions[batch_indices, :] = cpu(tops.data).numpy() return all_predictions
def BPR_train_original(dataset, recommend_model, loss_class, epoch, neg_k=1, w=None): Recmodel = recommend_model Recmodel.train() bpr: utils.BPRLoss = loss_class with timer(name="Sample"): S = utils.UniformSample_original(dataset) users = torch.Tensor(S[:, 0]).long() posItems = torch.Tensor(S[:, 1]).long() negItems = torch.Tensor(S[:, 2]).long() users = users.to(world.device) posItems = posItems.to(world.device) negItems = negItems.to(world.device) users, posItems, negItems = utils.shuffle(users, posItems, negItems) total_batch = len(users) // world.config['bpr_batch_size'] + 1 aver_loss = 0. for (batch_i, (batch_users, batch_pos, batch_neg)) in enumerate( utils.minibatch(users, posItems, negItems, batch_size=world.config['bpr_batch_size'])): cri = bpr.stageOne(batch_users, batch_pos, batch_neg) aver_loss += cri if False: #world.tensorboard: w.add_scalar( f'BPRLoss/BPR', cri, epoch * int(len(users) / world.config['bpr_batch_size']) + batch_i) aver_loss = aver_loss / total_batch time_info = timer.dict() timer.zero() return f"loss{aver_loss:.3f}-{time_info}"
def run_validation(self, valid_dataset, batch_size): accs = [] ret = [] for xbatch, labels in minibatch(valid_dataset, batch_size): word_seq, sequence_len = pad_sequences(xbatch) feed = { self.word_ids: word_seq, self.sequence_lengths: sequence_len, self.keep_dropout_rate: 1.0 } if self.use_crf: viterbi_sequences = [] logits_v, trans_params_v = self.sess.run( [self.logits, self.trans_params], feed_dict=feed) for logit, seq_length in zip(logits_v, sequence_len): logit = logit[:seq_length] viterbi_seq, viterbi_score = tf.contrib.crf.viterbi_decode( logit, trans_params_v) viterbi_sequences += [viterbi_seq] labels_pred_v = viterbi_sequences else: labels_pred_v = self.sess.run(label_pred, feed_dict=feed) for words, lab, lab_pred, seq_length in zip( xbatch, labels, labels_pred_v, sequence_len): lab = lab[:seq_length] lab_pred = lab_pred[:seq_length] acc = [a == b for (a, b) in zip(lab, lab_pred)] ret.append((words, lab, lab_pred, acc)) accs += acc overall_acc = np.mean(accs) return overall_acc, ret
def BPR_train_original(dataset, Recmodel, opt): Recmodel.train() S = utils.bpr_neg_samp(dataset.trainUniqueUsers, dataset.trainDataSize, support_dict=dataset.allPos, item_array=dataset.trainUniqueItems) users = torch.Tensor(S[:, 0]).long().to(args.device) posItems = torch.Tensor(S[:, 1]).long().to(args.device) negItems = torch.Tensor(S[:, 2]).long().to(args.device) users, posItems, negItems = utils.shuffle(users, posItems, negItems) # loss and update total_batch = len(users) // args.batch_size + 1 aver_loss = 0. for (batch_i, (batch_users, batch_pos, batch_neg)) \ in enumerate(utils.minibatch(users, posItems, negItems, batch_size=args.batch_size)): loss, reg_loss = Recmodel.bpr_loss(dataset.graph, batch_users, batch_pos, batch_neg) reg_loss = reg_loss * args.reg_lambda loss = loss + reg_loss opt.zero_grad() loss.backward() opt.step() aver_loss += loss.cpu().item() aver_loss = aver_loss / total_batch return aver_loss
def Test(dataset, Recmodel, epoch, w=None, multicore=0, valid=True): """evaluate models Args: dataset (BasicDatset): defined in dataloader.BasicDataset, loaded in register.py Recmodel (PairWiseModel): epoch (int): w (SummaryWriter, optional): Tensorboard writer multicore (int, optional): The num of cpu cores for testing. Defaults to 0. Returns: dict: summary of metrics """ u_batch_size = world.config['test_u_batch_size'] dataset: utils.BasicDataset testDict: dict if valid: testDict = dataset.validDict else: testDict = dataset.testDict Recmodel: model.LightGCN # eval mode with no dropout Recmodel.eval() max_K = max(world.topks) if multicore == 1: pool = multiprocessing.Pool(CORES) with torch.no_grad(): users = list(testDict.keys()) users_list = [] rating_list = [] groundTrue_list = [] # ratings = [] total_batch = len(users) // u_batch_size + 1 for batch_users in utils.minibatch(users, batch_size=u_batch_size): allPos = dataset.getUserPosItems(batch_users) groundTrue = [testDict[u] for u in batch_users] batch_users_gpu = torch.Tensor(batch_users).long() batch_users_gpu = batch_users_gpu.to(world.DEVICE) rating = Recmodel.getUsersRating(batch_users_gpu) #rating = rating.cpu() exclude_index = [] exclude_items = [] if not world.TESTDATA: for range_i, items in enumerate(allPos): exclude_index.extend([range_i] * len(items)) exclude_items.extend(items) rating[exclude_index, exclude_items] = -1e5 _, rating_K = torch.topk(rating, k=max_K) del rating users_list.append(batch_users) rating_list.append(rating_K.cpu()) groundTrue_list.append(groundTrue) assert total_batch == len(users_list) X = zip(rating_list, groundTrue_list) if world.ONE: results = { 'hr': np.zeros(len(world.topks)), 'ndcg': np.zeros(len(world.topks)) } if multicore == 1: pre_results = pool.map(test_one_batch, X) else: pre_results = [] for x in X: pre_results.append(test_one_batch_ONE(x)) scale = float(u_batch_size / len(users)) for result in pre_results: results['hr'] += result['hr'] results['ndcg'] += result['ndcg'] results['hr'] /= float(len(users)) results['ndcg'] /= float(len(users)) if w: w.add_scalars( f'Valid/HR@{world.topks}', { str(world.topks[i]): results['hr'][i] for i in range(len(world.topks)) }, epoch) w.add_scalars( f'Valid/NDCG@{world.topks}', { str(world.topks[i]): results['ndcg'][i] for i in range(len(world.topks)) }, epoch) else: results = { 'precision': np.zeros(len(world.topks)), 'recall': np.zeros(len(world.topks)), 'ndcg': np.zeros(len(world.topks)) } if multicore == 1: pre_results = pool.map(test_one_batch, X) else: pre_results = [] for x in X: pre_results.append(test_one_batch(x)) scale = float(u_batch_size / len(users)) for result in pre_results: results['recall'] += result['recall'] results['precision'] += result['precision'] results['ndcg'] += result['ndcg'] results['recall'] /= float(len(users)) results['precision'] /= float(len(users)) results['ndcg'] /= float(len(users)) if w: w.add_scalars( f'Valid/Recall@{world.topks}', { str(world.topks[i]): results['recall'][i] for i in range(len(world.topks)) }, epoch) w.add_scalars( f'Valid/Precision@{world.topks}', { str(world.topks[i]): results['precision'][i] for i in range(len(world.topks)) }, epoch) w.add_scalars( f'Valid/NDCG@{world.topks}', { str(world.topks[i]): results['ndcg'][i] for i in range(len(world.topks)) }, epoch) if multicore == 1: pool.close() return results
# saver.restore(sess, "loc_model_mse.ckpt") # print("Model restored.") for epoch in xrange(20): rand_batches = range(110) shuffle(rand_batches) for i in rand_batches: data = np.load( '/home/data2/vision6/ethpete/gaze_data/batch_{0}.npz'.format( i)) imgs = data['imgs'][:, :, :, 0] imgs = imgs.reshape( (imgs.shape[0], imgs.shape[1], imgs.shape[2], 1)) gazes = data['gazes'] img_batches = minibatch(imgs, 32, 1000) label_batches = minibatch(gazes, 32, 1000) # print("Processing training batch {0}".format(i)) for images, labels in zip(img_batches, label_batches): sess.run(optimizer, feed_dict={ x: images, y: labels, keep_prob: dropout }) # print("Processed training batch {0}".format(i)) avg_loss = 0 avg_acc = 0 nums = 0 for j in range(110, 126): data = np.load(
def fit( self, X_real, lambda_, n_steps=1000, batchsize=128, c_updates_per_epoch=5, verbose=True, ): """ Fit WGAN_GP on a training dataset. Parameters ---------- X_real : numpy array of shape (n_ex, n_feats) Training dataset lambda_ : float Gradient penalty coefficient for the critic loss n_steps : int (default: 1000) The maximum number of generator updates to perform batchsize : int (default: 128) Number of examples to use in each training minibatch c_updates_per_epoch : int The number of critic updates to perform at each generator update verbose : bool (default : True) Print loss values after each update. If false, only print loss every 100 steps. """ self.lambda_ = lambda_ self.verbose = verbose self.n_steps = n_steps self.batchsize = batchsize self.c_updates_per_epoch = c_updates_per_epoch # adjust output of the generator to match the dimensionality of X if not self.is_initialized: self.n_feats = X_real.shape[1] self._init_params() # (re-)initialize loss prev_C, prev_G = np.inf, np.inf self.loss = WGAN_GPLoss(lambda_=self.lambda_) # training loop NC, NG = self.c_updates_per_epoch, self.n_steps for i in range(NG): estart = time() batch_generator, _ = minibatch(X_real, batchsize, shuffle=False) for j, b_ix in zip(range(NC), batch_generator): bstart = time() X_batch = X_real[b_ix] C_loss = self.update_critic(X_batch) # for testing, don't perform gradient update so we can inspect each grad if not self.debug: self.update("C", C_loss) if self.verbose: fstr = "\t[Critic batch {}] Critic loss: {:.3f} {:.3f}∆ ({:.1f}s/batch)" print(fstr.format(j + 1, C_loss, prev_C - C_loss, time() - bstart)) prev_C = C_loss # generator update G_loss = self.update_generator(X_batch.shape) # for testing, don't perform gradient update so we can inspect each grad if not self.debug: self.update("G", G_loss) if i % 99 == 0: fstr = "[Epoch {}] Gen. loss: {:.3f} Critic loss: {:.3f}" print(fstr.format(i + 1, G_loss, C_loss)) elif self.verbose: fstr = "[Epoch {}] Gen. loss: {:.3f} {:.3f}∆ ({:.1f}s/epoch)" print(fstr.format(i + 1, G_loss, prev_G - G_loss, time() - estart)) prev_G = G_loss
def Test(dataset, Recmodel, epoch, w=None, multicore=0): u_batch_size = world.config['test_u_batch_size'] dataset: utils.BasicDataset testDict: dict = dataset.testDict Recmodel: model.LightGCN # eval mode with no dropout Recmodel = Recmodel.eval() max_K = max(world.topks) if multicore == 1: pool = multiprocessing.Pool(CORES) results = { 'precision': np.zeros(len(world.topks)), 'recall': np.zeros(len(world.topks)), 'ndcg': np.zeros(len(world.topks)) } with torch.no_grad(): users = list(testDict.keys()) try: assert u_batch_size <= len(users) / 10 except AssertionError: print( f"test_u_batch_size is too big for this dataset, try a small one {len(users) // 10}" ) users_list = [] rating_list = [] groundTrue_list = [] auc_record = [] # ratings = [] total_batch = len(users) // u_batch_size + 1 for batch_users in utils.minibatch(users, batch_size=u_batch_size): allPos = dataset.getUserPosItems(batch_users) groundTrue = [testDict[u] for u in batch_users] batch_users_gpu = torch.Tensor(batch_users).long() batch_users_gpu = batch_users_gpu.to(world.device) rating = Recmodel.getUsersRating(batch_users_gpu) #rating = rating.cpu() exclude_index = [] exclude_items = [] for range_i, items in enumerate(allPos): exclude_index.extend([range_i] * len(items)) exclude_items.extend(items) rating[exclude_index, exclude_items] = -(1 << 10) _, rating_K = torch.topk(rating, k=max_K) #rating = rating.cpu().numpy() '''aucs = [ utils.AUC(rating[i], dataset, test_data) for i, test_data in enumerate(groundTrue) ] auc_record.extend(aucs)''' del rating users_list.append(batch_users) rating_list.append(rating_K.cpu()) groundTrue_list.append(groundTrue) assert total_batch == len(users_list) X = zip(rating_list, groundTrue_list) if multicore == 1: pre_results = pool.map(test_one_batch, X) else: pre_results = [] for x in X: pre_results.append(test_one_batch(x)) scale = float(u_batch_size / len(users)) for result in pre_results: results['recall'] += result['recall'] results['precision'] += result['precision'] results['ndcg'] += result['ndcg'] results['recall'] /= float(len(users)) results['precision'] /= float(len(users)) results['ndcg'] /= float(len(users)) #results['auc'] = np.mean(auc_record) if world.tensorboard: w.add_scalars( f'Test/Recall@{world.topks}', { str(world.topks[i]): results['recall'][i] for i in range(len(world.topks)) }, epoch) w.add_scalars( f'Test/Precision@{world.topks}', { str(world.topks[i]): results['precision'][i] for i in range(len(world.topks)) }, epoch) w.add_scalars( f'Test/NDCG@{world.topks}', { str(world.topks[i]): results['ndcg'][i] for i in range(len(world.topks)) }, epoch) if multicore == 1: pool.close() print(results) return results
def test(dataset, Recmodel, epoch, w=None, multicore=0, best_results=None): u_batch_size = args.test_batch testDict = dataset.testDict Recmodel = Recmodel.eval() # eval mode with no dropout max_K = max(args.topks) if multicore == 1: pool = multiprocessing.Pool(args.n_cores) results = { 'precision': np.zeros(len(args.topks)), 'recall': np.zeros(len(args.topks)), 'ndcg': np.zeros(len(args.topks)) } with torch.no_grad(): users = list(testDict.keys()) try: assert u_batch_size <= len(users) / 10 except AssertionError: print( f"test_u_batch_size is too big for this dataset, try a small one {len(users) // 10}" ) users_list, rating_list, groundTrue_list, auc_record = [], [], [], [] t0 = time.time() total_batch = len(users) // u_batch_size + 1 for batch_users in utils.minibatch(users, batch_size=u_batch_size): allPos = dataset.getUserPosItems(batch_users, dataset.UserItemNet) # ********** groundTrue = [testDict[u] for u in batch_users] batch_users_gpu = torch.Tensor(batch_users).long() batch_users_gpu = batch_users_gpu.to(args.device) rating = Recmodel.getUsersRating(dataset.graph[args.test_graph], batch_users_gpu) # ********** exclude_index = [] exclude_items = [] for range_i, items in enumerate(allPos): exclude_index.extend([range_i] * len(items)) exclude_items.extend(items) rating[exclude_index, exclude_items] = -(1 << 10) _, rating_K = torch.topk(rating, k=max_K) rating = rating.cpu().numpy() aucs = [ utils.AUC(rating[i], dataset, test_data) for i, test_data in enumerate(groundTrue) ] auc_record.extend(aucs) del rating users_list.append(batch_users) rating_list.append(rating_K.cpu()) groundTrue_list.append(groundTrue) assert total_batch == len(users_list) X = zip(rating_list, groundTrue_list) if multicore == 1: pre_results = pool.map(test_one_batch, X) else: pre_results = [] for x in X: pre_results.append(test_one_batch(x)) scale = float(u_batch_size / len(users)) for result in pre_results: results['recall'] += result['recall'] results['precision'] += result['precision'] results['ndcg'] += result['ndcg'] results['recall'] /= float(len(users)) results['precision'] /= float(len(users)) results['ndcg'] /= float(len(users)) results['auc'] = np.mean(auc_record) if args.tensorboard: w.add_scalars( f'Test/Recall@{args.topks}', { str(args.topks[i]): results['recall'][i] for i in range(len(args.topks)) }, epoch) w.add_scalars( f'Test/Precision@{args.topks}', { str(args.topks[i]): results['precision'][i] for i in range(len(args.topks)) }, epoch) w.add_scalars( f'Test/NDCG@{args.topks}', { str(args.topks[i]): results['ndcg'][i] for i in range(len(args.topks)) }, epoch) if multicore == 1: pool.close() print('time consumption: %.2f' % (time.time() - t0)) print('recall:', results['recall'], 'precision:', results['precision'], 'ndcg:', results['ndcg'], 'auc:', results['auc']) if results['recall'][0] > best_results['recall'][0]: return results else: return best_results
# with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: with tf.Session() as sess: sess.run(init) saver.restore(sess, "loc_model.ckpt") print("Model restored.") for epoch in xrange(20): rand_batches = range(110) shuffle(rand_batches) for i in rand_batches: data = np.load('/home/data2/vision6/ethpete/gaze_data/batch_{0}.npz'.format(i)) imgs = data['imgs'][:, :, :, 0] imgs = imgs.reshape((imgs.shape[0], imgs.shape[1], imgs.shape[2], 1)) buckets = data['buckets'] img_batches = minibatch(imgs, 32, 1000) label_batches = minibatch(buckets, 32, 1000) # print("Processing training batch {0}".format(i)) for images, labels in zip(img_batches, label_batches): for label in labels: if label.sum() != 1: print(label, i) sess.run(optimizer, feed_dict={x: images, y: labels, keep_prob: dropout}) # print("Processed training batch {0}".format(i)) avg_loss = 0 avg_acc = 0 nums = 0 for j in range(110, 126): data = np.load('/home/data2/vision6/ethpete/gaze_data/batch_{0}.npz'.format(j)) imgs = data['imgs']
def test(dataset, testDict, Recmodel): u_batch_size = args.test_batch Recmodel = Recmodel.eval() # eval mode with no dropout max_K = max(args.topks) results = {'precision': np.zeros(len(args.topks)), 'recall': np.zeros(len(args.topks)), 'ndcg': np.zeros(len(args.topks))} with torch.no_grad(): users = list(testDict.keys()) users_list, rating_list, groundTrue_list, auc_record = [], [], [], [] total_batch = len(users) // u_batch_size + 1 for batch_users in utils.minibatch(users, batch_size=u_batch_size): allPos = dataset.getUserPosItems(batch_users) # batch user pos in train groundTrue = [testDict[u] for u in batch_users] # batch user pos in test batch_users_gpu = torch.Tensor(batch_users).long() batch_users_gpu = batch_users_gpu.to(args.device) rating = Recmodel.getUsersRating(dataset.graph, batch_users_gpu) # batch user rating / have used sigmoid # mask pos in train exclude_index = [] exclude_items = [] for range_i, items in enumerate(allPos): exclude_index.extend([range_i] * len(items)) exclude_items.extend(items) rating[exclude_index, exclude_items] = -(1 << 10) # get top k rating item index _, rating_K = torch.topk(rating, k=max_K) rating_K = rating_K.cpu().numpy() # compute auc rating = rating.cpu().numpy() aucs = [utils.AUC(rating[i], test_data) for i, test_data in enumerate(groundTrue)] auc_record.extend(aucs) # store batch del rating users_list.append(batch_users) rating_list.append(rating_K) groundTrue_list.append(groundTrue) # compute metric assert total_batch == len(users_list) pre_results = [] for rl, gt in zip(rating_list, groundTrue_list): batch_ks_result = np.zeros((3, len(args.topks)), dtype=np.float32) for i, k in enumerate(args.topks): batch_ks_result[:, i] = np.array([utils.test_batch(rl[:, :k], gt)], dtype=np.float32) pre_results.append(batch_ks_result) for result in pre_results: results['precision'] += result[0] results['recall'] += result[1] results['ndcg'] += result[2] results['precision'] /= float(len(users)) results['recall'] /= float(len(users)) results['ndcg'] /= float(len(users)) results['auc'] = np.mean(auc_record) return results