def validation_step(self, batch, batch_idx): x, y = batch logits, _ = self(x) logits[x.nonzero(as_tuple=True)] = .0 logits = logits.cpu().numpy() y = y.cpu().numpy() val_metric = eval_metric(logits, y) # This is too slow, cannot run evaluation on everything # val_metric, metrics, metrics_raw = eval_proced(logits, y, high_idxs, low_idxs, 'val') # return {"val_metric": val_metric, "metrics": metrics, 'metrics_raw': metrics_raw} return {'val_metric': val_metric}
epoch) # --- Validation --- # if epoch % VAE_LOG_VAL_EVERY == 0: model.eval() val_metrics = [] for x, y in vd_loader: x = x.to(device) logits, _ = model(x) # Removing items from training data logits[x.nonzero(as_tuple=True)] = .0 logits = logits.detach().cpu().numpy() val_metrics += list( eval_metric(logits, y, aggregated=False)) curr_value = np.mean(val_metrics) summ.add_scalar('val/ndcg_50', curr_value, epoch) summ.flush() if curr_value > best_value: print('New best model found') best_value = curr_value pickle_dump( config, os.path.join(log_val_str, 'best_config.pkl')) torch.save(model.state_dict(), os.path.join(log_val_str, 'best_model.pth')) # Logging hyperparams and metrics
for config in tqdm(pg, desc='configs'): summ = SummaryWriter(os.path.join(log_val_str, str(config))) Atild = BPR(A, config['factors'], config['lr'], config['reg'], config['iter']) # Only focusing on validation data Atild = Atild[:sp_vd_tr_data.shape[0]] # Removing entries from training data Atild[sp_vd_tr_data.nonzero()] = .0 preds = Atild true = sp_vd_te_data.toarray() curr_value = eval_metric(preds, true) # Logging hyperparams and metrics summ.add_hparams({ **config, 'fold_n': fold_n }, {'val/ndcg_50': curr_value}) summ.flush() if curr_value > best_value: print('New best model found') best_value = curr_value pickle_dump(config, os.path.join(log_val_str, 'best_config.pkl')) # Logging hyperparams and metrics