autoencoder = Autoencoder(args.encoder_layers, args.decoder_layers) classifier = LogisticRegression(args.encoder_layers[-1]) oracle = DL2_Oracle( learning_rate=args.dl2_lr, net=autoencoder, use_cuda=torch.cuda.is_available(), constraint=ConstraintBuilder.build( autoencoder, train_dataset, args.constraint ) ) binary_cross_entropy = nn.BCEWithLogitsLoss( pos_weight=train_dataset.pos_weight('train') if args.balanced else None ) optimizer = torch.optim.Adam( list(autoencoder.parameters()) + list(classifier.parameters()), lr=args.learning_rate, weight_decay=args.weight_decay ) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( optimizer, 'min', patience=args.patience, factor=0.5 ) def run(autoencoder, classifier, optimizer, loader, split, epoch): predictions, targets, l_inf_diffs = list(), list(), list() tot_mix_loss, tot_ce_loss, tot_dl2_loss = Statistics.get_stats(3) tot_l2_loss, tot_stat_par, tot_eq_odds = Statistics.get_stats(3) progress_bar = tqdm(loader) for data_batch, targets_batch, protected_batch in progress_bar:
input_size = train_dataset.get_len_cvec() print('input_size: ', input_size) # set hyper parameter hidden_size = 10000 num_class = 2 learning_rate = 0.0001 num_epoch = 5 # device configuration device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') model = LogisticRegression(input_size, hidden_size, num_class).to(device) criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) # train model total_step = len(train_loader) for epoch in range(num_epoch): print('epoch start') for i, (data, label) in enumerate(train_loader): label = label.squeeze().to(device) data = data.to(device) outputs = model(data) loss = criterion(outputs, label) optimizer.zero_grad() loss.backward() optimizer.step() if (i + 1) % 1 == 0: print('Epoch : [{}/{}], Step : [{}/{}], Loss : {:.4f}'.format(
dataset = getattr(__import__('datasets'), dataset.capitalize() + 'Dataset') train_dataset = dataset('train', args) test_dataset = dataset('test', args) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=1, shuffle=True) test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1, shuffle=False) autoencoder = Autoencoder(encoder_layers, decoder_layers) classifier = LogisticRegression(encoder_layers[-1]) for param in autoencoder.parameters(): param.data = param.double() for param in classifier.parameters(): param.data = param.double() prefix = f'{args.label}_' if args.label else '' postfix = '_robust' if args.adversarial else '' autoencoder.load_state_dict( torch.load(path.join( args.models_dir, prefix + f'autoencoder_{args.epoch}' + postfix + '.pt'), map_location=lambda storage, loc: storage)) classifier.load_state_dict( torch.load(path.join(args.models_dir, prefix + f'classifier_{args.epoch}' + postfix + '.pt'), map_location=lambda storage, loc: storage))
param.requires_grad_(False) autoencoder.load_state_dict( torch.load(path.join(models_dir, f'autoencoder_{args.load_epoch}.pt'), map_location=lambda storage, loc: storage)) oracle = DL2_Oracle(learning_rate=args.dl2_lr, net=autoencoder, use_cuda=torch.cuda.is_available(), constraint=ConstraintBuilder.build(autoencoder, train_dataset, args.constraint)) binary_cross_entropy = nn.BCEWithLogitsLoss( pos_weight=train_dataset.pos_weight('train') if args.balanced else None) optimizer = torch.optim.Adam(classifier.parameters(), lr=args.learning_rate, weight_decay=args.weight_decay) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min', patience=args.patience, factor=0.5) def run(autoencoder, classifier, optimizer, loader, split): predictions, targets = list(), list() tot_ce_loss, tot_stat_par, tot_eq_odds = Statistics.get_stats(3) progress_bar = tqdm(loader) if args.adversarial:
def train(batch_size=64, window_size=3, epochs=100): train_dataset = SupervisedDataset(mode='train', window_size=window_size, log_reg=True) train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True) val_dataset = SupervisedDataset(mode='val', window_size=window_size, log_reg=True) val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=True) base_lr_rate = 1e-3 weight_decay = 1e-6 #0.000016 #model = LSTM(input_size=78, hidden_size=128, num_classes=170, n_layers=2).to(device=torch.device('cuda:0')) model = LogisticRegression( num_keypoints=78, num_features=2, num_classes=170).to(device=torch.device('cuda:0')) ''' for name, param in model.named_parameters(): if 'bias' in name: nn.init.constant_(param, 0.0) elif 'weight' in name: nn.init.xavier_uniform_(param) ''' criterion = nn.BCEWithLogitsLoss( ) # Use this for Logistic Regression training #criterion = nn.BCELoss() # Use this for LSTM training (with Softmax) optimizer = optim.Adam( model.parameters(), lr=base_lr_rate) #, weight_decay=weight_decay, amsgrad=True) best_epoch_train_accuracy = 0.0 best_epoch_val_accuracy = 0.0 for current_epoch in range(epochs): current_train_iter = 0 current_val_iter = 0 running_train_loss = 0.0 current_average_train_loss = 0.0 running_val_loss = 0.0 current_average_val_loss = 0.0 num_train_data = 0 num_val_data = 0 running_train_correct_preds = 0 running_train_correct_classwise_preds = [0] * 170 running_val_correct_preds = 0 running_val_correct_classwise_preds = [0] * 170 for phase in ['train', 'val']: # Train loop if phase == 'train': train_epoch_since = time.time() model.train() for train_batch_window, train_batch_label in train_loader: current_train_iter += 1 outs = model(train_batch_window) #scheduler = poly_lr_scheduler(optimizer = optimizer, init_lr = base_lr_rate, iter = current_iter, lr_decay_iter = 1, # max_iter = max_iter, power = power) # max_iter = len(train_loader) optimizer.zero_grad() loss = criterion(outs, train_batch_label) gt_confidence, gt_index = torch.max(train_batch_label, dim=1) #loss = criterion(outs, gt_index) running_train_loss += loss.item() current_average_train_loss = running_train_loss / current_train_iter loss.backward(retain_graph=False) optimizer.step() pred_confidence, pred_index = torch.max(outs, dim=1) #gt_confidence, gt_index = torch.max(train_batch_label, dim=1) batch_correct_preds = torch.eq( pred_index, gt_index).long().sum().item() batch_accuracy = (batch_correct_preds / train_batch_window.shape[0]) * 100 num_train_data += train_batch_window.shape[0] running_train_correct_preds += batch_correct_preds if current_train_iter % 10 == 0: #print(outs) print( f"\nITER#{current_train_iter} ({current_epoch+1}) BATCH TRAIN ACCURACY: {batch_accuracy:.4f}, RUNNING TRAIN LOSS: {loss.item():.8f}" ) print( f"Predicted / GT index:\n{pred_index}\n{gt_index}\n" ) last_epoch_average_train_loss = current_average_train_loss epoch_accuracy = (running_train_correct_preds / num_train_data) * 100 print( f"\n\nEPOCH#{current_epoch+1} EPOCH TRAIN ACCURACY (BEST): {epoch_accuracy:.4f} ({best_epoch_train_accuracy:.4f}), AVERAGE TRAIN LOSS: {last_epoch_average_train_loss:.8f}\n\n" ) if epoch_accuracy > best_epoch_train_accuracy: best_epoch_train_accuracy = epoch_accuracy torch.save( model.state_dict(), f"./model_params/logistic_regression/train/train_logreg_epoch{current_epoch+1}_acc{best_epoch_train_accuracy:.4f}.pth" ) print("\n\nSAVED BASED ON TRAIN ACCURACY!\n\n") train_time_elapsed = time.time() - train_epoch_since # Validation loop elif phase == 'val': val_epoch_since = time.time() model.eval() with torch.no_grad(): for val_batch_window, val_batch_label in val_loader: current_val_iter += 1 outs = model(val_batch_window) gt_confidence, gt_index = torch.max(val_batch_label, dim=1) val_loss = criterion(outs, val_batch_label) #val_loss = criterion(outs, gt_index) running_val_loss += val_loss.item() current_average_val_loss = running_val_loss / current_val_iter pred_confidence, pred_index = torch.max(outs, dim=1) #gt_confidence, gt_index = torch.max(val_batch_label, dim=1) batch_correct_preds = torch.eq( pred_index, gt_index).long().sum().item() batch_accuracy = (batch_correct_preds / val_batch_window.shape[0]) * 100 num_val_data += val_batch_window.shape[0] running_val_correct_preds += batch_correct_preds if current_val_iter % 10 == 0: print( f"\nITER#{current_val_iter} ({current_epoch+1}) BATCH VALIDATION ACCURACY: {batch_accuracy:.4f}, RUNNING VALIDATION LOSS: {val_loss.item():.8f}" ) print( f"Predicted / GT index:\n{pred_index}\n{gt_index}\n" ) last_epoch_average_val_loss = current_average_val_loss epoch_accuracy = (running_val_correct_preds / num_val_data) * 100 print( f"\n\nEPOCH#{current_epoch+1} EPOCH VALIDATION ACCURACY (BEST): {epoch_accuracy:.4f} ({best_epoch_val_accuracy:.4f}), AVERAGE VALIDATION LOSS: {last_epoch_average_val_loss:.8f}\n\n" ) if epoch_accuracy > best_epoch_val_accuracy: best_epoch_val_accuracy = epoch_accuracy torch.save( model.state_dict(), f"./model_params/logistic_regression/val/val_logreg_epoch{current_epoch+1}_acc{best_epoch_val_accuracy:.4f}.pth" ) print("\n\nSAVED BASED ON VAL ACCURACY!\n\n") val_time_elapsed = time.time() - val_epoch_since
def main(): args = parser.parse_args() # seed everything to ensure reproducible results from different runs if args.seed is not None: random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) torch.backends.cudnn.deterministic = True ########################################################################### # Model ########################################################################### global best_acc1 # create model if args.arch == 'LogisticRegression': model = LogisticRegression(input_size=13, n_classes=args.classes) elif args.arch == 'NeuralNet': model = NeuralNet(input_size=13, hidden_size=[32, 16], n_classes=args.classes) #hidden_size=[64, 32] if args.gpu is not None: print("Use GPU: {} for training".format(args.gpu)) torch.cuda.set_device(args.gpu) torch.backends.cudnn.benchmark = True model = model.cuda(args.gpu) # print(model) if args.train_file: print(30 * '=') print(summary(model, input_size=(1, 13), batch_size=args.batch_size, device='cpu')) print(30 * '=') ########################################################################### # save directory ########################################################################### save_dir = os.path.join(os.getcwd(), args.save_dir) save_dir += ('/arch[{}]_optim[{}]_lr[{}]_lrsch[{}]_batch[{}]_' 'WeightedSampling[{}]').format(args.arch, args.optim, args.lr, args.lr_scheduler, args.batch_size, args.weighted_sampling) if args.suffix: save_dir += '_{}'.format(args.suffix) save_dir = save_dir[:] if not os.path.exists(save_dir): os.makedirs(save_dir) ########################################################################### # Criterion and optimizer ########################################################################### # Initialise criterion and optimizer if args.gpu is not None: criterion = nn.CrossEntropyLoss().cuda(args.gpu) else: criterion = nn.CrossEntropyLoss() # define optimizer print("=> using '{}' optimizer".format(args.optim)) if args.optim == 'sgd': optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay, nesterov=True) else: # default is adam optimizer = torch.optim.Adam(model.parameters(), args.lr, betas=(0.9, 0.999), eps=1e-08, weight_decay=args.weight_decay, amsgrad=False) ########################################################################### # Resume training and load a checkpoint ########################################################################### # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) if args.gpu is None: checkpoint = torch.load(args.resume) else: # Map model to be loaded to specified single gpu. loc = 'cuda:{}'.format(args.gpu) checkpoint = torch.load(args.resume, map_location=loc) args.start_epoch = checkpoint['epoch'] best_acc1 = checkpoint['best_acc1'] if args.gpu is not None: # best_acc1 may be from a checkpoint from a different GPU best_acc1 = best_acc1.to(args.gpu) model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})" .format(args.resume, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(args.resume)) ########################################################################### # Data Augmentation ########################################################################### # TODO ########################################################################### # Learning rate scheduler ########################################################################### print("=> using '{}' initial learning rate (lr)".format(args.lr)) # define learning rate scheduler scheduler = args.lr_scheduler if args.lr_scheduler == 'reduce': print("=> using '{}' lr_scheduler".format(args.lr_scheduler)) # Reduce learning rate when a metric has stopped improving. scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.5, patience=10) elif args.lr_scheduler == 'cyclic': print("=> using '{}' lr_scheduler".format(args.lr_scheduler)) scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=0.00005, max_lr=0.005) elif args.lr_scheduler == 'cosine': print("=> using '{}' lr_scheduler".format(args.lr_scheduler)) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=100, eta_min=0, last_epoch=-1) ########################################################################### # load train data ########################################################################### if args.train_file: train_dataset = HeartDiseaseDataset(csv=args.train_file, label_names=LABELS) if args.weighted_sampling: train_sampler = torch.utils.data.WeightedRandomSampler(train_dataset.sample_weights, len(train_dataset), replacement=True) else: train_sampler = None ########################################################################### # update criterion print('class_sample_count ', train_dataset.class_sample_count) print('class_probability ', train_dataset.class_probability) print('class_weights ', train_dataset.class_weights) print('sample_weights ', train_dataset.sample_weights) if args.weighted_loss: if args.gpu is not None: criterion = nn.CrossEntropyLoss(weight=train_dataset.class_weights).cuda(args.gpu) else: criterion = nn.CrossEntropyLoss(weight=train_dataset.class_weights) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), num_workers=args.workers, pin_memory=True, sampler=train_sampler) ########################################################################### # load validation data ########################################################################### if args.valid_file: valid_dataset = HeartDiseaseDataset(csv=args.valid_file, label_names=LABELS) val_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) if args.evaluate: # retrieve correct save path from saved model save_dir = os.path.split(args.resume)[0] validate(val_loader, model, criterion, save_dir, args) return ########################################################################### # Train the model ########################################################################### for epoch in range(args.start_epoch, args.epochs): # adjust_learning_rate(optimizer, epoch, args) print_learning_rate(optimizer, epoch) # train for one epoch train(train_loader, model, criterion, optimizer, scheduler, epoch, args) # evaluate on validation set acc1 = validate(val_loader, model, criterion, save_dir, args) # update learning rate based on lr_scheduler if args.lr_scheduler == 'reduce': scheduler.step(acc1) elif args.lr_scheduler == 'cosine': scheduler.step() # remember best acc@1 and save checkpoint is_best = acc1 >= best_acc1 best_acc1 = max(acc1, best_acc1) print("Saving model [{}]...".format(save_dir)) save_checkpoint({'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_acc1': best_acc1, 'optimizer': optimizer.state_dict(), 'criterion': criterion, }, is_best, save_dir=save_dir) print(30 * '=')