def main(): model = UNET(in_channels=1, out_channels=1).to(device=DEVICE) loss_fn = nn.BCEWithLogitsLoss() optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE) train_loader, val_loader = get_loaders(TRAIN_DIR, VAL_DIR, BATCH_SIZE, NUM_WORKER, PIN_MEMORY) if LOAD_MODEL: load_checkpoint(torch.load("mycheckpoint.pth.tar"), model) scaler = torch.cuda.amp.GradScaler() for epoch in range(NUM_EPOCHS): train_fn(train_loader, model, optimizer, loss_fn, scaler) # TODO : save model checkpoint = { 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict() } save_checkpoint(checkpoint) # TODO : check acuuracy check_accuracy(val_loader, model, device=DEVICE) # TODO : Print results to folder save_predictions_as_imgs(val_loader, model, folder='saved_imgs/')
def main(args): print('Dataset: {}, Normal Label: {}, LR: {}'.format(args.dataset, args.label, args.lr)) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") print(device) model_type = args.model if model_type == 'resnet': model = utils.get_resnet_model(resnet_type=args.resnet_type) if args.dataset in ['rsna3D']: model = ResNet3D(model) elif model_type == 'timesformer': model = utils.get_timesformer_model(mode=args.timesformer_mode) model = model.to(device) ewc_loss = None # Freezing Pre-trained model for EWC if args.ewc: frozen_model = deepcopy(model).to(device) frozen_model.eval() utils.freeze_model(frozen_model) fisher = torch.load(args.diag_path) ewc_loss = EWCLoss(frozen_model, fisher) utils.freeze_parameters(model) sorted_train_loader, shuffled_train_loader, test_loader = utils.get_loaders(dataset=args.dataset, label_class=args.label, batch_size=args.batch_size, lookup_tables_paths=(args.train_lookup_table, args.test_lookup_table)) train_model(model, sorted_train_loader, shuffled_train_loader, test_loader, device, args, ewc_loss)
def main(): train_transform = A.Compose([ A.Resize(height=IMAGE_HEIGHT, width=IMAGE_WIDTH), A.Rotate(limit=35, p=1.0), A.HorizontalFlip(p=0.5), A.VerticalFlip(p=0.1), A.Normalize( mean=[0.0, 0.0, 0.0], std=[1.0, 1.0, 1.0], max_pixel_value=255.0, ), ToTensorV2(), ], ) val_transforms = A.Compose([ A.Resize(height=IMAGE_HEIGHT, width=IMAGE_WIDTH), A.Normalize( mean=[0.0, 0.0, 0.0], std=[1.0, 1.0, 1.0], max_pixel_value=255.0, ), ToTensorV2(), ], ) model = UNET(3, 1).to(DEVICE) loss_fn = nn.BCEWithLogitsLoss() optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE) train_loader, val_loader = get_loaders( TRAIN_IMG_DIR, TRAIN_MASK_DIR, VAL_IMG_DIR, VAL_MASK_DIR, BATCH_SIZE, train_transform, val_transforms, NUM_WORKERS, PIN_MEMORY, ) check_accuracy(val_loader, model, device=DEVICE) scaler = torch.cuda.amp.GradScaler() for epoch in range(NUM_EPOCHS): train_fn(train_loader, model, optimizer, loss_fn, scaler) checkpoint = { "state_dict": model.state_dict(), "optimizer": optimizer.state_dict(), } save_checkpoint(checkpoint) # check accuracy check_accuracy(val_loader, model, device=DEVICE) # print some examples to a folder save_predictions_as_imgs(val_loader, model, folder="saved_images/", device=DEVICE)
def main(): train_transform = A.Compose([ A.Resize(height=config.IMAGE_HEIGHT, width=config.IMAGE_WIDTH), A.Rotate(limit=35, p=1.0), A.HorizontalFlip(p=0.5), A.VerticalFlip(p=0.1), A.normalize(mean=[0.0, 0.0, 0.0], std=[1.0, 1.0, 1.0], max_pixel_value=255.0), ToTensorV2, ]) val_transform = A.Compose([ A.Resize(height=config.IMAGE_HEIGHT, width=config.IMAGE_WIDTH), A.normalize(mean=[0.0, 0.0, 0.0], std=[1.0, 1.0, 1.0], max_pixel_value=255.0), ToTensorV2, ]) model = UNet(in_channels=3, out_channels=1).to(config.DEVICE) loss_fn = nn.BCEWithLogitsLoss() optimizer = optim.Adam(model.parameters(), lr=config.LEARNING_RATE) train_loader, val_loader = get_loaders( config.TRAIN_IMAGE_DIR, config.TRAIN_MASK_DIR, config.VAL_IMG_DIR, config.VAL_MASK_DIR, config.BATCH_SIZE, train_transform, val_transform, ) if config.LOAD_MODEL: load_checkpoint(torch.load('my_checkpoint.pth.tar'), model) scaler = torch.cuda.amp.GradScaler() for epoch in range(config.NUM_EPOCHS): train_fn(train_loader, model, optimizer, loss_fn, scaler) # save model checkpoint = { 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), } save_checkpoint(checkpoint) # check acc check_accuracy(val_loader, model, device=config.DEVICE) # print some examples to a folder save_predictions_as_imgs(val_loader, model, folder='saved_images', device=config.DEVICE)
def main(): model = YOLOv3(num_classes=config.NUM_CLASSES).to(config.DEVICE) optimizer = optim.Adam(model.parameters(), lr=config.LEARNING_RATE, weight_decay=config.WEIGHT_DECAY) loss_fn = YoloLoss() scaler = torch.cuda.amp.GradScaler() train_loader, test_loader, train_eval_loader = get_loaders( train_csv_path=config.DATASET + "/train.csv", test_csv_path=config.DATASET + "/test.csv") if config.LOAD_MODEL: load_checkpoint(config.CHECKPOINT_FILE, model, optimizer, config.LEARNING_RATE) print("loaded model") scaled_anchors = (torch.tensor(config.ANCHORS) * torch.tensor( config.S).unsqueeze(1).unsqueeze(1).repeat(1, 3, 2)).to(config.DEVICE) for epoch in range(config.NUM_EPOCHS): print(f"\nEpoch [{epoch}]") #plot_couple_examples(model, test_loader, 0.6, 0.5, scaled_anchors) train_fn(train_loader, model, optimizer, loss_fn, scaler, scaled_anchors) #if config.SAVE_MODEL: # save_checkpoint(model, optimizer, filename=f"checkpoint.pth.tar") #print("On Train Eval loader:") #print("On Train loader:") #check_class_accuracy(model, train_loader, threshold=config.CONF_THRESHOLD) if epoch > 0 and epoch % 4 == 0: valid_fn(train_eval_loader, model, loss_fn, scaled_anchors) check_class_accuracy(model, test_loader, threshold=config.CONF_THRESHOLD) pred_boxes, true_boxes = get_evaluation_bboxes( test_loader, model, iou_threshold=config.NMS_IOU_THRESH, anchors=config.ANCHORS, threshold=config.CONF_THRESHOLD, ) mapval = mean_average_precision( pred_boxes, true_boxes, iou_threshold=config.MAP_IOU_THRESH, box_format="midpoint", num_classes=config.NUM_CLASSES, ) print(f"MAP: {mapval.item()}") print("\nnResuming Training\n") model.train()
def main(): # get args args = get_args() # set up gpus os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu assert torch.cuda.is_available() # set up writer, logger, and save directory for models save_root = os.path.join('checkpoints', 'dverge', 'seed_{:d}'.format(args.seed), '{:d}_{:s}{:d}_eps_{:.2f}'.format( args.model_num, args.arch, args.depth, args.distill_eps) ) if args.distill_fixed_layer: save_root += '_fixed_layer_{:d}'.format(args.distill_layer) if args.plus_adv: save_root += '_plus_adv_coeff_{:.1f}'.format(args.dverge_coeff) if args.start_from == 'scratch': save_root += '_start_from_scratch' if not os.path.exists(save_root): os.makedirs(save_root) else: print('*********************************') print('* The checkpoint already exists *') print('*********************************') writer = SummaryWriter(save_root.replace('checkpoints', 'runs')) # dump configurations for potential future references with open(os.path.join(save_root, 'cfg.json'), 'w') as fp: json.dump(vars(args), fp, indent=4, sort_keys=True) with open(os.path.join(save_root.replace('checkpoints', 'runs'), 'cfg.json'), 'w') as fp: json.dump(vars(args), fp, indent=4, sort_keys=True) # set up random seed torch.manual_seed(args.seed) random.seed(args.seed) # initialize models if args.start_from == 'baseline': args.model_file = os.path.join('checkpoints', 'baseline', 'seed_0', '{:d}_{:s}{:d}'.format(args.model_num, args.arch, args.depth), 'epoch_200.pth') elif args.divtrain_start_from == 'scratch': args.model_file = None models = utils.get_models(args, train=True, as_ensemble=False, model_file=args.model_file) # get data loaders trainloader, testloader = utils.get_loaders(args) # get optimizers and schedulers optimizers = utils.get_optimizers(args, models) schedulers = utils.get_schedulers(args, optimizers) # train the ensemble trainer = DVERGE_Trainer(models, optimizers, schedulers, trainloader, testloader, writer, save_root, **vars(args)) trainer.run()
def main(): model = YOLOv3(num_classes=config.NUM_CLASSES).to(config.DEVICE) optimizer = optim.Adam( model.parameters(), lr=config.LEARNING_RATE, weight_decay=config.WEIGHT_DECAY ) loss_fn = YoloLoss() scaler = torch.cuda.amp.GradScaler() train_loader, test_loader, train_eval_loader = get_loaders( train_csv_path=config.DATASET + "/train.csv", test_csv_path=config.DATASET + "/test.csv" ) if config.LOAD_MODEL: load_checkpoint( config.CHECKPOINT_FILE, model, optimizer, config.LEARNING_RATE ) scaled_anchors = ( torch.tensor(config.ANCHORS) * torch.tensor(config.S).unsqueeze(1).unsqueeze(1).repeat(1, 3, 2) ).to(config.DEVICE) for epoch in range(config.NUM_EPOCHS): train_fn(train_loader, model, optimizer, loss_fn, scaler, scaled_anchors) if epoch > 0 and epoch % 3 == 0: check_class_accuracy(model, test_loader, threshold=config.CONF_THRESHOLD) pred_boxes, true_boxes = get_evaluation_bboxes( test_loader, model, iou_threshold=config.NMS_IOU_THRESH, anchors=config.ANCHORS, threshold=config.CONF_THRESHOLD, ) mapval = mean_average_precision( pred_boxes, true_boxes, iou_threshold=config.MAP_IOU_THRESH, box_format="midpoint", num_classes=config.NUM_CLASSES, ) print(f"MAP: {mapval.item()}") model.train() if epoch > 99: for x, y in train_loader: x = x.to(DEVICE) for idx in range(8): bboxes = cellboxes_to_boxes(model(x)) bboxes = non_max_suppression(bboxes[idx], iou_threshold=0.5, threshold=0.4) plot_image(x[idx].permute(1,2,0).to("cpu"), bboxes, idx) if __name__ == "__main__": main()
def _test_get_loaders(features, labels): train_loader, valid_loader, test_loader = get_loaders(features, labels, prnt=False) train_x, train_y = iter(train_loader).next() valid_x, valid_y = iter(valid_loader).next() test_x, test_y = iter(test_loader).next() assert isinstance(train_loader, torch.utils.data.dataloader.DataLoader) assert isinstance(valid_loader, torch.utils.data.dataloader.DataLoader) assert isinstance(test_loader, torch.utils.data.dataloader.DataLoader) assert train_x.shape == valid_x.shape == test_x.shape assert train_y.shape == valid_y.shape == test_y.shape
def main(): train_transforms = A.Compose([ A.Resize(height=IMAGE_HEIGHT, width=IMAGE_WIDTH), A.Rotate(limit=35, p=1.0), A.HorizontalFlip(p=0.5), A.VerticalFlip(p=0.1), A.Normalize(mean=[0.0, 0.0, 0.0], std=[1.0, 1.0, 1.0], max_pixel_value=255.0), ToTensorV2() ]) val_transforms = A.Compose([ A.Resize(height=IMAGE_HEIGHT, width=IMAGE_WIDTH), A.Normalize(mean=[0.0, 0.0, 0.0], std=[1.0, 1.0, 1.0], max_pixel_value=255.0), ToTensorV2() ]) model = UNET(in_channels=3, out_channels=1).to(DEVICE) loss_fn = nn.BCEWithLogitsLoss() loss_fn = ComboLoss({'bce': 0.4, 'dice': 0.5, 'focal': 0.1}) optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE) train_loader, val_loader = get_loaders(TRAIN_IMG_DIR, TRAIN_MASK_DIR, VAL_IMG_DIR, VAL_MASK_DIR, BATCH_SIZE, train_transforms, val_transforms) if LOAD_MODEL: load_checkpoint(torch.load('checkpoint.pth'), model) scaler = torch.cuda.amp.GradScaler() for epoch in range(NUM_EPOCHS): train_fn(train_loader, model, optimizer, loss_fn, scaler) #save model checkpoint = { "state_dict": model.state_dict(), "optimizer": optimizer.state_dict() } # save_checkpoint(checkpoint) #check accuracy check_accuracy(val_loader, model, DEVICE) #print some example to a folder save_predictions_as_imgs(val_loader, model, device=DEVICE)
def main(): # get args args = get_args() # set up gpus os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu assert torch.cuda.is_available() # set up writer, logger, and save directory for models save_root = os.path.join( 'checkpoints', 'baseline', 'seed_{:d}'.format(args.seed), '{:d}_{:s}{:d}'.format(args.model_num, args.arch, args.depth)) if not os.path.exists(save_root): os.makedirs(save_root) else: print('*********************************') print('* The checkpoint already exists *') print('*********************************') writer = SummaryWriter(save_root.replace('checkpoints', 'runs')) # dump configurations for potential future references with open(os.path.join(save_root, 'cfg.json'), 'w') as fp: json.dump(vars(args), fp, indent=4) with open( os.path.join(save_root.replace('checkpoints', 'runs'), 'cfg.json'), 'w') as fp: json.dump(vars(args), fp, indent=4) # set up random seed torch.manual_seed(args.seed) # initialize models models = utils.get_models(args, train=True, as_ensemble=False, model_file=None) # get data loaders trainloader, testloader = utils.get_loaders(args) # get optimizers and schedulers optimizers = utils.get_optimizers(args, models) schedulers = utils.get_schedulers(args, optimizers) # train the ensemble trainer = Baseline_Trainer(models, optimizers, schedulers, trainloader, testloader, writer, save_root, **vars(args)) trainer.run()
def predict(): print('asdasdasd') model = UNET(in_channels=3, out_channels=1).to(DEVICE) load_checkpoint(torch.load("my_checkpoint.pth.tar", map_location=torch.device('cpu')), model) train_transform = A.Compose( [ A.Resize(height=IMAGE_HEIGHT, width=IMAGE_WIDTH), A.Rotate(limit=35, p=1.0), A.HorizontalFlip(p=0.5), A.VerticalFlip(p=0.1), A.Normalize( mean=[0.0, 0.0, 0.0], std=[1.0, 1.0, 1.0], max_pixel_value=255.0, ), ToTensorV2(), ], ) val_transforms = A.Compose( [ A.Resize(height=IMAGE_HEIGHT, width=IMAGE_WIDTH), A.Normalize( mean=[0.0, 0.0, 0.0], std=[1.0, 1.0, 1.0], max_pixel_value=255.0, ), ToTensorV2(), ], ) train_loader, val_loader = get_loaders( TRAIN_IMG_DIR, TRAIN_MASK_DIR, VAL_IMG_DIR, VAL_MASK_DIR, BATCH_SIZE, train_transform, val_transforms, NUM_WORKERS, PIN_MEMORY, ) save_predictions_as_imgs( val_loader, model, folder="saved_images/", device=DEVICE )
def data_test(): train_transform = A.Compose( [ A.Resize(height=IMAGE_HEIGHT, width=IMAGE_WIDTH), A.Rotate(limit=35, p=1.0), A.HorizontalFlip(p=0.5), A.VerticalFlip(p=0.1), A.Normalize( mean=[0.0, 0.0, 0.0], std=[1.0, 1.0, 1.0], max_pixel_value=255.0, ), ToTensorV2(), ], ) val_transforms = A.Compose( [ A.Resize(height=IMAGE_HEIGHT, width=IMAGE_WIDTH), A.Normalize( mean=[0.0, 0.0, 0.0], std=[1.0, 1.0, 1.0], max_pixel_value=255.0, ), ToTensorV2(), ], ) train_loader, val_loader = get_loaders( TRAIN_IMG_DIR, TRAIN_MASK_DIR, VAL_IMG_DIR, VAL_MASK_DIR, BATCH_SIZE, train_transform, val_transforms, NUM_WORKERS, PIN_MEMORY, ) print(len(val_loader)) for idx, (x, y) in enumerate(val_loader): print(idx) # plt.imshow(np.transpose(y[0].numpy(), (1, 2, 0))) plt.imshow(y[0].numpy()) plt.show()
def main(): model = Yolo(num_classes=config.NUM_CLASSES).to(config.DEVICE) optimizer = optim.Adam(model.parameters(), lr=config.LEARNING_RATE, weight_decay=config.WEIGHT_DECAY) lossfunction = Yololoss() scaler = torch.cuda.amp.GradScaler() train_loader, test_loader, train_eval_loader = get_loaders( train_csv_path=config.DATASET + "/train.csv", test_csv_path=config.DATASET + "/test.csv") if config.LOAD_MODEL: load_checkpoint(config.CHECKPOINT_FILE, model, optimizer, config.LEARNING_RATE) scaled_anchors = (torch.tensor(config.ANCHORS) * torch.tensor( config.GRID_SIZE).unsqueeze(1).unsqueeze(1).repeat(1, 3, 2)).to( config.DEVICE) for epoch in range(config.NUM_EPOCHS): train(train_loader, model, optimizer, lossfunction, scaler, scaled_anchors) if config.SAVE_MODEL: save_checkpoint(model, optimizer, filename=config.CHECKPOINT_FILE) if epoch > 0 and epoch % 3 == 0: check_class_accuracy(model, test_loader, threshold=config.CONF_THRESHOLD) pred_boxes, true_boxes = get_evaluation_bboxes( test_loader, model, iou_threshold=config.NMS_IOU_THRESH, anchors=config.ANCHORS, threshold=config.CONF_THRESHOLD, ) mapval = mean_average_precision( pred_boxes, true_boxes, iou_threshold=config.MAP_IOU_THRESH, box_format="midpoint", num_classes=config.NUM_CLASSES, ) print(f"MAP: {mapval.item()}") model.train()
def main(args): print('Dataset: {}, Label: {}, LR: {}'.format(args.dataset, args.label, args.lr)) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") model = utils.get_resnet_model(resnet_type=args.resnet_type) # Change last layer model.fc = torch.nn.Linear(args.latent_dim_size, 1) model = model.to(device) utils.freeze_parameters(model, train_fc=True) train_loader, test_loader = utils.get_loaders(dataset=args.dataset, label_class=args.label, batch_size=args.batch_size) outliers_loader = utils.get_outliers_loader(args.batch_size) train_model(model, train_loader, outliers_loader, test_loader, device, args.epochs, args.lr)
def main(): model = UNET(in_channels=3, out_channels=1).to(config.DEVICE) BCE = nn.BCEWithLogitsLoss() optimizer = optim.Adam(model.parameters(), lr=config.LEARNING_RATE) train_loader, val_loader = get_loaders( train_dir=config.TRAIN_IMG_DIR, train_mask_dir=config.TRAIN_MASK_DIR, val_dir=config.VAL_IMG_DIR, val_mask_dir=config.VAL_MASK_DIR, batch_size=config.BATCH_SIZE, train_transform=config.train_transform, val_transform=config.val_transform, num_workers=config.NUM_WORKERS, pin_memory=config.PIN_MEMORY, ) if config.LOAD_MODEL: load_checkpoint(torch.load(config.CHECKPOINT_PTH), model) check_accuracy(val_loader, model) scaler = torch.cuda.amp.GradScaler() for epoch in range(config.NUM_EPOCHS): train_fn(train_loader, model, optimizer, BCE, scaler, val_loader) # save model if config.SAVE_MODEL: checkpoint = { "state_dict": model.state_dict(), "optimizer": optimizer.state_dict(), } save_checkpoint(checkpoint) # check accuracy check_accuracy(val_loader, model) # print some example save_predictions_as_imgs(val_loader, model, folder=config.SAVE_IMAGES)
def main(args): print('Dataset: {}, Normal Label: {}, LR: {}'.format( args.dataset, args.label, args.lr)) device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") print(device) model = utils.get_resnet_model(resnet_type=args.resnet_type) model = model.to(device) ewc_loss = None # Freezing Pre-trained model for EWC if args.ewc: frozen_model = deepcopy(model).to(device) frozen_model.eval() utils.freeze_model(frozen_model) fisher = torch.load(args.diag_path) ewc_loss = EWCLoss(frozen_model, fisher) utils.freeze_parameters(model) train_loader, test_loader = utils.get_loaders(dataset=args.dataset, label_class=args.label, batch_size=args.batch_size) train_model(model, train_loader, test_loader, device, args, ewc_loss)
def main(): train_transform = tr.Compose([ tr.Resize((160, 240)), tr.ToTensor(), tr.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)), ]) model = UNET(in_channels=3, out_channels=3).to(DEVICE) loss_fn = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE) train_loader, val_loader = get_loaders( TRAIN_IMG_DIR, TRAIN_BLUR_DIR, BATCH_SIZE, train_transform=train_transform, num_workers=NUM_WORKERS, pin_memory=PIN_MEMORY, ) if LOAD_MODEL: load_checkpoint(torch.load("my_checkpoint.pth.tar"), model) check_accuracy(val_loader, model, device=DEVICE) scaler = torch.cuda.amp.GradScaler() for epoch in range(NUM_EPOCHS): train_model(train_loader, model, optimizer, loss_fn, scaler) checkpoint = { "state_dict": model.state_dict(), "optimizer": optimizer.state_dict(), } save_checkpoint(checkpoint) check_accuracy(val_loader, model, device=DEVICE)
args = parser.parse_args() if __name__ == '__main__': # For easy debug # args.dataset = 'mnist' # args.metric = 'psnr' # args.net_G = 'unet_64' # args.pixel_loss = 'minimum_pixel_loss' # args.max_num_epochs = 200 # args.batch_size = 2 # args.enable_d1d2 = True # args.enable_d3 = True # args.enable_synfake = True dataloaders = utils.get_loaders(args) # you may run the following if you want to check whether the data is loading correctly # for i in range(100): # data = next(iter(dataloaders['train'])) # vis_input = utils.make_numpy_grid(data['input']) # vis_pred1 = utils.make_numpy_grid(data['gt1']) # vis_pred2 = utils.make_numpy_grid(data['gt2']) # vis = np.concatenate([vis_input, vis_pred1, vis_pred2], axis=0) # vis = np.clip(vis, a_min=0.0, a_max=1.0) # plt.imshow(vis) # plt.show() unmix = um.UnmixGAN(args=args, dataloaders=dataloaders) unmix.train_models()
help='batchsize', default=3000) argparser.add_argument( '--get_trfid', type=int, help= 'if this is 1, we compute training fids also, the value is either 0 or 1', default=1) arguments = argparser.parse_args() arguments.cuda = torch.cuda.is_available() arguments.data = 'mnist' arguments.input_type = 'autoenc' train_loader, test_loader = ut.get_loaders(arguments.batch_size, c=0, arguments=arguments) L1 = L2 = 784 M = N = 28 results = [] results_impl = [] Kss = [[K, 600] for K in range(20, 21, 20)] model = arguments.model num_samples = 3 base_path = os.path.expanduser('~') + '/Dropbox' model_path = base_path + '/two_step_learning/models' if model == 'VAE':
args.train, args.valid, args.test, load_vocab=args.load_vocab, vocab_file=args.vocab_file) torch.save(corpus, fn) if args.save_vocab: with open('{}/{}'.format(path, args.vocab_file), 'wb') as f: torch.save( [corpus.dictionary.word2idx, corpus.dictionary.idx2word], f) vocab_sz = len(corpus.dictionary) # Produce dataloaders train_loader = get_loaders(corpus.train, args.bs, args.bptt, use_var_bptt=args.use_var_bptt) valid_loader = get_loaders(corpus.valid, args.eval_bs, args.bptt) test_loader = get_loaders(corpus.test, args.eval_bs, args.bptt) # Construct encoder if args.encoder == 'awd_lstm': encoder = AWDLSTMEncoder(vocab_sz=vocab_sz, emb_dim=args.emb_dim, hidden_dim=args.hidden_dim, num_layers=args.num_layers, emb_dp=args.emb_dp, weight_dp=args.weight_dp, input_dp=args.input_dp, hidden_dp=args.hidden_dp, tie_weights=args.tie_weights)
def main(): args = get_args() if not os.path.exists(args.out_dir): os.mkdir(args.out_dir) logfile = os.path.join(args.out_dir, 'output.log') if os.path.exists(logfile): os.remove(logfile) logging.basicConfig(format='[%(asctime)s] - %(message)s', datefmt='%Y/%m/%d %H:%M:%S', level=logging.INFO, filename=os.path.join(args.out_dir, 'output.log')) logger.info(args) np.random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) torch.backends.cudnn.benchmark = True device = torch.device('cuda') train_loader, test_loader = get_loaders(args.data_dir, args.batch_size) epsilon = (args.epsilon / 255.) / std alpha = (args.alpha / 255.) / std model = WideResNet().cuda() opt = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) scheduler = torch.optim.lr_scheduler.MultiStepLR(opt, milestones=[75, 90], gamma=0.1) criterion = RFAT(args.beta) # Training start_train_time = time.time() logger.info('Epoch \t Train Loss \t Train Acc \t LR \t Time') for epoch in range(args.epochs): start_epoch_time = time.time() train_loss = 0 train_acc = 0 train_n = 0 model.train() for i, (X, y) in enumerate(train_loader): X, y = X.cuda(), y.cuda() delta = torch.zeros_like(X).cuda() for j in range(len(epsilon)): delta[:, j, :, :].uniform_(-epsilon[j][0][0].item(), epsilon[j][0][0].item()) delta.data = clamp(delta, lower_limit - X, upper_limit - X) delta.requires_grad = True output = model(X + delta) loss = F.cross_entropy(output, y) loss.backward() grad = delta.grad.detach() delta.data = clamp(delta + alpha * torch.sign(grad), -epsilon, epsilon) delta.data[:X.size(0)] = clamp(delta[:X.size(0)], lower_limit - X, upper_limit - X) delta = delta.detach() opt.zero_grad() output = model(X) adv_output = model(X + delta) loss = criterion(output, adv_output, y) loss.backward() opt.step() train_loss += loss.item() * y.size(0) train_acc += (output.max(1)[1] == y).sum().item() train_n += y.size(0) epoch_time = time.time() lr = opt.param_groups[0]['lr'] scheduler.step() logger.info('%d \t %.4f \t %.4f \t %.4f \t %.4f', epoch, train_loss / train_n, train_acc / train_n * 100., lr, epoch_time - start_epoch_time) train_time = time.time() best_state_dict = model.state_dict() torch.save(best_state_dict, os.path.join(args.out_dir, 'model.pth')) logger.info('Total train time: %.4f minutes', (train_time - start_train_time) / 60) # Evaluation model_test = WideResNet().cuda() model_test.load_state_dict(best_state_dict) model_test.float() model_test.eval() logger.info('Attack Iters \t Loss \t Acc') pgd_loss, pgd_acc = evaluate_standard(test_loader, model_test) logger.info('Nautral Test : %d \t %.4f \t %.4f', 0, pgd_loss, pgd_acc * 100.) pgd_loss, pgd_acc = evaluate_fgsm(test_loader, model_test) logger.info('FGSM Attack : %d \t %.4f \t %.4f', 1, pgd_loss, pgd_acc * 100.) pgd_loss, pgd_acc = evaluate_pgd(test_loader, model_test, 10, 1) logger.info('PGD Attack : %d \t %.4f \t %.4f', 10, pgd_loss, pgd_acc * 100.) pgd_loss, pgd_acc = evaluate_pgd(test_loader, model_test, 20, 1) logger.info('PGD Attack : %d \t %.4f \t %.4f', 20, pgd_loss, pgd_acc * 100.)
def main(): # TODO: Might be worth trying the normalization from assignment 2 train_transform = A.Compose([ A.Resize(height=IMAGE_HEIGHT, width=IMAGE_WIDTH), A.Rotate(limit=35, p=1.0), A.HorizontalFlip(p=0.5), A.VerticalFlip(p=0.1), A.Normalize( mean=[0.0, 0.0, 0.0], std=[1.0, 1.0, 1.0], max_pixel_value=255.0, ), ToTensorV2(), ], ) val_transforms = A.Compose([ A.Resize(height=IMAGE_HEIGHT, width=IMAGE_WIDTH), A.Normalize( mean=[0.0, 0.0, 0.0], std=[1.0, 1.0, 1.0], max_pixel_value=255.0, ), ToTensorV2(), ], ) model = UNET(in_channels=3, out_channels=1).to(DEVICE) """ We're using with logitsLoss because we're not using sigmoid on the, final output layer. If we wanted to have several output channels, we'd change the loss_fn to a cross entropy loss instead. """ loss_fn = nn.BCEWithLogitsLoss() optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE) train_loader, val_loader = get_loaders( TRAIN_IMG_DIR, TRAIN_MASK_DIR, VAL_IMG_DIR, VAL_MASK_DIR, BATCH_SIZE, train_transform, val_transforms, NUM_WORKERS, PIN_MEMORY, ) if LOAD_MODEL: load_checkpoint(torch.load("my_checkpoint.pth.tar"), model) scaler = torch.cuda.amp.GradScaler( ) # Scales the gradients to avoid underflow. Requires a GPU for epoch in range(NUM_EPOCHS): train_fn(train_loader, model, optimizer, loss_fn, scaler) # save model checkpoint = { "state_dict": model.state_dict(), "optimizer": optimizer.state_dict(), } save_checkpoint(checkpoint) # check accuracy check_accuracy(val_loader, model, device=DEVICE) # print some examples to a folder save_predictions_as_imgs(val_loader, model, folder="saved_images/", device=DEVICE)
assert isinstance(model, HateSpeechClassifier) if __name__ == "__main__": tweets = np.load("data/tweets.npy") labels = np.load("data/hate_original.npy") with open('vocab_to_int.json', 'r') as fp: vocab_to_int = json.load(fp) with open('int_to_vocab.json', 'r') as fp: int_to_vocab = json.load(fp) train_on_gpu = torch.cuda.is_available() print("Testing on GPU." if train_on_gpu else "Testing on CPU.") train_loader, valid_loader, test_loader = get_loaders(tweets, labels, prnt=False) print("Testing HateSpeechClassifier class...\n") _test_HateSpeechClassifier() print("Testing forward back propagation function...\n") _test_forward_back_prop(HateSpeechClassifier, forward_back_prop, train_on_gpu) sequence_length = tweets.shape[1] # number of words in a sequence n_epochs = 1 learning_rate = 0.01 vocab_size = len(vocab_to_int) output_size = pd.Series(labels).nunique() embedding_dim = 10 hidden_dim = 16 batch_size = 64
def main(): check_path(SAVE_PATH) train_transform = A.Compose([ A.Resize(height=IMAGE_HEIGHT, width=IMAGE_WIDTH), A.OneOf([ A.Rotate(limit=35, p=0.5), A.HorizontalFlip(p=0.5), A.VerticalFlip(p=0.5), A.RandomRotate90(p=0.5), A.Transpose(p=0.5) ]), A.Normalize(mean=[0.625, 0.448, 0.688], std=[0.131, 0.177, 0.101], max_pixel_value=255.0), ToTensorV2(), ]) model = get_model(data_channel=CHANNEL_NUM, encoder=ENCODER, encoder_weight=ENCODER_WEIGHT).to(device=DEVICE) print(model) loss_fn = nn.CrossEntropyLoss().to(device=DEVICE) optimizer = optim.Adam(params=model.parameters(), lr=LEARNING_RATE) ##plot plot_train_loss = [] plot_val_loss = [] plot_dice = [] plot_miou = [] learning_lr = [] # Define Scheduler #scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,factor=0.1, patience=10, # verbose=True) scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, cycle_momentum=False, base_lr=1.25e-4, max_lr=0.001, step_size_up=2000, mode="triangular2", verbose=False) best_iou = 0 best_dice = 0 train_loader, val_loader = get_loaders(train_dir=TRAIN_IMG_DIR, train_maskdir=TRAIN_MASK_DIR, batch_size=BATCH_SIZE, train_transform=train_transform, num_workers=NUM_WORKS, pin_memory=PIN_MEMORY) scaler = torch.cuda.amp.GradScaler() for epoch in range(NUM_EPOCHS): epoch_loss, current_lr = train_fn(train_loader, model, optimizer, scheduler, loss_fn, scaler, epoch, aux_loss='lovasz_softmax') plot_train_loss.append(epoch_loss) print(epoch_loss) learning_lr.append(current_lr) #save_checkpoint(check_point, filename=f"/data3/mry/results/best_checkpoint_{flod_idx}fold_{epoch}epoch.pth.tar") ##check valid metric m_dice, miou, val_loss = check_valid_metric(val_loader, model, device=DEVICE, loss_fn=loss_fn, aux_loss='lovasz_softmax', channel_nums=CHANNEL_NUM) plot_val_loss.append(val_loss if val_loss < 100 else 100) plot_dice.append(m_dice) plot_miou.append(miou) if best_iou < miou: best_iou = miou ##save model check_point = { 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict() } save_checkpoint(check_point, filename=f"{SAVE_PATH}{epoch}epoch.pth.tar") ##plot metric and save fig = plt.figure(figsize=(24, 12)) x = [i for i in range(epoch + 1)] ax = fig.add_subplot(2, 3, 1) ax.plot(x, plot_train_loss, label='train loss') ax.set_xlabel('Epoch') ax.set_ylabel('train loss') ax.grid(True) ax = fig.add_subplot(2, 3, 2) ax.plot(x, plot_val_loss, label='val loss') ax.set_xlabel('Epoch') ax.set_ylabel('val loss') ax.grid(True) ax = fig.add_subplot(2, 3, 3) ax.plot(x, learning_lr, label='Learning Rate') ax.set_xlabel('Epoch') ax.set_ylabel('Learning Rate') ax.grid(True) ax = fig.add_subplot(2, 3, 4) ax.plot(x, plot_miou, label='mIOU') ax.set_xlabel('Epoch') ax.set_ylabel('mIOU') ax.grid(True) ax = fig.add_subplot(2, 3, 5) ax.plot(x, plot_dice, label='mDICE') ax.set_xlabel('Epoch') ax.set_ylabel('mDICE') ax.grid(True) fig.savefig(PLOT_PATH) plt.show()
def main(): parser = argparse.ArgumentParser(description='PixelCNN') parser.add_argument('--epochs', type=int, default=25, help='Number of epochs to train model for') parser.add_argument('--batch-size', type=int, default=32, help='Number of images per mini-batch') parser.add_argument( '--dataset', type=str, default='mnist', help='Dataset to train model on. Either mnist, fashionmnist or cifar.') parser.add_argument('--causal-ksize', type=int, default=7, help='Kernel size of causal convolution') parser.add_argument('--hidden-ksize', type=int, default=7, help='Kernel size of hidden layers convolutions') parser.add_argument( '--color-levels', type=int, default=2, help= 'Number of levels to quantisize value of each channel of each pixel into' ) parser.add_argument( '--hidden-fmaps', type=int, default=30, help='Number of feature maps in hidden layer (must be divisible by 3)') parser.add_argument('--out-hidden-fmaps', type=int, default=10, help='Number of feature maps in outer hidden layer') parser.add_argument( '--hidden-layers', type=int, default=6, help='Number of layers of gated convolutions with mask of type "B"') parser.add_argument('--learning-rate', '--lr', type=float, default=0.0001, help='Learning rate of optimizer') parser.add_argument('--weight-decay', type=float, default=0.0001, help='Weight decay rate of optimizer') parser.add_argument('--max-norm', type=float, default=1., help='Max norm of the gradients after clipping') parser.add_argument('--epoch-samples', type=int, default=25, help='Number of images to sample each epoch') parser.add_argument('--cuda', type=str2bool, default=True, help='Flag indicating whether CUDA should be used') cfg = parser.parse_args() wandb.init(project="PixelCNN") wandb.config.update(cfg) torch.manual_seed(42) EPOCHS = cfg.epochs model = PixelCNN(cfg=cfg) device = torch.device( "cuda" if torch.cuda.is_available() and cfg.cuda else "cpu") model.to(device) train_loader, test_loader, HEIGHT, WIDTH = get_loaders( cfg.dataset, cfg.batch_size, cfg.color_levels, TRAIN_DATASET_ROOT, TEST_DATASET_ROOT) optimizer = optim.Adam(model.parameters(), lr=cfg.learning_rate, weight_decay=cfg.weight_decay) scheduler = optim.lr_scheduler.CyclicLR(optimizer, cfg.learning_rate, 10 * cfg.learning_rate, cycle_momentum=False) wandb.watch(model) losses = [] params = [] for epoch in range(EPOCHS): train(cfg, model, device, train_loader, optimizer, scheduler, epoch) test_and_sample(cfg, model, device, test_loader, HEIGHT, WIDTH, losses, params, epoch) print('\nBest test loss: {}'.format(np.amin(np.array(losses)))) print('Best epoch: {}'.format(np.argmin(np.array(losses)) + 1)) best_params = params[np.argmin(np.array(losses))] if not os.path.exists(MODEL_PARAMS_OUTPUT_DIR): os.mkdir(MODEL_PARAMS_OUTPUT_DIR) MODEL_PARAMS_OUTPUT_FILENAME = '{}_cks{}hks{}cl{}hfm{}ohfm{}hl{}_params.pth'\ .format(cfg.dataset, cfg.causal_ksize, cfg.hidden_ksize, cfg.color_levels, cfg.hidden_fmaps, cfg.out_hidden_fmaps, cfg.hidden_layers) torch.save( best_params, os.path.join(MODEL_PARAMS_OUTPUT_DIR, MODEL_PARAMS_OUTPUT_FILENAME))
def main(): # get user arguments args = get_command_line_args() # determine if gpu support is enabled has_gpu_support = torch.cuda.is_available() and args.gpu print("Data directory:", args.data_dir) if has_gpu_support: print("GPU support enabled. Training on GPU.") else: print("GPU support disabled. Training on CPU.") print("Selected architecture:", args.arch) if args.save_dir: print("Directory for checkpoint saving:", args.save_dir) print("Learning rate:", args.learning_rate) print("Hidden layers:", args.hidden_units) print("Epochs:", args.epochs) # Get data loaders dataloaders, class_to_idx = utils.get_loaders(args.data_dir) for key, value in dataloaders.items(): print("{} data loader retrieved".format(key)) # Build the model model, optimizer, criterion = utils.build_model(args.arch, args.hidden_units, args.learning_rate) model.class_to_idx = class_to_idx # check for gpu support if has_gpu_support: print("Moving tensors to GPU...") # move tensors to gpu model.cuda() criterion.cuda() # Train the model utils.train_model(model, args.epochs, criterion, optimizer, dataloaders, has_gpu_support) # Save the checkpoint if args.save_dir: has_save_dir = os.path.exists(args.save_dir) if not os.path.exists(args.save_dir): os.makedirs(args.save_dir) save_path = args.save_dir + '/' + args.arch + '_checkpoint.pth' else: save_path = args.arch + '_checkpoint.pth' print("About to save checkpoint to {}...".format(save_path)) # save checkpoint save(args.arch, args.learning_rate, args.hidden_units, args.epochs, save_path, model, optimizer) print("Checkpoint has been saved") # get test loss and accuracy accuracy_rate, test_loss_rate = utils.validate(model, criterion, dataloaders['test'], has_gpu_support) print("Test Accuracy: {:.2f}".format(accuracy_rate)) print("Test Loss: {:.2f}".format(test_loss_rate))
def main(): args = get_args() if not os.path.exists(args.out_dir): os.mkdir(args.out_dir) logfile = os.path.join(args.out_dir, 'output.log') if os.path.exists(logfile): os.remove(logfile) logging.basicConfig( format='[%(asctime)s] - %(message)s', datefmt='%Y/%m/%d %H:%M:%S', level=logging.INFO, filename=logfile) logger.info(args) np.random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) train_loader, test_loader = get_loaders(args.data_dir, args.batch_size) epsilon = (args.epsilon / 255.) / std model = PreActResNet18().cuda() model.train() opt = torch.optim.SGD(model.parameters(), lr=args.lr_max, momentum=args.momentum, weight_decay=args.weight_decay) amp_args = dict(opt_level=args.opt_level, loss_scale=args.loss_scale, verbosity=False) if args.opt_level == 'O2': amp_args['master_weights'] = args.master_weights model, opt = amp.initialize(model, opt, **amp_args) criterion = nn.CrossEntropyLoss() delta = torch.zeros(args.batch_size, 3, 32, 32).cuda() delta.requires_grad = True lr_steps = args.epochs * len(train_loader) * args.minibatch_replays if args.lr_schedule == 'cyclic': scheduler = torch.optim.lr_scheduler.CyclicLR(opt, base_lr=args.lr_min, max_lr=args.lr_max, step_size_up=lr_steps / 2, step_size_down=lr_steps / 2) elif args.lr_schedule == 'multistep': scheduler = torch.optim.lr_scheduler.MultiStepLR(opt, milestones=[lr_steps / 2, lr_steps * 3 / 4], gamma=0.1) # Training start_train_time = time.time() logger.info('Epoch \t Seconds \t LR \t \t Train Loss \t Train Acc') for epoch in range(args.epochs): start_epoch_time = time.time() train_loss = 0 train_acc = 0 train_n = 0 for i, (X, y) in enumerate(train_loader): X, y = X.cuda(), y.cuda() for _ in range(args.minibatch_replays): output = model(X + delta[:X.size(0)]) loss = criterion(output, y) opt.zero_grad() with amp.scale_loss(loss, opt) as scaled_loss: scaled_loss.backward() grad = delta.grad.detach() delta.data = clamp(delta + epsilon * torch.sign(grad), -epsilon, epsilon) delta.data[:X.size(0)] = clamp(delta[:X.size(0)], lower_limit - X, upper_limit - X) opt.step() delta.grad.zero_() scheduler.step() train_loss += loss.item() * y.size(0) train_acc += (output.max(1)[1] == y).sum().item() train_n += y.size(0) epoch_time = time.time() lr = scheduler.get_lr()[0] logger.info('%d \t %.1f \t \t %.4f \t %.4f \t %.4f', epoch, epoch_time - start_epoch_time, lr, train_loss/train_n, train_acc/train_n) train_time = time.time() torch.save(model.state_dict(), os.path.join(args.out_dir, 'model.pth')) logger.info('Total train time: %.4f minutes', (train_time - start_train_time)/60) # Evaluation model_test = PreActResNet18().cuda() model_test.load_state_dict(model.state_dict()) model_test.float() model_test.eval() pgd_loss, pgd_acc = evaluate_pgd(test_loader, model_test, 50, 10) test_loss, test_acc = evaluate_standard(test_loader, model_test) logger.info('Test Loss \t Test Acc \t PGD Loss \t PGD Acc') logger.info('%.4f \t \t %.4f \t %.4f \t %.4f', test_loss, test_acc, pgd_loss, pgd_acc)
def train(args): json_options = json_file_to_pyobj(args.config) training_configurations = json_options.training wrn_depth = training_configurations.wrn_depth wrn_width = training_configurations.wrn_width dataset = training_configurations.dataset.lower() seeds = [int(seed) for seed in training_configurations.seeds] log = True if training_configurations.log.lower() == 'true' else False if log: logfile = 'WideResNet-{}-{}-{}.txt'.format( wrn_depth, wrn_width, training_configurations.dataset) with open(logfile, 'w') as temp: temp.write('WideResNet-{}-{} on {}\n'.format( wrn_depth, wrn_width, training_configurations.dataset)) else: logfile = '' checkpoint = True if training_configurations.checkpoint.lower( ) == 'true' else False loaders = get_loaders(dataset) if torch.cuda.is_available(): device = torch.device('cuda:0') else: device = torch.device('cpu') test_set_accuracies = [] for seed in seeds: set_seed(seed) if log: with open(logfile, 'a') as temp: temp.write( '------------------- SEED {} -------------------\n'.format( seed)) strides = [1, 1, 2, 2] net = WideResNet(d=wrn_depth, k=wrn_width, n_classes=10, input_features=3, output_features=16, strides=strides) net = net.to(device) checkpointFile = 'wrn-{}-{}-seed-{}-{}-dict.pth'.format( wrn_depth, wrn_width, dataset, seed) if checkpoint else '' best_test_set_accuracy = _train_seed(net, loaders, device, dataset, log, checkpoint, logfile, checkpointFile) if log: with open(logfile, 'a') as temp: temp.write('Best test set accuracy of seed {} is {}\n'.format( seed, best_test_set_accuracy)) test_set_accuracies.append(best_test_set_accuracy) mean_test_set_accuracy, std_test_set_accuracy = np.mean( test_set_accuracies), np.std(test_set_accuracies) if log: with open(logfile, 'a') as temp: temp.write( 'Mean test set accuracy is {} with standard deviation equal to {}\n' .format(mean_test_set_accuracy, std_test_set_accuracy))
type=int, default='100') parser.add_argument( '--arch', type=str, default='resnet_34', choices=('AlexNet', 'vgg_16_bn','resnet_34','vgg_19_bn','mobile_net_v1'), help='The architecture to prune') args = parser.parse_args() lr_decay_step = list(map(int, args.lr_decay_step.split(','))) trainloader,testloader = get_loaders(args.dataset, args.data_dir,args.train_batch_size,args.eval_batch_size,args.arch) project_root_path = os.path.abspath(os.path.dirname(__file__)) if sys.platform == 'linux': tmp_dir = '/content/drive/MyDrive/' else: tmp_dir = os.path.join(project_root_path, 'tmp') if not Path(tmp_dir).exists(): os.mkdir(tmp_dir) print(vars(args)) # Model device = 'cuda:0' if torch.cuda.is_available() else 'cpu'
STRT_EPOCH, best_val_loss = 0, 10.0 # cuda and GPU settings if args.gpu == 99: net = torch.nn.DataParallel(net, device_ids=[0, 1]).cuda() else: torch.cuda.set_device(args.gpu) net.cuda() cudnn.benchmark = True criterion = nn.CrossEntropyLoss().cuda() optimizer = optim.Adam(net.parameters(), lr=args.lr) train_loader, val_loader, len_train = get_loaders(args.batch_size, args.num_workers, args.imsize) # training loop def train(): net.train() # keep track of accuracy total = 0 correct = 0 # keep track of losses iter_loss = 0. iter_correct = 0. num_batch_epoch = 0 for i, data in enumerate(train_loader): # get the inputs