def main(): config = [(64, 3, 1, 1), (64, 3, 1, 1), (1, 3, 1, 1)] #config = [(64, 9, 1, 4), (32, 1, 1, 0), (3, 5, 1, 2)] #config = [(64, 9, 1, 0), (32, 1, 1, 0), (3, 5, 1, 0)] # config: (output_ch, kernel_size, stride, padding_size) model = SRCNN(config).to(DEVICE) loss_function = nn.MSELoss(reduction='mean') optimizer = optim.Adam(model.parameters(), lr=0.0001) test_data = generate_data('test') for epoch in range(EPOCH): train_data = generate_data('train') train(model, train_data, loss_function, optimizer, DEVICE) test(model, test_data, loss_function, epoch, DEVICE)
def main_(model_main): #def main(CFG): model = model_main.to(CFG['device']) # 模型训练、补训、测试 ################################################################# if CFG['train_or_test'] =='train': #optimizer = optim.Adadelta(model.parameters(), lr=CFG['lr']) # 求解器 optimizer = optim.SGD(model.parameters(), lr=CFG['lr'], momentum=CFG['momentum'],) scheduler = StepLR(optimizer, step_size=2, gamma=CFG['gamma']) # 学习速率衰减方式 result_trace = np.zeros([1,7]) loss_trace = np.zeros([1,12]) # 逐个记录每个epoch的loss和acc acc_trace = np.zeros([1,12]) for epoch in range(0,20): # 遍历每个epoch start_time = datetime.datetime.now() # 训练开始时间 train_loss_epoch_i, train_acc_epoch_i = train(CFG, model, train_loader, optimizer, epoch) # end_time = datetime.datetime.now(); time_cost = (end_time - start_time).seconds ; print('耗时:',time_cost) # 训练耗时 test_loss_epoch_i, test_acc_epoch_i, f4t_and_label = test(CFG, model, test_loader, is_print=True) result_epoch_i = [epoch, train_acc_epoch_i[0], train_loss_epoch_i[0], test_acc_epoch_i[0],test_loss_epoch_i[0], scheduler.get_lr()[0],time_cost] result_trace = np.vstack([result_trace, np.array(result_epoch_i).reshape(1,len(result_epoch_i))]) loss_trace = np.vstack([loss_trace, np.array([train_loss_epoch_i + test_loss_epoch_i]).reshape(1,12)]) acc_trace = np.vstack([acc_trace, np.array([train_acc_epoch_i + test_acc_epoch_i]).reshape(1,12)]) if epoch>0: scheduler.step() if CFG['save_model'] : pt_name = '[STI]_' + model.name+ '_'+CFG['dataset_choose'] +'_epoch'+str(0)+'to'+str(epoch) +'_sample'+str(CFG['NO_train'][0])+'to'+str(CFG['NO_train'][1]) # 文件主命名, STI表示Source Trained In # plot_curve(result_trace[:,0],[result_trace[:,1],result_trace[:,3]],'结果图/'+pt_name+'_ACC.png', xlabel='Epoch',ylabel='ACC',title='ACC',legend=['Training_Accuracy','Testing_Accuracy']) pt = {'model':model.state_dict(), 'optimizer':optimizer.state_dict(),'scheduler':scheduler.state_dict(),'CFG':CFG, 'model_name':model.name, 'result_trace':result_trace, 'loss_trace':loss_trace, 'acc_trace':acc_trace } torch.save(pt, pt_name + '_' + device.type + '_rand' + str(CFG['random_seed']) + '.pt')
def main(): train_loader, test_loader, criterion, model, optimizer, scheduler, \ starting_epoch, logfilename, model_path, device, writer = prologue(args) for epoch in range(starting_epoch, args.epochs): before = time.time() train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch, args.noise_sd, device, writer) test_loss, test_acc = test(test_loader, model, criterion, epoch, args.noise_sd, device, writer, args.print_freq) after = time.time() log( logfilename, "{}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}".format( epoch, after - before, scheduler.get_lr()[0], train_loss, train_acc, test_loss, test_acc)) # In PyTorch 1.1.0 and later, you should call `optimizer.step()` before `lr_scheduler.step()`. # See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate scheduler.step(epoch) torch.save( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), }, model_path)
def main(): generator = srgan.SRGAN_gen().to(device) discriminator = srgan.SRGAN_dis().to(device) params = list(generator.parameters()) + list(discriminator.parameters()) optimizer = optim.Adam(params, lr=1e-4) trainset = TrainDataset() train_loader = DataLoader(dataset=trainset, batch_size=BATCH_SIZE, shuffle=True) test_data = Image.open('./SR_dataset/Set5/001_HR.png') test_data = transforms.ToTensor()(test_data) test_data = test_data.unsqueeze(0) test_data = test_data.to(device) for epoch in range(10000): train(generator, discriminator, optimizer, train_loader, device, epoch) if epoch % 1000 == 0: test(generator, discriminator, test_data, epoch, device)
def main(): train_loader, test_loader, criterion, model, optimizer, scheduler, \ starting_epoch, logfilename, model_path, device, writer = prologue(args) if args.attack == 'PGD': print('Attacker is PGD') attacker = PGD_L2(steps=args.num_steps, device=device, max_norm=args.epsilon) elif args.attack == 'DDN': print('Attacker is DDN') attacker = DDN(steps=args.num_steps, device=device, max_norm=args.epsilon, init_norm=args.init_norm_DDN, gamma=args.gamma_DDN) else: raise Exception('Unknown attack') for epoch in range(starting_epoch, args.epochs): attacker.max_norm = np.min( [args.epsilon, (epoch + 1) * args.epsilon / args.warmup]) attacker.init_norm = np.min( [args.epsilon, (epoch + 1) * args.epsilon / args.warmup]) before = time.time() train_loss, train_acc = train(train_loader, model, criterion, optimizer, epoch, args.noise_sd, attacker, device, writer) test_loss, test_acc = test(test_loader, model, criterion, epoch, args.noise_sd, device, writer, args.print_freq) after = time.time() log( logfilename, "{}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}".format( epoch, after - before, scheduler.get_lr()[0], train_loss, train_acc, test_loss, test_acc)) # In PyTorch 1.1.0 and later, you should call `optimizer.step()` before `lr_scheduler.step()`. # See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate scheduler.step(epoch) torch.save( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), }, model_path)
def predict(self, X): args = self.args test_x = X test_y = np.zeros(X.shape[0]) test_loader = basic_loader(test_x, test_y, args.batch_size, train_shuffle=False) model = self.model model.load_state_dict(self.best_state) test_loss, test_score, output = test(args, self.model, self.device, test_loader, 'test', return_output=True) return torch.cat(output, dim=0).numpy()
def main(tiny_images=None, model="cnn", augment=False, use_scattering=False, batch_size=2048, mini_batch_size=256, lr=1, lr_start=None, optim="SGD", momentum=0.9, noise_multiplier=1, max_grad_norm=0.1, epochs=100, bn_noise_multiplier=None, max_epsilon=None, data_size=550000, delta=1e-6, logdir=None): logger = Logger(logdir) device = get_device() bs = batch_size assert bs % mini_batch_size == 0 n_acc_steps = bs // mini_batch_size train_data, test_data = get_data("cifar10", augment=augment) train_loader = torch.utils.data.DataLoader(train_data, batch_size=100, shuffle=False, num_workers=4, pin_memory=True) test_loader = torch.utils.data.DataLoader(test_data, batch_size=100, shuffle=False, num_workers=4, pin_memory=True) if isinstance(tiny_images, torch.utils.data.Dataset): train_data_aug = tiny_images else: print("loading tiny images...") train_data_aug, _ = get_data("cifar10_500K", augment=augment, aux_data_filename=tiny_images) scattering, K, (h, w) = None, None, (None, None) pre_scattered = False if use_scattering: scattering, K, (h, w) = get_scatter_transform("cifar10_500K") scattering.to(device) # if the whole data fits in memory, pre-compute the scattering if use_scattering and data_size <= 50000: loader = torch.utils.data.DataLoader(train_data_aug, batch_size=100, shuffle=False, num_workers=4) train_data_aug = get_scattered_dataset(loader, scattering, device, data_size) pre_scattered = True assert data_size <= len(train_data_aug) num_sup = min(data_size, 50000) num_batches = int(np.ceil(50000 / mini_batch_size)) # cifar-10 equivalent train_batch_sampler = SemiSupervisedSampler(data_size, num_batches, mini_batch_size) train_loader_aug = torch.utils.data.DataLoader( train_data_aug, batch_sampler=train_batch_sampler, num_workers=0 if pre_scattered else 4, pin_memory=not pre_scattered) rdp_norm = 0 if model == "cnn": if use_scattering: save_dir = f"bn_stats/cifar10_500K" os.makedirs(save_dir, exist_ok=True) bn_stats, rdp_norm = scatter_normalization( train_loader, scattering, K, device, data_size, num_sup, noise_multiplier=bn_noise_multiplier, orders=ORDERS, save_dir=save_dir) model = CNNS["cifar10"](K, input_norm="BN", bn_stats=bn_stats) model = model.to(device) if not pre_scattered: model = nn.Sequential(scattering, model) else: model = CNNS["cifar10"](in_channels=3, internal_norm=False) elif model == "linear": save_dir = f"bn_stats/cifar10_500K" os.makedirs(save_dir, exist_ok=True) bn_stats, rdp_norm = scatter_normalization( train_loader, scattering, K, device, data_size, num_sup, noise_multiplier=bn_noise_multiplier, orders=ORDERS, save_dir=save_dir) model = ScatterLinear(K, (h, w), input_norm="BN", bn_stats=bn_stats) model = model.to(device) if not pre_scattered: model = nn.Sequential(scattering, model) else: raise ValueError(f"Unknown model {model}") model.to(device) if pre_scattered: test_loader = get_scattered_loader(test_loader, scattering, device) print(f"model has {get_num_params(model)} parameters") if optim == "SGD": optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=momentum) else: optimizer = torch.optim.Adam(model.parameters(), lr=lr) privacy_engine = PrivacyEngine( model, bs, data_size, alphas=ORDERS, noise_multiplier=noise_multiplier, max_grad_norm=max_grad_norm, ) privacy_engine.attach(optimizer) best_acc = 0 flat_count = 0 for epoch in range(0, epochs): print(f"\nEpoch: {epoch} ({privacy_engine.steps} steps)") train_loss, train_acc = train(model, train_loader_aug, optimizer, n_acc_steps=n_acc_steps) test_loss, test_acc = test(model, test_loader) if noise_multiplier > 0: print(f"sample_rate={privacy_engine.sample_rate}, " f"mul={privacy_engine.noise_multiplier}, " f"steps={privacy_engine.steps}") rdp_sgd = get_renyi_divergence( privacy_engine.sample_rate, privacy_engine.noise_multiplier) * privacy_engine.steps epsilon, _ = get_privacy_spent(rdp_norm + rdp_sgd, target_delta=delta) epsilon2, _ = get_privacy_spent(rdp_sgd, target_delta=delta) print(f"ε = {epsilon:.3f} (sgd only: ε = {epsilon2:.3f})") if max_epsilon is not None and epsilon >= max_epsilon: return else: epsilon = None logger.log_epoch(epoch, train_loss, train_acc, test_loss, test_acc, epsilon) logger.log_scalar("epsilon/train", epsilon, epoch) logger.log_scalar("cifar10k_loss/train", train_loss, epoch) logger.log_scalar("cifar10k_acc/train", train_acc, epoch) if test_acc > best_acc: best_acc = test_acc flat_count = 0 else: flat_count += 1 if flat_count >= 20: print("plateau...") return
def main(): args = parse_args() args.num_gpus = len(get_available_gpus()) eval(args.NET + '_setup')(args) set_seed(args.seed) setup(args) # Create model and optimizer if args.resume or args.eval or args.benchmark: last_epoch, best_epoch, best_val_loss, num_params, \ enc_params, dec_params = parse_experiment(args.odir) i = last_epoch if args.eval or args.benchmark: i = best_epoch args.resume = model_at(args, i) model, stats = tf_resume(args, i) else: check_overwrite(os.path.join(args.odir, 'trainlog.txt')) model = eval(args.NET + '_create_model')(args) stats = [] print('Will save to ' + args.odir) if not os.path.exists(args.odir): os.makedirs(args.odir) if not os.path.exists(args.odir + '/models'): os.makedirs(args.odir + '/models') with open(os.path.join(args.odir, 'cmdline.txt'), 'w') as f: f.write(" ".join([ "'" + a + "'" if (len(a) == 0 or a[0] != '-') else a for a in sys.argv ])) args.model = model args.step = eval(args.NET + '_step') # Training loop epoch = args.start_epoch train_data_queue, train_data_processes = data_setup(args, 'train', args.nworkers, repeat=True) if args.eval == 0: for epoch in range(args.start_epoch, args.epochs): print('Epoch {}/{} ({}):'.format(epoch + 1, args.epochs, args.odir)) loss = train(args, epoch, train_data_queue, train_data_processes)[0] if (epoch + 1) % args.test_nth_epoch == 0 or epoch + 1 == args.epochs: loss_val = test('val', args)[0] print('-> Train Loss: {}, \tVal loss: {}'.format( loss, loss_val)) stats.append({ 'epoch': epoch + 1, 'loss': loss, 'loss_val': loss_val }) else: loss_val = 0 print('-> Train loss: {}'.format(loss)) stats.append({'epoch': epoch + 1, 'loss': loss}) if (epoch + 1) % args.save_nth_epoch == 0 or epoch + 1 == args.epochs: with open(os.path.join(args.odir, 'trainlog.txt'), 'w') as outfile: json.dump(stats, outfile) save_model(args, epoch) if (epoch + 1) % args.test_nth_epoch == 0 and epoch + 1 < args.epochs: split = 'val' predictions = samples(split, args, 20) cache_pred(predictions, split, args) metrics(split, args, epoch) if math.isnan(loss): break if len(stats) > 0: with open(os.path.join(args.odir, 'trainlog.txt'), 'w') as outfile: json.dump(stats, outfile) kill_data_processes(train_data_queue, train_data_processes) split = 'val' predictions = samples(split, args, 20) cache_pred(predictions, split, args) metrics(split, args, epoch) if args.benchmark: benchmark_results('test', args)
def fit(self, X, y): x_train, x_valid, y_train, y_valid = train_test_split(X, y, test_size=0.2) # from some github repo... torch.multiprocessing.set_sharing_strategy('file_system') args = self.args args.input_dim = X.shape[1] args.output_dim = 1 args.task = 'regression' use_cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) np.random.seed(args.seed) device = torch.device("cuda" if use_cuda else "cpu") self.device = device train_loader = basic_loader(x_train, y_train, args.batch_size) valid_loader = basic_loader(x_valid, y_valid, args.batch_size, train_shuffle=False) # train_loader, valid_loader, test_loader = get_data_loaders(args.dataset, args.batch_size, # sub_task=args.sub_task, dim=args.input_dim) # if args.dataset in ['sider_split/', 'tox21_split/']: # args.dataset = args.dataset[:-1] + '-' + str(args.sub_task) print('batch number: train={}, valid={}'.format( len(train_loader), len(valid_loader))) model = Net(input_dim=args.input_dim, output_dim=args.output_dim, hidden_dim=args.hidden_dim, num_layer=args.depth, num_back_layer=args.back_n, dense=True, drop_type=args.drop_type, net_type=args.net_type, approx=args.anneal, device=device).to(device) self.model = model if args.optimizer == 'SGD': optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, nesterov=True) elif args.optimizer == 'AMSGrad': optimizer = optim.Adam(model.parameters(), lr=args.lr, amsgrad=True) scheduler = StepLR(optimizer, step_size=args.lr_step_size, gamma=args.gamma) best_score = -1e30 start_epoch = 1 # start from epoch 1 or last checkpoint epoch if args.anneal == 'approx': args.net_type = 'approx_' + args.net_type best_model_name = './checkpoint/{}/{}/best_seed{}_depth{}_ckpt.t7'.format( args.dataset.strip('/'), args.net_type, args.seed, args.depth) last_model_name = './checkpoint/{}/{}/last_seed{}_depth{}_ckpt.t7'.format( args.dataset.strip('/'), args.net_type, args.seed, args.depth) best_log_file = 'log/' + args.dataset.strip( '/') + '/{}/depth{}_backn{}_drop{}_p{}_best.log'.format( args.net_type, args.depth, args.back_n, args.drop_type, args.p) last_log_file = 'log/' + args.dataset.strip( '/') + '/{}/depth{}_backn{}_drop{}_p{}_last.log'.format( args.net_type, args.depth, args.back_n, args.drop_type, args.p) model_dir = './checkpoint/{}/{}/'.format(args.dataset.strip('/'), args.net_type) if not os.path.exists(model_dir): os.makedirs(model_dir) log_dir = 'log/' + args.dataset.strip('/') + '/{}/'.format( args.net_type) if not os.path.exists(log_dir): os.makedirs(log_dir) if not os.path.isdir('checkpoint'): os.mkdir('checkpoint') for epoch in range(start_epoch, args.epochs + start_epoch): scheduler.step(epoch) alpha = get_alpha(epoch, args.epochs) train_approximate_loss = train(args, model, device, train_loader, optimizer, epoch, args.anneal, alpha) # used for plotting learning curves train_loss, train_score = test(args, model, device, train_loader, 'train') valid_loss, valid_score = test(args, model, device, valid_loader, 'valid') # test_loss, test_score = test(args, model, device, test_loader, 'test') print(train_score, valid_score) # early stopping version if valid_score > best_score: self.best_state = model.state_dict() state = {'model': model.state_dict()} torch.save(state, best_model_name) best_score = valid_score # "convergent" version state = {'model': model.state_dict()} torch.save(state, last_model_name) # print('Training finished. Loading models from validation...') # for model_name, log_file, setting in zip([best_model_name, last_model_name], [best_log_file, last_log_file], # ['best', 'last']): # print('\nLoading the {} model...'.format(setting)) # # checkpoint = torch.load(model_name) # model.load_state_dict(checkpoint['model']) # train_loss, train_score = test(args, model, device, train_loader, 'train') # valid_loss, valid_score = test(args, model, device, valid_loader, 'valid') # test_loss, test_score = test(args, model, device, test_loader, 'test ') return self
def main(): parser = argparse.ArgumentParser() parser.add_argument('--DATASET_PATH', type=str, default='/home/zhangdong/database/DUTS/') parser.add_argument('--WEIGHTS_PATH', type=str, default='/home/yangle/DAVIS/result/models/') parser.add_argument('--EXPERIMENT', type=str, default='/home/yangle/DAVIS/result/TrainNet/') parser.add_argument('--N_EPOCHS', type=int, default=200) parser.add_argument('--MAX_PATIENCE', type=int, default=30) parser.add_argument('--batch_size', type=int, default=32) parser.add_argument('--seed', type=int, default=0) parser.add_argument('--N_CLASSES', type=int, default=2) parser.add_argument('--LEARNING_RATE', type=float, default=1e-4) parser.add_argument('--LR_DECAY', type=float, default=0.995) parser.add_argument('--DECAY_LR_EVERY_N_EPOCHS', type=int, default=1) parser.add_argument('--WEIGHT_DECAY', type=float, default=0.0001) parser.add_argument('--CUDNN', type=bool, default=True) args = parser.parse_args() torch.cuda.manual_seed(args.seed) cudnn.benchmark = args.CUDNN normalize = transforms.Normalize(mean=saliency.mean, std=saliency.std) train_joint_transformer_img = transforms.Compose([joint_transforms.JointResize(224)]) mask_size_list = [14, 28, 56, 112, 224] train_dset = saliency.Saliency( args.DATASET_PATH, 'train',train_joint_transformer_img, mask_size_list, transform=transforms.Compose([transforms.ToTensor(), normalize, ])) train_loader = torch.utils.data.DataLoader( train_dset, batch_size=args.batch_size, shuffle=True) test_joint_transforms_img = transforms.Compose([joint_transforms.JointResize(224)]) val_dset = saliency.Saliency( args.DATASET_PATH, 'val',test_joint_transforms_img, mask_size_list, transform=transforms.Compose([transforms.ToTensor(),normalize])) val_loader = torch.utils.data.DataLoader( val_dset, batch_size=args.batch_size, shuffle=False) print("TrainImages: %d" % len(train_loader.dataset.imgs)) print("ValImages: %d" % len(val_loader.dataset.imgs)) # example_inputs, example_targets = next(iter(train_loader)) # print("InputsBatchSize: ", example_inputs.size()) # print("TargetsBatchSize: ", len(example_targets)) # print("\nInput (size, max, min) ---") # # input # i = example_inputs[0] # print(i.size()) # print(i.max()) # print(i.min()) # print("Target (size, max, min) ---") # # target # for mask in example_targets: # print(mask.size()) # print(mask.max()) # print(mask.min()) # initialize ResNet18 from the pre-trained classification model resnet = torchvision.models.resnet50(pretrained=True) pre_trained_dict = resnet.state_dict() model = SegNet.resnet50() model_dict = model.state_dict() # 1. filter out unnecessary keys pre_trained_dict = {k: v for k, v in pre_trained_dict.items() if k in model_dict} # 2. overwrite entries in the existing state dict model_dict.update(pre_trained_dict) # 3. load the new state dict model.load_state_dict(model_dict) model = model.cuda() #model = torch.nn.DataParallel(model).cuda() print(' + Number of params: {}'.format( sum([p.data.nelement() for p in model.parameters()]))) # model.apply(utils.weights_init) optimizer = optim.RMSprop(model.parameters(), lr=args.LEARNING_RATE, weight_decay=args.WEIGHT_DECAY, eps=1e-12) criterion = nn.NLLLoss2d().cuda() exp_dir = args.EXPERIMENT + 'test' if os.path.exists(exp_dir): shutil.rmtree(exp_dir) exp = experiment.Experiment('test', args.EXPERIMENT) exp.init() START_EPOCH = exp.epoch END_EPOCH = START_EPOCH + args.N_EPOCHS for epoch in range(START_EPOCH, END_EPOCH): since = time.time() # ### Train ### trn_loss, trn_err = utils.train(model, train_loader, optimizer, criterion, epoch) print('Epoch {:d}: Train - Loss: {:.4f}\tErr: {:.4f}'.format(epoch, trn_loss, trn_err)) time_elapsed = time.time() - since print('Train Time {:.0f}m {:.0f}s'.format( time_elapsed // 60, time_elapsed % 60)) ### Test ### val_loss, val_err = utils.test(model, val_loader, criterion, epoch) print('Val - Loss: {:.4f}, Error: {:.4f}'.format(val_loss, val_err)) time_elapsed = time.time() - since print('Total Time {:.0f}m {:.0f}s\n'.format( time_elapsed // 60, time_elapsed % 60)) ### Save Metrics ### exp.save_history('train', trn_loss, trn_err) exp.save_history('val', val_loss, val_err) ### Checkpoint ### exp.save_weights(model, trn_loss, val_loss, trn_err, val_err) exp.save_optimizer(optimizer, val_loss) ## Early Stopping ## if (epoch - exp.best_val_loss_epoch) > args.MAX_PATIENCE: print(("Early stopping at epoch %d since no " +"better loss found since epoch %.3").format(epoch, exp.best_val_loss)) break # Adjust Lr ###--old method utils.adjust_learning_rate(args.LEARNING_RATE, args.LR_DECAY, optimizer, epoch, args.DECAY_LR_EVERY_N_EPOCHS) exp.epoch += 1
def main(feature_path=None, batch_size=2048, mini_batch_size=256, lr=1, optim="SGD", momentum=0.9, nesterov=False, noise_multiplier=1, max_grad_norm=0.1, max_epsilon=None, epochs=100, logdir=None): logger = Logger(logdir) device = get_device() # get pre-computed features x_train = np.load(f"{feature_path}_train.npy") x_test = np.load(f"{feature_path}_test.npy") train_data, test_data = get_data("cifar10", augment=False) y_train = np.asarray(train_data.targets) y_test = np.asarray(test_data.targets) trainset = torch.utils.data.TensorDataset(torch.from_numpy(x_train), torch.from_numpy(y_train)) testset = torch.utils.data.TensorDataset(torch.from_numpy(x_test), torch.from_numpy(y_test)) bs = batch_size assert bs % mini_batch_size == 0 n_acc_steps = bs // mini_batch_size train_loader = torch.utils.data.DataLoader(trainset, batch_size=mini_batch_size, shuffle=True, num_workers=1, pin_memory=True, drop_last=True) test_loader = torch.utils.data.DataLoader(testset, batch_size=mini_batch_size, shuffle=False, num_workers=1, pin_memory=True) n_features = x_train.shape[-1] try: mean = np.load(f"{feature_path}_mean.npy") var = np.load(f"{feature_path}_var.npy") except FileNotFoundError: mean = np.zeros(n_features, dtype=np.float32) var = np.ones(n_features, dtype=np.float32) bn_stats = (torch.from_numpy(mean).to(device), torch.from_numpy(var).to(device)) model = nn.Sequential(StandardizeLayer(bn_stats), nn.Linear(n_features, 10)).to(device) if optim == "SGD": optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=momentum, nesterov=nesterov) else: optimizer = torch.optim.Adam(model.parameters(), lr=lr) privacy_engine = PrivacyEngine( model, sample_rate=bs / len(train_data), alphas=ORDERS, noise_multiplier=noise_multiplier, max_grad_norm=max_grad_norm, ) privacy_engine.attach(optimizer) for epoch in range(0, epochs): print(f"\nEpoch: {epoch}") train_loss, train_acc = train(model, train_loader, optimizer, n_acc_steps=n_acc_steps) test_loss, test_acc = test(model, test_loader) if noise_multiplier > 0: rdp_sgd = get_renyi_divergence( privacy_engine.sample_rate, privacy_engine.noise_multiplier) * privacy_engine.steps epsilon, _ = get_privacy_spent(rdp_sgd) print(f"ε = {epsilon:.3f}") if max_epsilon is not None and epsilon >= max_epsilon: return else: epsilon = None logger.log_epoch(epoch, train_loss, train_acc, test_loss, test_acc, epsilon)
def main(args): # Setting warnings.simplefilter("ignore", UserWarning) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # Args Parser hj_method = args.hj_method kr_method = args.kr_method batch_size = args.batch_size beam_size = args.beam_size hidden_size = args.hidden_size embed_size = args.embed_size vocab_size = args.vocab_size max_len = args.max_len padding_index = args.pad_id n_layers = args.n_layers stop_ix = args.stop_ix # Load saved model & Word2vec save_path = 'save_{}_{}_{}_maxlen_{}'.format(vocab_size, hj_method, kr_method, max_len) save_list = sorted(glob.glob(f'./save/{save_path}/*.*')) save_pt = save_list[-1] print('Will load {} pt file...'.format(save_pt)) word2vec_hj = Word2Vec.load('./w2v/word2vec_hj_{}_{}.model'.format( vocab_size, hj_method)) # SentencePiece model load spm_kr = spm.SentencePieceProcessor() spm_kr.Load("./spm/m_korean_{}.model".format(vocab_size)) # Test data load with open('./test_dat.pkl', 'rb') as f: test_dat = pickle.load(f) test_dataset = CustomDataset(test_dat['test_hanja'], test_dat['test_korean']) test_loader = getDataLoader(test_dataset, pad_index=padding_index, shuffle=False, batch_size=batch_size) # Model load print('Model loading...') encoder = Encoder(vocab_size, embed_size, hidden_size, word2vec_hj, n_layers=n_layers, padding_index=padding_index) decoder = Decoder(embed_size, hidden_size, vocab_size, n_layers=n_layers, padding_index=padding_index) seq2seq = Seq2Seq(encoder, decoder, beam_size).cuda() #optimizer = optim.Adam(seq2seq.parameters(), lr=lr, weight_decay=w_decay) #scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=scheduler_step_size, gamma=lr_decay) print(seq2seq) print('Testing...') start_time = time.time() results = test(seq2seq, test_loader, vocab_size, load_pt=save_pt, stop_ix=stop_ix) print(time.time() - start_time) print('Done!') print("Decoding...") pred_list = list() for result_text in tqdm(results): text = torch.Tensor(result_text).squeeze().tolist() text = [int(x) for x in text] prediction_sentence = spm_kr.decode_ids( text).strip() # Decode with strip pred_list.append(prediction_sentence) ref_list = list() for ref_text in tqdm(test_dat['test_korean'][:stop_ix]): ref_list.append(spm_kr.decode_ids(ref_text).strip()) print('Done!') with open(f'./save/{save_path}/test_result.pkl', 'wb') as f: pickle.dump({ 'pred': pred_list, 'reference': ref_list, }, f) print('Save file; /test_dat.pkl') # Calculate BLEU Score print('Calculate BLEU4, METEOR, Rogue-L...') chencherry = SmoothingFunction() bleu4 = corpus_bleu(test_dat['reference'], test_dat['pred'], smoothing_function=chencherry.method4) print('BLEU Score is {}'.format(bleu4)) # Calculate METEOR Score meteor = meteor_score(test_dat['reference'], test_dat['pred']) print('METEOR Score is {}'.format(meteor)) # Calculate Rouge-L Score r = Rouge() total_test_length = len(test_dat['reference']) precision_all = 0 recall_all = 0 f_score_all = 0 for i in range(total_test_length): [precision, recall, f_score] = r.rouge_l([test_dat['reference'][i]], [test_dat['pred'][i]]) precision_all += precision recall_all += recall f_score_all += f_score print('Precision : {}'.foramt(round(precision_all / total_test_length, 4))) print('Recall : {}'.foramt(round(recall_all / total_test_length, 4))) print('F Score : {}'.foramt(round(f_score_all / total_test_length, 4)))
def main(dataset, augment=False, batch_size=2048, mini_batch_size=256, sample_batches=False, lr=1, optim="SGD", momentum=0.9, nesterov=False, noise_multiplier=1, max_grad_norm=0.1, epochs=100, input_norm=None, num_groups=None, bn_noise_multiplier=None, max_epsilon=None, logdir=None): logger = Logger(logdir) device = get_device() train_data, test_data = get_data(dataset, augment=augment) scattering, K, (h, w) = get_scatter_transform(dataset) scattering.to(device) bs = batch_size assert bs % mini_batch_size == 0 n_acc_steps = bs // mini_batch_size # Batch accumulation and data augmentation with Poisson sampling isn't implemented if sample_batches: assert n_acc_steps == 1 assert not augment train_loader = torch.utils.data.DataLoader(train_data, batch_size=mini_batch_size, shuffle=True, num_workers=1, pin_memory=True) test_loader = torch.utils.data.DataLoader(test_data, batch_size=mini_batch_size, shuffle=False, num_workers=1, pin_memory=True) rdp_norm = 0 if input_norm == "BN": # compute noisy data statistics or load from disk if pre-computed save_dir = f"bn_stats/{dataset}" os.makedirs(save_dir, exist_ok=True) bn_stats, rdp_norm = scatter_normalization( train_loader, scattering, K, device, len(train_data), len(train_data), noise_multiplier=bn_noise_multiplier, orders=ORDERS, save_dir=save_dir) model = ScatterLinear(K, (h, w), input_norm="BN", bn_stats=bn_stats) else: model = ScatterLinear(K, (h, w), input_norm=input_norm, num_groups=num_groups) model.to(device) trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad) print(f'model: {model}\n') print(f'has {trainable_params / 1e6:.4f} million trainable parameters') if augment: model = nn.Sequential(scattering, model) train_loader = torch.utils.data.DataLoader(train_data, batch_size=mini_batch_size, shuffle=True, num_workers=1, pin_memory=True, drop_last=True) preprocessor = None else: preprocessor = lambda x, y: (scattering(x), y) # baseline Logistic Regression without privacy if optim == "LR": assert not augment X_train = [] y_train = [] X_test = [] y_test = [] for data, target in train_loader: with torch.no_grad(): data = data.to(device) X_train.append(data.cpu().numpy().reshape(len(data), -1)) y_train.extend(target.cpu().numpy()) for data, target in test_loader: with torch.no_grad(): data = data.to(device) X_test.append(data.cpu().numpy().reshape(len(data), -1)) y_test.extend(target.cpu().numpy()) import numpy as np X_train = np.concatenate(X_train, axis=0) X_test = np.concatenate(X_test, axis=0) y_train = np.asarray(y_train) y_test = np.asarray(y_test) print(X_train.shape, y_train.shape, X_test.shape, y_test.shape) for idx, C in enumerate([0.01, 0.1, 1.0, 10, 100]): clf = LogisticRegression(C=C, fit_intercept=True) clf.fit(X_train, y_train) train_acc = 100 * clf.score(X_train, y_train) test_acc = 100 * clf.score(X_test, y_test) print(f"C={C}, " f"Acc train = {train_acc: .2f}, " f"Acc test = {test_acc: .2f}") logger.log_epoch(idx, 0, train_acc, 0, test_acc, None) return print(f"model has {get_num_params(model)} parameters") if optim == "SGD": optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=momentum, nesterov=nesterov) else: optimizer = torch.optim.Adam(model.parameters(), lr=lr) privacy_engine = PrivacyEngine( model, batch_size=bs, sample_size=len(train_data), alphas=ORDERS, noise_multiplier=noise_multiplier, max_grad_norm=max_grad_norm, ) privacy_engine.attach(optimizer) for epoch in range(0, epochs): print(f"\nEpoch: {epoch}") train_loss, train_acc = train(model, train_loader, optimizer, n_acc_steps=n_acc_steps, preprocessor=preprocessor) test_loss, test_acc = test(model, test_loader, preprocessor=preprocessor) if noise_multiplier > 0: rdp_sgd = get_renyi_divergence( privacy_engine.sample_rate, privacy_engine.noise_multiplier) * privacy_engine.steps epsilon, _ = get_privacy_spent(rdp_norm + rdp_sgd) epsilon2, _ = get_privacy_spent(rdp_sgd) print(f"ε = {epsilon:.3f} (sgd only: ε = {epsilon2:.3f})") if max_epsilon is not None and epsilon >= max_epsilon: return else: epsilon = None logger.log_epoch(epoch, train_loss, train_acc, test_loss, test_acc, epsilon) logger.log_scalar("epsilon/train", epsilon, epoch)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # # set dataloader and model if not opt.train == 3 or not opt.train == 4: dataloader, dataloader_test = data_loader() netG, netD, desc = define_model(device) # config loss if opt.coeff_color_loss: print('use color loss') gradient_critcriterion = MaskedGradient(opt).to(device) # set noise NZ = opt.imageSize // 2 ** nDep noise = torch.FloatTensor(opt.batchSize, opt.zLoc, NZ, NZ) fixnoise = torch.FloatTensor(opt.batchSize, opt.zLoc, NZ * 4, NZ * 4) noise = noise.to(device) fixnoise = fixnoise.to(device) Noise = [NZ, noise, fixnoise] # setup optimizer optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) # netD.parameters() optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) if opt.train == 0: test(opt, dataloader_test, device, netD, netG, Noise, desc) elif opt.train == 1: train(opt, dataloader, dataloader_test, device, netD, netG, desc, Noise, optimizerD, optimizerG) elif opt.train == 2: save_embedding(opt, dataloader_test, device, netD) elif opt.train == 3: interpolation(opt, device, netG, Noise) elif opt.train == 4: all_composition(opt, device, netG, Noise)
print('STARTING TRAINING') for epoch in range(1, args.epochs + 1): train(args, model, device, train_loader, optimizer, epoch, start_time=time.time()) f1 = get_mean_F1(model, validation_loader) print('after epoch {} got f1 score of {}'.format(epoch, f1)) if f1 > info['highest F1']: info['highest F1'] = np.copy(f1) info['saved epoch'] = epoch test(args, model, device, test_loader, epoch, trainDataset, testDataset, path_submission) torch.save(model, path_model) print('currently best model --> saved') print('TRAINING DONE') print(info) # ### Load Model # In[ ]: #if load_model: # model = torch.load(path_model) # In[ ]:
# start_epoch = chechpoint['epoch'] # model.load_state_dict(chechpoint['state_dict']) # optimizer.load_state_dict(chechpoint['optimizer']) # logger = Logger(os.path.join(checkpoint_dir, 'log.txt'), title=title, resume=True) # else: # logger = Logger(os.path.join(checkpoint_path, 'log.txt'), title=title) # logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) # train ------------ for epoch in range(start_epoch, tot_epoch): print('\nEpoch [%d | %d] LR: %f' % (epoch + 1, tot_epoch, optimizer.param_groups[0]['lr'])) train_loss, train_acc, train_5 = train(train_dataloader, model, criterion, optimizer, epoch, device, writer) test_loss, test_acc, test_5 = test(val_dataloader, model, criterion, epoch, device, writer) # logger.append([ lr, train_loss, test_loss, train_acc, test_acc ]) print( 'train_loss:%f, val_loss:%f, train_acc:%f, train_5:%f, val_acc:%f, val_5:%f' % (train_loss, test_loss, train_acc, train_5, test_acc, test_5)) is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint( { 'fold': 1, 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'train_acc': train_acc,
def main(dataset, augment=False, use_scattering=False, size=None, batch_size=2048, mini_batch_size=256, sample_batches=False, lr=1, optim="SGD", momentum=0.9, nesterov=False, noise_multiplier=1, max_grad_norm=0.1, epochs=100, input_norm=None, num_groups=None, bn_noise_multiplier=None, max_epsilon=None, logdir=None, early_stop=True, seed=0): torch.manual_seed(seed) logger = Logger(logdir) device = get_device() train_data, test_data = get_data(dataset, augment=augment) if use_scattering: scattering, K, _ = get_scatter_transform(dataset) scattering.to(device) else: scattering = None K = 3 if len(train_data.data.shape) == 4 else 1 bs = batch_size assert bs % mini_batch_size == 0 n_acc_steps = bs // mini_batch_size # Batch accumulation and data augmentation with Poisson sampling isn't implemented if sample_batches: assert n_acc_steps == 1 assert not augment train_loader = torch.utils.data.DataLoader(train_data, batch_size=mini_batch_size, shuffle=True, num_workers=1, pin_memory=True) test_loader = torch.utils.data.DataLoader(test_data, batch_size=mini_batch_size, shuffle=False, num_workers=1, pin_memory=True) rdp_norm = 0 if input_norm == "BN": # compute noisy data statistics or load from disk if pre-computed save_dir = f"bn_stats/{dataset}" os.makedirs(save_dir, exist_ok=True) bn_stats, rdp_norm = scatter_normalization( train_loader, scattering, K, device, len(train_data), len(train_data), noise_multiplier=bn_noise_multiplier, orders=ORDERS, save_dir=save_dir) model = CNNS[dataset](K, input_norm="BN", bn_stats=bn_stats, size=size) else: model = CNNS[dataset](K, input_norm=input_norm, num_groups=num_groups, size=size) model.to(device) if use_scattering and augment: model = nn.Sequential(scattering, model) train_loader = torch.utils.data.DataLoader(train_data, batch_size=mini_batch_size, shuffle=True, num_workers=1, pin_memory=True, drop_last=True) else: # pre-compute the scattering transform if necessery train_loader = get_scattered_loader(train_loader, scattering, device, drop_last=True, sample_batches=sample_batches) test_loader = get_scattered_loader(test_loader, scattering, device) print(f"model has {get_num_params(model)} parameters") if optim == "SGD": optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=momentum, nesterov=nesterov) else: optimizer = torch.optim.Adam(model.parameters(), lr=lr) privacy_engine = PrivacyEngine( model, batch_size=bs, sample_size=len(train_data), alphas=ORDERS, noise_multiplier=noise_multiplier, max_grad_norm=max_grad_norm, ) privacy_engine.attach(optimizer) best_acc = 0 flat_count = 0 results = dict(train_zeon=[], train_xent=[], test_zeon=[], test_xent=[], epoch=[]) for epoch in range(0, epochs): print(f"\nEpoch: {epoch}") train_loss, train_acc = train(model, train_loader, optimizer, n_acc_steps=n_acc_steps) test_loss, test_acc = test(model, test_loader) results['train_zeon'].append(train_acc) results['train_xent'].append(train_loss) results['test_zeon'].append(test_acc) results['test_xent'].append(test_loss) results['epoch'].append(epoch) if noise_multiplier > 0: rdp_sgd = get_renyi_divergence( privacy_engine.sample_rate, privacy_engine.noise_multiplier) * privacy_engine.steps epsilon, _ = get_privacy_spent(rdp_norm + rdp_sgd) epsilon2, _ = get_privacy_spent(rdp_sgd) print(f"ε = {epsilon:.3f} (sgd only: ε = {epsilon2:.3f})") if max_epsilon is not None and epsilon >= max_epsilon: return else: epsilon = None logger.log_epoch(epoch, train_loss, train_acc, test_loss, test_acc, epsilon) logger.log_scalar("epsilon/train", epsilon, epoch) # stop if we're not making progress if test_acc > best_acc: best_acc = test_acc flat_count = 0 else: flat_count += 1 if flat_count >= 20 and early_stop: print("plateau...") break # Write to file. record = { **results, **{ 'best_acc': best_acc, 'seed': seed, 'dataset': dataset } } record_path = os.path.join('.', 'record', f'{dataset}-{seed}.json') os.makedirs(os.path.dirname(record_path), exist_ok=True) with open(record_path, 'w') as f: json.dump(record, f, indent=4) import logging logging.warning(f'Wrote to file: {record_path}')
def main(): parser = argparse.ArgumentParser() # parser.add_argument('--DATASET_PATH', type=str, default='/data/davis/dataset/DUTS/') # parser.add_argument('--EXPERIMENT', type=str, default='/data/davis/result/TrainNet/') parser.add_argument('--DATASET_PATH', type=str, default='/home/yangle/DAVIS/dataset/DUTS/') parser.add_argument('--EXPERIMENT', type=str, default='/home/yangle/DAVIS/result/TrainNet/') parser.add_argument('--N_EPOCHS', type=int, default=200) parser.add_argument('--MAX_PATIENCE', type=int, default=30) parser.add_argument('--batch_size', type=int, default=32) parser.add_argument('--seed', type=int, default=0) parser.add_argument('--N_CLASSES', type=int, default=2) parser.add_argument('--LEARNING_RATE', type=float, default=1e-4) parser.add_argument('--LR_DECAY', type=float, default=0.995) parser.add_argument('--DECAY_LR_EVERY_N_EPOCHS', type=int, default=1) parser.add_argument('--WEIGHT_DECAY', type=float, default=0.0001) parser.add_argument('--CUDNN', type=bool, default=True) args = parser.parse_args() torch.cuda.manual_seed(args.seed) cudnn.benchmark = args.CUDNN normalize = transforms.Normalize(mean=dataset.mean, std=dataset.std) train_joint_transformer_img = transforms.Compose( [joint_transforms.JointResize(224)]) mask_size_list = [28, 28, 28, 56, 112] train_dset = dataset.Saliency(args.DATASET_PATH, 'train', train_joint_transformer_img, mask_size_list, transform=transforms.Compose([ transforms.ToTensor(), normalize, ])) train_loader = torch.utils.data.DataLoader(train_dset, batch_size=args.batch_size, shuffle=True) test_joint_transforms_img = transforms.Compose( [joint_transforms.JointResize(224)]) val_dset = dataset.Saliency(args.DATASET_PATH, 'val', test_joint_transforms_img, mask_size_list, transform=transforms.Compose( [transforms.ToTensor(), normalize])) val_loader = torch.utils.data.DataLoader(val_dset, batch_size=args.batch_size, shuffle=False) print("TrainImages: %d" % len(train_loader.dataset.imgs)) print("ValImages: %d" % len(val_loader.dataset.imgs)) example_inputs, example_targets = next(iter(train_loader)) print("InputsBatchSize: ", example_inputs.size()) print("TargetsBatchSize: ", len(example_targets)) print("\nInput (size, max, min) ---") # input i = example_inputs[0] print(i.size()) print(i.max()) print(i.min()) print("Target (size, max, min) ---") # target for mask in example_targets: print(mask.size()) print(mask.max()) print(mask.min()) resnet50 = torchvision.models.resnet50(pretrained=True) dict_resnet50 = resnet50.state_dict() model = SegNet.resnet50() # # initialize model.apply(utils.weights_init) SegNet_dict = model.state_dict() pretrained_dict = { k: v for k, v in dict_resnet50.items() if k in SegNet_dict } # for k in pretrained_dict: # print(k) SegNet_dict.update(pretrained_dict) model.load_state_dict(SegNet_dict) # seperate layers, to set different lr param_exist = [] param_add = [] for k, (name, module) in enumerate(model.named_children()): # existing layers including: conv1 bn1 relu maxpool # layer1 layer2 layer3 layer4 if k < 8: for param in module.parameters(): param_exist.append(param) # adding layers including: bottleneck skip3 skip2 skip1 skip0 # conv_end_1 bn_end_1 salmap Sigmoid mask0 mask4 mask3 mask2 mask1 else: for param in module.parameters(): param_add.append(param) model = torch.nn.DataParallel(model).cuda() optimizer = optim.SGD([{ 'params': param_exist, 'lr': 0.001 }, { 'params': param_add }], lr=0.01, momentum=0.9, weight_decay=0.0005) print(' + Number of params: {}'.format( sum([p.data.nelement() for p in model.parameters()]))) # optimizer = optim.RMSprop(model.parameters(), lr=args.LEARNING_RATE, # weight_decay=args.WEIGHT_DECAY, eps=1e-12) criterion = nn.BCELoss().cuda() exp_dir = args.EXPERIMENT + 'ResNet50' if os.path.exists(exp_dir): shutil.rmtree(exp_dir) exp = experiment.Experiment('ResNet50', args.EXPERIMENT) exp.init() START_EPOCH = exp.epoch END_EPOCH = START_EPOCH + args.N_EPOCHS for epoch in range(START_EPOCH, END_EPOCH): since = time.time() # ### Train ### trn_loss, trn_err = utils.train(model, train_loader, optimizer, criterion, epoch) print('Epoch {:d}: Train - Loss: {:.4f}\tErr: {:.4f}'.format( epoch, trn_loss, trn_err)) time_elapsed = time.time() - since print('Train Time {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60)) ### Test ### val_loss, val_err = utils.test(model, val_loader, criterion, epoch) print('Val - Loss: {:.4f}, Error: {:.4f}'.format(val_loss, val_err)) time_elapsed = time.time() - since print('Total Time {:.0f}m {:.0f}s\n'.format(time_elapsed // 60, time_elapsed % 60)) ### Save Metrics ### exp.save_history('train', trn_loss, trn_err) exp.save_history('val', val_loss, val_err) ### Checkpoint ### exp.save_weights(model, trn_loss, val_loss, trn_err, val_err) exp.save_optimizer(optimizer, val_loss) ## Early Stopping ## if (epoch - exp.best_val_loss_epoch) > args.MAX_PATIENCE: print(("Early stopping at epoch %d since no " + "better loss found since epoch %.3").format( epoch, exp.best_val_loss)) break # Adjust Lr ###--old method # utils.adjust_learning_rate(args.LEARNING_RATE, args.LR_DECAY, optimizer, # epoch, args.DECAY_LR_EVERY_N_EPOCHS) epoch_finish = epoch - 1 if epoch_finish > 0 and epoch % 2 == 0: utils.adjust_learning_rate_SGD(optimizer, decay_rate=0.5) exp.epoch += 1