def get_inception_score(args, images): splits = args.num_splits inps = [] input_transform = transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) for img in images: img = img.astype(np.float32) inps.append(np.expand_dims(img, 0)) preds = [] n_batches = int(math.ceil(float(len(inps)) / float(args.batch_size))) n_preds = 0 net = ResNet18().cuda() net.load_state_dict(torch.load(args.model_dir)) print("load model successfully") for i in range(n_batches): sys.stdout.write(".") sys.stdout.flush() inp = inps[(i * args.batch_size):min((i + 1) * args.batch_size, len(inps))] inp = np.concatenate(inp, 0) inp = torch.from_numpy(np.rollaxis(inp, 3, 1)).cuda() outputs = net(inp) pred = outputs.data.tolist() #pred = softmax(pred) preds.append(pred) n_preds += outputs.shape[0] preds = np.concatenate(preds, 0) preds = np.exp(preds) / np.sum(np.exp(preds), 1, keepdims=True) mean_, std_ = preds2score(preds, splits) return mean_, std_
def train_CIFAR(CIFAR10=True, n_epochs=100, noise_rate=0.0, model_path='./model/CIFAR.mdl'): output_features = 10 if CIFAR10 else 100 dataset_name = 'CIFAR10' if CIFAR10 else 'CIFAR100' layers_in_each_block_list = [2, 2, 2, 2] model = ResNet18(layers_in_each_block_list, output_features).to(device) """Prepare data""" print('==> Preparing data..') if CIFAR10: train_loader, test_loader = datasets.load_cifar10_dataset() else: train_loader, test_loader = datasets.load_cifar100_dataset() """training for 10 epochs""" print(f'==> Start training {dataset_name} ' f'with noise level {noise_rate} ' f'for {n_epochs} epochs') criterion = torch.nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=0.1, momentum=0.9, weight_decay=5e-4) scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=200, eta_min=0.001) (train_loss_per_epoch, test_loss_per_epoch, correct_per_epoch, memorized_per_epoch, incorrect_per_epoch) = train( model, criterion, optimizer, n_epochs=n_epochs, train_loader=train_loader, test_loader=test_loader, scheduler=scheduler, noise_rate=noise_rate) """Plot learning curve and accuracy""" print(f'acc={correct_per_epoch[-1]}, memorized={memorized_per_epoch[-1]}') plot_title = f'{dataset_name}, noise_level={noise_rate}' plot_learning_curve_and_acc(train_loss_per_epoch, test_loss_per_epoch, correct_per_epoch, memorized_per_epoch, incorrect_per_epoch, plot_title) Path("./models").mkdir(parents=True, exist_ok=True) torch.save(model.state_dict(), model_path)
def run(size, rank, epoch, batchsize): #print('run') if MODEL == 'CNN' and DATA_SET == 'KWS': model = CNNKws() if MODEL == 'CNN' and DATA_SET == 'Cifar10': model = CNNCifar() if MODEL == 'ResNet18' and DATA_SET == 'Cifar10': model = ResNet18() model = model.cuda() optimizer = torch.optim.SGD(model.parameters(), lr=LR, weight_decay=1e-3) loss_func = torch.nn.CrossEntropyLoss() train_loader = get_local_data(size, rank, batchsize) if rank == 0: test_loader = get_testset(rank) #fo = open("file_multi"+str(rank)+".txt", 'w') group_list = [i for i in range(size)] group = dist.new_group(group_list) model, round = load_model(model, group, rank) while round < MAX_ROUND: sys.stdout.flush() if rank == 0: accuracy = 0 positive_test_number = 0 total_test_number = 0 for step, (test_x, test_y) in enumerate(test_loader): test_x = test_x.cuda() test_y = test_y.cuda() test_output = model(test_x) pred_y = torch.max(test_output, 1)[1].data.cpu().numpy() positive_test_number += ( pred_y == test_y.data.cpu().numpy()).astype(int).sum() # print(positive_test_number) total_test_number += float(test_y.size(0)) accuracy = positive_test_number / total_test_number print('Round: ', round, ' Rank: ', rank, '| test accuracy: %.4f' % accuracy) #fo.write(str(round) + " " + str(rank) + " " + str(accuracy) + "\n") for epoch_cnt in range(epoch): for step, (b_x, b_y) in enumerate(train_loader): b_x = b_x.cuda() b_y = b_y.cuda() optimizer.zero_grad() output = model(b_x) loss = loss_func(output, b_y) loss.backward() optimizer.step() model = all_reduce(model, size, group) #if (round+1) % ROUND_NUMBER_FOR_REDUCE == 0: #model = all_reduce(model, size, group) if (round + 1) % ROUND_NUMBER_FOR_SAVE == 0: save_model(model, round + 1, rank) round += 1
def __init__(self, celeba_loader, config): """Initialize configurations.""" # Data loader. self.celeba_loader = celeba_loader # Model configurations. self.c_dim = config.c_dim self.image_size = config.image_size self.g_conv_dim = config.g_conv_dim self.d_conv_dim = config.d_conv_dim self.g_repeat_num = config.g_repeat_num self.d_repeat_num = config.d_repeat_num self.lambda_cls = config.lambda_cls self.lambda_rec = config.lambda_rec # Training configurations. self.dataset = config.dataset self.batch_size = config.batch_size self.num_iters = config.num_iters self.num_iters_decay = config.num_iters_decay self.g_lr = config.g_lr self.d_lr = config.d_lr self.beta1 = config.beta1 self.beta2 = config.beta2 self.resume_iters = config.resume_iters self.selected_attrs = config.selected_attrs # Test configurations. self.test_iters = config.test_iters # Miscellaneous. self.use_tensorboard = config.use_tensorboard self.device = torch.device( 'cuda' if torch.cuda.is_available() else 'cpu') # Directories. self.log_dir = config.log_dir self.sample_dir = config.sample_dir self.model_save_dir = config.model_save_dir self.result_dir = config.result_dir # Step size. self.log_step = config.log_step self.fid_step = config.fid_step self.sample_step = config.sample_step self.model_save_step = config.model_save_step self.lr_update_step = config.lr_update_step # Classifier for computing FID self.classifier = ResNet18().to(self.device) self.classifier.eval() # Build the model and tensorboard. self.build_model() if self.use_tensorboard: self.build_tensorboard()
def __init__(self, X, Y, idxs_lb, net, handler, args, experiment): super(RandomNetworkDistillation, self).__init__(X, Y, idxs_lb, net, handler, args, experiment) self.predictor_net = ResNet18(in_channels=3).to(self.device) self.predictor_state = self.predictor_net.state_dict()
def get_model(): if (USE_CUDA): if args.use_half: model = ResNet18(use_half=args.use_half).cuda().half() else: model = ResNet18(use_half=args.use_half).cuda() else: if args.use_half: model = ResNet18(use_half=args.use_half).half() else: model = ResNet18(use_half=args.use_half) if args.resume: checkpoint_file = args.resume if os.path.isfile(checkpoint_file): print("loading checkpoint {}".format(args.resume)) checkpoint = torch.load(checkpoint_file) model.load_state_dict(checkpoint['state_dict']) print("loaded checkpoint {} )".format(checkpoint_file )) # -- # Initialize optimizer print('cifar10.py: initializing optimizer...', file=sys.stderr) lr_scheduler = getattr(LRSchedule, args.lr_schedule)(lr_init=args.lr_init, epochs=args.epochs, extra=args.extra) model.init_optimizer( opt=torch.optim.SGD, params=model.parameters(), lr_scheduler=lr_scheduler, momentum=args.momentum, weight_decay=args.weight_decay, nesterov=True ) return model
def run(fold_index): train = pd.read_csv("/fred/oz138/test/data/train_folds.csv") test = pd.read_csv("/fred/oz138/test/data/test.csv") submission = pd.read_csv("/fred/oz138/test/data/sample_submission.csv") # label encoding train["ebird_label"] = LabelEncoder().fit_transform(train['ebird_code']) train_df = train[~train.kfold.isin([fold_index])] train_dataset = BirdDataset(df=train_df) valid_df = train[train.kfold.isin([fold_index])] valid_dataset = BirdDataset(df=valid_df, valid=True) device = "cuda" MX = ResNet18(pretrained=True) model = MX.to(device) train_loader = DataLoader(dataset=train_dataset, batch_size=args.batch_size, pin_memory=True, drop_last=False) valid_loader = DataLoader(dataset=valid_dataset, batch_size=args.batch_size, pin_memory=True, drop_last=False) optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr, betas=args.betas, eps=args.eps, weight_decay=args.wd) best_acc = 0 for epoch in range(args.epochs): train_loss = Engine.train_fn(train_loader, model, optimizer, device, epoch) valid_loss, valid_acc = Engine.valid_fn(valid_loader, model, device, epoch) print( f"Fold {fold_index} ** Epoch {epoch + 1} **==>** Accuracy = {valid_acc}" ) if valid_acc > best_acc: torch.save(model.state_dict(), os.path.join(args.MODEL_PATH, f"fold_{fold_index}.bin")) best_acc = valid_acc
def load_model(group, rank): if MODEL == 'CNN' and DATA_SET == 'Mnist': model = CNNMnist() if MODEL == 'CNN' and DATA_SET == 'Cifar10': model = CNNCifar() if MODEL == 'ResNet18' and DATA_SET == 'Cifar10': model = ResNet18() if SAVE and os.path.exists('autoencoder'+str(rank)+'.t7'): logging('===> Try resume from checkpoint') checkpoint = torch.load('autoencoder'+str(rank)+'.t7') model.load_state_dict(checkpoint['state']) round = checkpoint['round'] print('===> Load last checkpoint data') else: round = 0 init_param(model, 0, group) return model, round
def run_train(seed): device = torch.device("cuda") print(device) model = ResNet18(2).to(device) print('Num parameters: %d' % model.count_parameters()) #### read hyps here #### cs = config_spaces.get_hyperparameter_search_space(seed) hyps = cs.sample_configuration(1).get_dictionary() lr = hyps['learning_rate_init'] mom = hyps['momentum'] batch_size = hyps['batch_size'] epochs = hyps['epochs'] weight_decay = hyps['weight_decay'] lr_decay = 1.0 / hyps['lr_decay'] patience = hyps['patience'] tolerance = hyps['tolerance'] resize_crop = hyps['resize_crop'] h_flip = hyps['h_flip'] v_flip = hyps['v_flip'] shuffle = hyps['shuffle'] train_loader, test_loader = load_data(shuffle, batch_size, resize_crop, h_flip, v_flip) optimizer = optim.SGD(model.parameters(), lr=lr, momentum=mom, weight_decay=weight_decay) scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=lr_decay, patience=patience, threshold=tolerance) acc_list = [] loss_list = [] time_list = [] start = time() for epoch in range(epochs): train(model, device, train_loader, optimizer, epoch) test_acc, test_loss = test(model, device, test_loader) scheduler.step(test_acc / 100) acc_list.append(test_acc) loss_list.append(test_loss) time_list.append(time() - start) return acc_list, loss_list, time_list, hyps
def load_model(group, rank): if MODEL == 'CNN' and DATASET == 'Mnist': model = CNNMnist() if MODEL == 'CNN' and DATASET == 'Cifar10': model = CNNCifar() if MODEL == 'ResNet18' and DATASET == 'Cifar10': model = ResNet18() if CUDA: model.cuda() if False and SAVE and os.path.exists('autoencoder' + str(rank) + '.t7'): logging('===> Try resume from checkpoint') checkpoint = torch.load('autoencoder' + str(rank) + '.t7') model.load_state_dict(checkpoint['state']) logging('model loaded') else: init_param(model, 0, group) logging('model created') return model
def get_model(): if args.model == 'ResNet18': return ResNet18(p_dropout=args.dropout) elif args.model == 'ResNet34': return ResNet34(p_dropout=args.dropout) elif args.model == 'ResNet50': return ResNet50(p_dropout=args.dropout) elif args.model == 'ResNet101': return ResNet101(p_dropout=args.dropout) elif args.model == 'ResNet152': return ResNet152(p_dropout=args.dropout) elif args.model == 'VGG11': return VGG('VGG11', p_dropout=args.dropout) elif args.model == 'VGG13': return VGG('VGG13', p_dropout=args.dropout) elif args.model == 'VGG16': return VGG('VGG16', p_dropout=args.dropout) elif args.model == 'VGG19': return VGG('VGG19', p_dropout=args.dropout) else: raise 'Model Not found'
transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) # Build train and test dataset train_data = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train) test_data = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test) #Build dataloader of train and test train_loader = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, num_workers=2) test_loader = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, num_workers=2) # Build a Resnet18 model if args.model == 'resnet18': model = ResNet18() elif args.model == 'resnet34': model = ResNet34() else: raise Exception('Put \'resent18\' or \'resnet34\' for model argument') model = model.to(device) print(model) criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), lr=args.lr) def train(epoch): print('=============================================') print('Epoch: %d' % epoch) print('=============================================') print('[Train Started]')
def load_model(model_file, device): net = ResNet18().to(device) model = torch.load(model_file) net.load_state_dict(model.state_dict()) return net
def main(): args = parser.parse_args() args.cuda = args.cuda == 'yes' args.disable_pbar = args.disable_pbar == 'yes' args.stable_sgd = args.stable_sgd == 'yes' print(f"args={vars(args)}") device = torch.device("cuda:0" if torch.cuda.is_available() and args.cuda else "cpu") print(f'Using device: {device}') # unique identifier uid = uuid.uuid4().hex if args.uid is None else args.uid now = str(datetime.datetime.now().date()) + "_" + ':'.join(str(datetime.datetime.now().time()).split(':')[:-1]) runname = 'T={}_id={}'.format(now, uid) if not args.resume else args.resume # Paths setupname = [args.strategy, args.exp_name, args.model, args.scenario] parentdir = os.path.join(args.save_path, '_'.join(setupname)) results_path = Path(os.path.join(parentdir, runname)) results_path.mkdir(parents=True, exist_ok=True) tb_log_dir = os.path.join(results_path, 'tb_run') # Group all runs # Eval results eval_metric = 'Top1_Acc_Stream/eval_phase/test_stream' eval_results_dir = results_path / eval_metric.split('/')[0] eval_results_dir.mkdir(parents=True, exist_ok=True) eval_result_files = [] # To avg over seeds seeds = [args.seed] if args.seed is not None else list(range(args.n_seeds)) for seed in seeds: # initialize seeds print("STARTING SEED {}/{}".format(seed, len(seeds) - 1)) set_seed(seed) # create scenario if args.scenario == 'smnist': inputsize = 28 * 28 scenario = SplitMNIST(n_experiences=5, return_task_id=False, seed=seed, fixed_class_order=[i for i in range(10)]) elif args.scenario == 'CIFAR10': scenario = SplitCIFAR10(n_experiences=5, return_task_id=False, seed=seed, fixed_class_order=[i for i in range(10)]) inputsize = (3, 32, 32) elif args.scenario == 'miniimgnet': scenario = SplitMiniImageNet(args.dset_rootpath, n_experiences=20, return_task_id=False, seed=seed, fixed_class_order=[i for i in range(100)]) inputsize = (3, 84, 84) else: raise ValueError("Wrong scenario name.") print(f"Scenario = {args.scenario}") if args.model == 'simple_mlp': model = MyMLP(input_size=inputsize, hidden_size=args.hs) elif args.model == 'resnet18': if not args.stable_sgd: assert args.drop_prob == 0 model = ResNet18(inputsize, scenario.n_classes, drop_prob=args.drop_prob) criterion = torch.nn.CrossEntropyLoss() optimizer = torch.optim.SGD(model.parameters(), lr=args.lr) # Paths eval_results_file = eval_results_dir / f'seed={seed}.csv' # LOGGING tb_logger = TensorboardLogger(tb_log_dir=tb_log_dir, tb_log_exp_name=f'seed={seed}.pt') # log to Tensorboard print_logger = TextLogger() if args.disable_pbar else InteractiveLogger() # print to stdout eval_logger = EvalTextLogger(metric_filter=eval_metric, file=open(eval_results_file, 'a')) eval_result_files.append(eval_results_file) # METRICS eval_plugin = EvaluationPlugin( accuracy_metrics(experience=True, stream=True), loss_metrics(minibatch=True, experience=True), ExperienceForgetting(), # Test only StreamConfusionMatrix(num_classes=scenario.n_classes, save_image=True), # LOG OTHER STATS # timing_metrics(epoch=True, experience=False), # cpu_usage_metrics(experience=True), # DiskUsageMonitor(), # MinibatchMaxRAM(), # GpuUsageMonitor(0), loggers=[print_logger, tb_logger, eval_logger]) plugins = None if args.strategy == 'replay': plugins = [RehRevPlugin(n_total_memories=args.mem_size, mode=args.replay_mode, # STEP-BACK aversion_steps=args.aversion_steps, aversion_lr=args.aversion_lr, stable_sgd=args.stable_sgd, # Stable SGD lr_decay=args.lr_decay, init_epochs=args.init_epochs # First task epochs )] # CREATE THE STRATEGY INSTANCE (NAIVE) strategy = Naive(model, optimizer, criterion, train_epochs=args.epochs, device=device, train_mb_size=args.bs, evaluator=eval_plugin, plugins=plugins ) # train on the selected scenario with the chosen strategy print('Starting experiment...') for experience in scenario.train_stream: if experience.current_experience == args.until_task: print("CUTTING OF TRAINING AT TASK ", experience.current_experience) break else: print("Start training on step ", experience.current_experience) strategy.train(experience) print("End training on step ", experience.current_experience) print('Computing accuracy on the test set') res = strategy.eval(scenario.test_stream[:args.until_task]) # Gathered by EvalLogger final_results_file = eval_results_dir / f'seed_summary.pt' stat_summarize(eval_result_files, final_results_file) print(f"[FILE:TB-RESULTS]: {tb_log_dir}") print(f"[FILE:FINAL-RESULTS]: {final_results_file}") print("FINISHED SCRIPT")
# reproducibility is da best np.random.seed(run) torch.manual_seed(run) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False # fetch data data = locate('data.get_%s' % args.dataset)(args) # make dataloaders train_loader, val_loader, test_loader = [ CLDataLoader(elem, args, train=t) for elem, t in zip(data, [True, False, False]) ] model = ResNet18(args.n_classes, nf=20, input_size=args.input_size).to(args.device) opt = torch.optim.SGD(model.parameters(), lr=0.1) gen = CVAE(20, args).cuda() # this is actually an autoencoder opt_gen = torch.optim.Adam(gen.parameters()) # build buffer if args.store_latents: buffer = Buffer(args, input_size=(20 * 4 * 4, )) else: buffer = Buffer(args) buffer.min_per_class = 0 print('multiple heads ', args.multiple_heads) if run == 0:
def trainer( train_set: torch.utils.data.Dataset, test_set: torch.utils.data.Dataset, size_dict: Dict[int, int], model: str = "ResNet50", device: torch.device = torch.device("cpu"), batch_size: int = 500, num_epochs: int = 10, learning_rate: float = 0.001, weight_decay: float = 0, dropout: float = 0, ) -> float: """ Get the best test accuracy after training for `num_epochs` epochs. """ # create data-loaders train_loader = DataLoader( dataset=train_set, batch_size=batch_size, drop_last=True, shuffle=True, pin_memory=True, ) test_loader = DataLoader(dataset=test_set, batch_size=batch_size, shuffle=False, pin_memory=True) # set model, loss function and optimizer num_classes = len(size_dict) if model == "ResNet50": model = ResNet50(num_classes, dropout).to(device) elif model == "ResNet18": model = ResNet18(num_classes, dropout).to(device) criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(params=model.parameters(), lr=learning_rate, weight_decay=weight_decay) # here class encoding is necessary since we need the dimension # of one-hot encoding identical to the number of classes class_encoding = { class_id: i for i, (class_id, _) in enumerate(size_dict.items()) } # start training best_acc = 0 for epoch in range(num_epochs): model.train() for images, labels in train_loader: labels = labels.apply_(lambda id: class_encoding[id]) images, labels = images.to(device), labels.to(device) # forward pass pred = model(images) loss = criterion(pred, labels) # backward pass optimizer.zero_grad() loss.backward() optimizer.step() # test after each epoch model.eval() accuracies = [] with torch.no_grad(): for images, labels in test_loader: labels = labels.apply_(lambda id: class_encoding[id]) images, labels = images.to(device), labels.to(device) # forward pass pred = model(images) batch_acc = accuracy(pred, labels) accuracies.append(batch_acc) epoch_acc = sum(accuracies) / len(accuracies) best_acc = max(best_acc, epoch_acc) return float(round(best_acc, 4))
]) transform_test = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ]) trainset = torchvision.datasets.CIFAR10( root=dataDir, train=True, download=True, transform=transform_train) testset = torchvision.datasets.CIFAR10( root=dataDir, train=False, download=True, transform=transform_test) testloader = torch.utils.data.DataLoader( testset, batch_size=batch_size, shuffle=True) net = ResNet18().cuda() optimizer = optim.SGD(net.parameters(), lr=0.1,momentum=0.9, weight_decay=5e-4) checkpoint = torch.load(os.path.join(checkpointDir, 'forget_ckpt.pth')) start_epoch = int(checkpoint['epoch'])+1 print ("Path : %s, Epoch : %d, Acc : %f" %(os.path.join(checkpointDir, 'forget_ckpt.pth'),start_epoch-1,checkpoint['acc'])) net.load_state_dict(checkpoint['net']) optimizer.load_state_dict(checkpoint['optimizer_state_dict']) criterion = nn.CrossEntropyLoss() def compute_correct_incorrect(): net.eval() correct_ones = torch.from_numpy(np.full(len(trainset), False)).cuda() with torch.no_grad(): for i in range(0,len(trainset),batch_size): batch_ind = torch.from_numpy(np.arange(i, min(i+batch_size,len(trainset)))) transformed_trainset=[]
# Begin Run Loop for run in range(args.n_runs): mir_tries, mir_success = 0, 0 # REPRODUCTIBILITY if args.reproc: np.random.seed(run) torch.manual_seed(run) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False # CLASSIFIER if args.use_conv: cls = ResNet18(args.n_classes, nf=20, input_size=args.input_size).to( args.device) # xuji added else: #cls = MLP(args) cls = classifier(args).to(args.device) opt = torch.optim.SGD(cls.parameters(), lr=args.lr) if run == 0: print("number of classifier parameters:", sum([np.prod(p.size()) for p in cls.parameters()])) # GENERATIVE MODELING if (args.method is not 'no_rehearsal') or (args.gen_method is not 'no_rehearsal'): gen = VAE.VAE(args).to(args.device) #if args.cuda: # gen = gen.cuda()
def main(): train_loader, test_loader = get_dataloader() if args.mode == "train": args.round = 0 model = ResNet18(num_classes=10) train_model(model, train_loader, test_loader) elif args.mode == "prune": previous_ckpt = "./checkpoints/resnet18-round%d.pth" % (args.round - 1) print("Pruning round %d, load model from %s" % (args.round, previous_ckpt)) model = torch.load(previous_ckpt) prune_model(model) print(model) params = sum([np.prod(p.size()) for p in model.parameters()]) print("Number of Parameters: %.1fM" % (params / 1e6)) train_model(model, train_loader, test_loader) elif args.mode == "test": ckpt = "./checkpoints/resnet18-round%d.pth" % (args.round) print("Load model from %s" % (ckpt)) model = torch.load(ckpt) params = sum([np.prod(p.size()) for p in model.parameters()]) print("Number of Parameters: %.1fM" % (params / 1e6)) acc = eval(model, test_loader) print("Acc=%.4f\n" % (acc)) elif args.mode == "tensorrt": ckpt = "./checkpoints/resnet18-round%d.pth" % (args.round) print("Load model from %s" % (ckpt)) model = torch.load(ckpt) params = sum([np.prod(p.size()) for p in model.parameters()]) print("Number of Parameters: %.1fM" % (params / 1e6)) torch_in = torch.ones((1, 3, 32, 32)).cuda() torch.onnx.export( model, torch_in, "./checkpoints/model_onnx.onnx", verbose=False, opset_version=12, ) onnx_model = onnx.load("./checkpoints/model_onnx.onnx") model_simp, check = simplify(onnx_model) onnx.save(model_simp, "./checkpoints/model_onnx.onnx") cmd = ( "onnx2trt " + "./checkpoints/model_onnx.onnx" + " -o " + "./checkpoints/tensorrt_engine.engine" + " -b " + "1" + " -w " + str(1024 * 1024 * 1024) + " -d 32" ) os.system(cmd) trt_model = TRT_Engine("./checkpoints/tensorrt_engine.engine", max_batch_size=1) num_iter = 2000 total_time_list = [] with torch.no_grad(): for i in range(num_iter): start = time.time() trt_model(torch_in) total_time_list.append(time.time() - start) print( "total FPS -> avg:{}, max:{}, min:{}".format( 1 / (sum(total_time_list[100:]) / (num_iter - 100)), 1 / (max(total_time_list[100:])), 1 / (min(total_time_list[100:])), ) )
test_baseline = parser.baseline num_epochs = parser.epochs ngpu = parser.ngpu iters = 0 # write into file log_dir = parser.outpdir + "/resnetlogs" ckpt_dir = parser.outpdir if not os.path.exists(ckpt_dir): os.makedirs(ckpt_dir) if not os.path.exists(log_dir): os.makedirs(log_dir) # Load model net = ResNet18().to(device) if (device.type == 'cuda') and ngpu > 1: net = nn.DataParallel(net, list(range(ngpu))) #print(net) # set up your loss function and optmizer criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(net.parameters(), lr=parser.lr) #, momentum=momentum) print('#######################Start Training#######################') with open('resnet_timelogs.csv', 'w') as timefile: with open('resnet_acc.csv', 'w') as accfile: ############################################## ## Train at every epoch ## ############################################## for epoch in range(num_epochs):
# -------------- # Begin Run Loop for run in range(args.n_runs): mir_tries, mir_success = 0, 0 # REPRODUCTIBILITY if args.reproc: np.random.seed(run) torch.manual_seed(run) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False # CLASSIFIER if args.use_conv: cls = ResNet18(args.n_classes, nf=20, input_size=args.input_size) else: #cls = MLP(args) cls = classifier(args).to(args.device) opt = torch.optim.SGD(cls.parameters(), lr=args.lr) if run == 0: print("number of classifier parameters:", sum([np.prod(p.size()) for p in cls.parameters()])) # GENERATIVE MODELING if (args.method is not 'no_rehearsal') or (args.gen_method is not 'no_rehearsal'): gen = VAE.VAE(args).to(args.device) #if args.cuda: # gen = gen.cuda()
generator.load_state_dict(ckpt["g"], strict=False) discriminator_photo.load_state_dict(ckpt["d_p"]) discriminator_cari.load_state_dict(ckpt["d_c"]) discriminator_feat_p.load_state_dict(ckpt["d_feat_p"]) discriminator_feat_c.load_state_dict(ckpt["d_feat_c"]) g_ema.load_state_dict(ckpt["g_ema"], strict=False) g_optim.load_state_dict(ckpt["g_optim"]) d_optim_p.load_state_dict(ckpt["d_optim_p"]) d_optim_c.load_state_dict(ckpt["d_optim_c"]) d_optim_fp.load_state_dict(ckpt["d_optim_fp"]) d_optim_fc.load_state_dict(ckpt["d_optim_fc"]) ### Attribute classifier ### c_cls = ResNet18().to(device) p_cls = ResNet18().to(device) ckpt_pa = torch.load(args.ckpt_pa) ckpt_ca = torch.load(args.ckpt_ca) p_cls.load_state_dict(ckpt_pa) c_cls.load_state_dict(ckpt_ca) p_cls.eval() c_cls.eval() id_net = InceptionResnetV1(pretrained='vggface2').to(device).eval() with torch.no_grad(): mean_latent = generator.photo_generator.mean_latent(4096)
def main(): prior = torch.tensor(opt.prior) input_size = 32 # prepare training data transform = { 'train': transforms.Compose([ transforms.Resize(input_size), transforms.RandomCrop(input_size, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) ]), 'val': transforms.Compose([ transforms.Resize(input_size), transforms.CenterCrop(input_size), transforms.ToTensor(), transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]) ]), } label_trainset = datasets.CIFAR10(root=opt.label_dir, train=True, download=True, transform=transform['train']) label_testset = datasets.CIFAR10(root=opt.label_dir, train=False, download=True, transform=transform['val']) labelset_index = [] classnum = np.ones(opt.num_classes) * opt.pos_num i = 0 while (np.sum(classnum) > 0): image = label_trainset[i][0] label = label_trainset[i][1] if (classnum[label] > 0): labelset_index.append(i) classnum[label] -= 1 i += 1 label_train_subset = torch.utils.data.Subset(label_trainset, labelset_index) target_transform = transforms.Lambda( lambda target: target + opt.num_classes) unlabel_set = datasets.ImageFolder(opt.unlabel_dir, transform['train'], target_transform=target_transform) trainloader = torch.utils.data.DataLoader(label_train_subset + unlabel_set, batch_size=opt.pu_batchsize, shuffle=True, num_workers=32) testloader = torch.utils.data.DataLoader(label_testset, batch_size=opt.pu_batchsize, shuffle=False, num_workers=32) dataloader = {} dataloader['train'] = trainloader dataloader['val'] = testloader # stage1: get positive data from unlabeled dataset model_pu = nn.DataParallel( initialize_model(use_pu=True, num_classes=opt.num_classes)).cuda() optimizer_pu = optim.SGD(model_pu.parameters(), lr=opt.pu_lr, momentum=opt.momentum, weight_decay=opt.pu_weight_decay) scheduler_pu = optim.lr_scheduler.MultiStepLR(optimizer_pu, milestones=[50, 100, 150], gamma=0.1, last_epoch=-1) model_pu = train_pu(model_pu, dataloader, optimizer_pu, scheduler_pu, prior=prior, num_classes=opt.num_classes, num_epochs=opt.pu_num_epochs) trainloader2 = torch.utils.data.DataLoader(unlabel_set, batch_size=opt.pu_batchsize, shuffle=False, num_workers=32) positive_index_all = get_positive(model_pu, trainloader2) print("We have {} positive unlabeled images for all!".format( len(positive_index_all))) unlabel_positive_set = torch.utils.data.Subset(unlabel_set, positive_index_all) trainloader3 = torch.utils.data.DataLoader(label_train_subset + unlabel_positive_set, batch_size=opt.batchsize, shuffle=True, num_workers=32) dataloader['train'] = trainloader3 # stage2: train student model with rkd method teacher = nn.DataParallel( initialize_model(use_pu=False, num_classes=opt.num_classes)).cuda() student = nn.DataParallel(ResNet18(num_classes=opt.num_classes)).cuda() class_weight = get_class_weight(teacher, trainloader3, num_classes=opt.num_classes) print(class_weight) class_weights = perturb(class_weight, opt.epsilon, opt.perturb_num) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(student.parameters(), lr=opt.lr, momentum=opt.momentum, weight_decay=opt.weight_decay) scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[50, 100, 150], gamma=0.1, last_epoch=-1) model_s, hist = train(student, teacher, class_weights, dataloader, criterion, optimizer, scheduler, num_epochs=opt.num_epochs)
axes[i][j].set_axis_off() axes[i][j].imshow(X_train_orig[k].reshape((28, 28))) #plt.show() # Normalize image vectors X_train = X_train_orig / 255. X_test = X_test_orig / 255. # Convert training and test labels to one hot matrices label_binrizer = LabelBinarizer() Y_train = label_binrizer.fit_transform(Y_train_orig) Y_test = label_binrizer.fit_transform(Y_test_orig) print("number of training examples = " + str(X_train.shape[0])) print("number of test examples = " + str(X_test.shape[0])) print("X_train shape: " + str(X_train.shape)) print("Y_train shape: " + str(Y_train.shape)) print("X_test shape: " + str(X_test.shape)) print("Y_test shape: " + str(Y_test.shape)) # train the neural network model = ResNet18(input_shape=(28, 28, 1), classes=24) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) model.fit(X_train, Y_train, epochs=2, batch_size=32) # test the neural network preds = model.evaluate(X_test, Y_test) print("Loss = " + str(preds[0])) print("Test Accuracy = " + str(preds[1]))
train_tfm = tfm.Compose([tfm.RandomCrop(28, padding=4), tfm.RandomHorizontalFlip(), tfm.ToTensor(), tfm.Normalize(mean=(0.1307,), std=(0.3081,)) ]) test_tfm = tfm.Compose([ tfm.ToTensor(), tfm.Normalize(mean=(0.1307,), std=(0.3081,)) ]) fmnist_train = FashionMNIST(args.save_dir, train=True, transform=train_tfm, download=True) fmnist_test = FashionMNIST(args.save_dir, train=False, transform=test_tfm, download=True) device = 'cuda' if torch.cuda.is_available() else 'cpu' clf = ResNet18(nc=1) clf.to(device) optimizer = optim.SGD(clf.parameters(), lr=args.lr, weight_decay=args.wd, momentum=args.mom) criterion = nn.CrossEntropyLoss() # Multiplies the LR with 0.1 at epoch 100 and 150 as mentioned in the paper lmd = lambda x: 0.1 if x in [100,150] else 1 scheduler = lr_scheduler.MultiplicativeLR(optimizer, lr_lambda=lmd) trainloader = DataLoader(fmnist_train, batch_size=args.batch_size, shuffle=True) testloader = DataLoader(fmnist_test, batch_size=args.batch_size, shuffle=False) best_loss = np.inf for epoch in range(args.epochs):