def get_experiment_dataloaders(experiment_name: str, data_dir: str = './data/'): first_level = get_num_levels(experiment_name)[0] config = get_experiment_config(experiment_name, first_level) config = config_dict_to_namespace(config) config.data_dir = os.path.join(data_dir, config.dataset) torch.manual_seed(config.random_seed) kwargs = {} if not config.disable_cuda and torch.cuda.is_available(): use_gpu = True torch.cuda.manual_seed_all(config.random_seed) kwargs = { 'num_workers': config.num_workers, 'pin_memory': config.pin_memory } else: use_gpu = False data_dict = get_dataset(config.dataset, config.data_dir, 'test') kwargs.update(data_dict) config.num_classes = data_dict['num_classes'] test_loader = get_test_loader(batch_size=config.batch_size, **kwargs) if 'cifar' in config.dataset: valid_loader = test_loader else: valid_loader = get_test_loader(batch_size=config.batch_size, **kwargs) if config.is_train: data_dict = get_dataset(config.dataset, config.data_dir, 'train') teachers = [] kwargs.update(data_dict) train_loader = get_train_loader(batch_size=config.batch_size, padding=config.padding, padding_mode=config.padding_mode, random_seed=config.random_seed, shuffle=config.shuffle, model_num=len(config.model_names), teachers=teachers, cuda=use_gpu, **kwargs) else: train_loader = None return train_loader, valid_loader, test_loader
def train(opt): # Load models print('----------- Network Initialization --------------') student = select_model(dataset=opt.data_name, model_name=opt.s_name, pretrained=False, pretrained_models_path=opt.s_model, n_classes=opt.num_class).to(opt.device) print('finished student model init...') nets = {'snet': student} # initialize optimizer optimizer = torch.optim.SGD(student.parameters(), lr=opt.lr, momentum=opt.momentum, weight_decay=opt.weight_decay, nesterov=True) # define loss functions if opt.cuda: criterionCls = nn.CrossEntropyLoss().cuda() else: criterionCls = nn.CrossEntropyLoss() print('----------- DATA Initialization --------------') train_loader = get_backdoor_loader(opt) test_clean_loader, test_bad_loader = get_test_loader(opt) print('----------- Train Initialization --------------') for epoch in range(1, opt.epochs): _adjust_learning_rate(optimizer, epoch, opt.lr) # train every epoch criterions = {'criterionCls': criterionCls} train_step(opt, train_loader, nets, optimizer, criterions, epoch) # evaluate on testing set print('testing the models......') acc_clean, acc_bad = test(opt, test_clean_loader, test_bad_loader, nets, criterions, epoch) # remember best precision and save checkpoint if opt.save: is_best = acc_bad[0] > opt.threshold_bad opt.threshold_bad = min(acc_bad[0], opt.threshold_bad) best_clean_acc = acc_clean[0] best_bad_acc = acc_bad[0] s_name = opt.s_name + '-S-model_best.pth' save_checkpoint( { 'epoch': epoch, 'state_dict': student.state_dict(), 'best_clean_acc': best_clean_acc, 'best_bad_acc': best_bad_acc, 'optimizer': optimizer.state_dict(), }, is_best, opt.checkpoint_root, s_name)
def run(config): kwargs = {} if config.use_gpu: # ensure reproducibility torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.manual_seed(0) np.random.seed(0) kwargs = {'num_workers': config.num_workers} # instantiate data loaders if config.is_train: data_loader = get_train_loader(config.data_dir, config.batch_size, is_shuffle=True**kwargs) else: data_loader = get_test_loader(config.data_dir, config.batch_size, is_shuffle=False, **kwargs) # instantiate trainer trainer = Trainer(config, data_loader) # either train if config.is_train: trainer.train() # or load a pretrained model and test else: trainer.test()
def test(self): # Load best model model = SiameseNet() _, _, _, model_state, _ = self.load_checkpoint(best=self.config.best) model.load_state_dict(model_state) if self.config.use_gpu: model.cuda() test_loader = get_test_loader(self.config.data_dir, self.config.way, self.config.test_trials, self.config.seed, self.config.num_workers, self.config.pin_memory) correct_sum = 0 num_test = test_loader.dataset.trials print(f"[*] Test on {num_test} pairs.") pbar = tqdm(enumerate(test_loader), total=num_test, desc="Test") with torch.no_grad(): for i, (x1, x2, _) in pbar: if self.config.use_gpu: x1, x2 = x1.to(self.device), x2.to(self.device) # compute log probabilities out = model(x1, x2) y_pred = torch.sigmoid(out) y_pred = torch.argmax(y_pred) if y_pred == 0: correct_sum += 1 pbar.set_postfix_str(f"accuracy: {correct_sum / num_test}") test_acc = (100. * correct_sum) / num_test print(f"Test Acc: {correct_sum}/{num_test} ({test_acc:.2f}%)")
def main(config): # ensure directories are setup prepare_dirs(config) # ensure reproducibility torch.manual_seed(config.random_seed) kwargs = {} if torch.cuda.is_available(): torch.cuda.manual_seed(config.random_seed) kwargs = {'num_workers': 4, 'pin_memory': False} # instantiate data loaders if config.is_train: data_loader = get_train_valid_loader(config.data_dir, config.dataset, config.batch_size, config.random_seed, config.exp, config.valid_size, config.shuffle, **kwargs) else: data_loader = get_test_loader(config.data_dir, config.dataset, config.batch_size, config.exp, config.familiar, **kwargs) # instantiate trainer trainer = Trainer(config, data_loader) if config.is_train: trainer.train() else: if config.attack: trainer.test_attack() else: trainer.test()
def main(config): # ensure directories are setup prepare_dirs(config) # ensure reproducibility torch.manual_seed(config.random_seed) kwargs = {} if config.use_gpu: torch.cuda.manual_seed(config.random_seed) kwargs = {'num_workers': 1, 'pin_memory': True} # instantiate data loaders if config.is_train: data_loader = get_train_valid_loader(config.data_dir, config.batch_size, config.random_seed, config.valid_size, config.shuffle, config.show_sample, **kwargs) else: data_loader = get_test_loader(config.data_dir, config.batch_size, **kwargs) # instantiate trainer trainer = Trainer(config, data_loader) # either train if config.is_train: save_config(config) trainer.train() # or load a pretrained model and test else: trainer.test()
def train(model, early_stopping): optimizer = optim.Adam(model.parameters(), lr=1e-3) # data_iter = data_loader.get_loader(batch_size=args.batch_size) data_iter = data_loader.get_train_loader(batch_size=args.batch_size) for epoch in range(args.epochs): model.train() run_loss = 0.0 for idx, data in enumerate(data_iter): data = utils.to_var(data) ret = model.run_on_batch(data, optimizer, epoch) run_loss += ret['loss'].item() print('\r Progress epoch {}, {:.2f}%, average loss {}'.format( epoch, (idx + 1) * 100.0 / len(data_iter), run_loss / (idx + 1.0))) test_data_iter = data_loader.get_test_loader( batch_size=args.batch_size) valid_loss = evaluate(model, test_data_iter) # early stop early_stopping(valid_loss, model) if early_stopping.early_stop: print("Early stopping") break
def extract_feature(dir_path, net): features = [] infos = [] # get test dataset test_loader = get_test_loader(dir_path) # change mode to test net.eval() for i, (images, info) in enumerate(test_loader): if torch.cuda.is_available(): images = Variable(images, volatile=True).cuda() else: images = Variable(images, volatile=True) feature, output = net(images) feature = feature.cpu() feature = feature.data.numpy() feature = np.squeeze(feature) features.append(feature) # features.append(np.squeeze(feature)) person = int(np.squeeze(info[0].numpy())) camera = int(np.squeeze(info[1].numpy())) info = (person, camera) infos.append(info) features = np.asarray(features) return features, infos
def main(config): utils.prepare_dirs(config) # ensure reproducibility torch.manual_seed(config.random_seed) kwargs = {} if config.use_gpu: torch.cuda.manual_seed(config.random_seed) kwargs = {"num_workers": 1, "pin_memory": True} # instantiate data loaders if config.is_train: dloader = data_loader.get_train_valid_loader( config.data_dir, config.batch_size, config.random_seed, config.valid_size, config.shuffle, config.show_sample, **kwargs, ) else: dloader = data_loader.get_test_loader( config.data_dir, config.batch_size, **kwargs, ) trainer = Trainer(config, dloader) # either train if config.is_train: utils.save_config(config) trainer.train() # or load a pretrained model and test else: trainer.test()
def main(config): # ensure reproducibility torch.manual_seed(config.random_seed) kwargs = {} if config.cuda: torch.cuda.manual_seed(config.random_seed) kwargs = {'num_workers': 1, 'pin_memory': True} scores = [] # instantiate data loaders count = 0 times = [] for i in [1, 2, 3]: start = time.time() count = i train_data, test_data = load_dataset(config.data_dir, str(count)) # instantiate data loaders data_loader = get_train_valid_loader(train_data, config.batch_size, config.random_seed, config.valid_size, config.shuffle, config.show_sample, **kwargs) test_loader = get_test_loader(test_data, config.batch_size, **kwargs) # instantiate trainer trainer = Trainer(config, count, data_loader, test_loader) trainer.train() result = trainer.test() scores.append(result) elapsed = time.time() - start times.append(elapsed) scores = np.array(scores) times = np.array(times) print('>>> scores:', scores) print('aver time', times.mean()) # print('avg\tacc\tf1\tprec\trec\tauc') print('acc:', scores.mean(axis=0)[0], '\nf1', scores.mean(axis=0)[1], '\nprec', scores.mean(axis=0)[2], '\nrec', scores.mean(axis=0)[3]) print('>>> std') print('acc:', scores.std(axis=0)[0], '\nf1', scores.std(axis=0)[1], '\nprec', scores.std(axis=0)[2], '\nrec', scores.std(axis=0)[3])
def main(): args = get_args() print('-' * 10, 'Args', '-' * 10) print(args) print('-' * 10, '----', '-' * 10) root_dir = os.path.join(args.NAS_dir, 'data') root_dir = os.path.join(root_dir, args.data_type) model_name = os.path.basename(root_dir)+'_'+args.network+'_epochs:'+str(args.epochs)+\ '_slide_level-'+str(args.slide_level)+\ '_p_size-'+str(args.p_size)+'.md' device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') # print(device) # sys.exit() criterion = nn.CrossEntropyLoss() if args.mode == 'train': print('train mode start') model_conv, _, _ = initialize_model(args.network, 2) dataloaders, dataset_sizes = get_train_val_loader( root_dir, batch_size=args.batch_size, slide_level=args.slide_level, p_size=args.p_size) model_conv = model_conv.to(device) optimizer = optim.SGD(model_conv.parameters(), lr=args.learning_rate, momentum=0.9) scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1) model = train(model_conv, dataloaders, dataset_sizes, device, criterion, optimizer, scheduler, num_epochs=args.epochs) print('save model : {}'.format(os.path.join('model', model_name))) save_model(model, 'model', model_name) elif args.mode == 'test': print('test mode start') print('model name is:', model_name) model_conv = torch.load(os.path.join('model', model_name)) dataloaders, dataset_sizes = get_test_loader( root_dir, slide_level=args.slide_level, p_size=args.p_size) test(model_conv, dataloaders, dataset_sizes, device, criterion)
def main(args): # for fast training. torch.backends.cudnn.benchmark = True setup_seed(args.seed) # create directories if not exist. create_folder(args.save_root_dir, args.version, args.model_save_path) create_folder(args.save_root_dir, args.version, args.sample_path) create_folder(args.save_root_dir, args.version, args.log_path) create_folder(args.save_root_dir, args.version, args.val_result_path) create_folder(args.save_root_dir, args.version, args.test_result_path) if args.mode == 'train': loaders = Munch(ref=get_train_loader(root=args.train_img_dir, img_size=args.image_size, resize_size=args.resize_size, batch_size=args.train_batch_size, shuffle=args.shuffle, num_workers=args.num_workers, drop_last=args.drop_last), val=get_test_loader(root=args.val_img_dir, batch_size=args.val_batch_size, shuffle=True, num_workers=args.num_workers)) trainer = Trainer(loaders, args) trainer.train() elif args.mode == 'test': loaders = Munch(tes=get_test_loader(root=args.test_img_dir, img_size=args.test_img_size, batch_size=args.val_batch_size, shuffle=True, num_workers=args.num_workers)) tester = Tester(loaders, args) tester.test() else: raise NotImplementedError('Mode [{}] is not found'.format(args.mode))
def main(config): # ensure directories are setup prepare_dirs(config) # ensure reproducibility torch.manual_seed(config.random_seed) kwargs = {} if config.use_gpu: torch.cuda.manual_seed(config.random_seed) kwargs = {'num_workers': 1, 'pin_memory': True} # instantiate data loaders if config.is_train: data_loader = get_train_valid_loader( config.data_dir, config.batch_size, config.random_seed, config.valid_size, config.shuffle, config.show_sample, **kwargs ) else: data_loader = get_test_loader( config.data_dir, config.batch_size, **kwargs ) # for data, target in data_loader: # print(data.size()) # print(target.size()) # break # # inputs, classes = next(iter(data_loader)) # print(data_loader) # out = torchvision.utils.make_grid(inputs) # class_names = np.arange(0,10) # imshow(out, title=[class_names[x] for x in classes]) # instantiate trainer trainer = Trainer(config, data_loader) # either train if config.is_train: save_config(config) trainer.train() # or load a pretrained model and test else: trainer.test()
def __init__(self, arch, model_file, test_data, hyperparam, verbose): ################################################################ # cuda setting ################################################################ self.use_cuda = hyperparam['use_cuda'] and torch.cuda.is_available() self.device = "cuda" if self.use_cuda else "cpu" ################################################################ # inference model and test loader ################################################################ self.inference_model = self.load_model(arch, model_file, hyperparam, verbose) self.test_loader = get_test_loader( ibm_data=test_data, to_tensor=hyperparam['to_tensor'], batch_size=hyperparam['valid_batch_size'], mean_for_normalize=hyperparam['train_mean'], std_for_normalize=hyperparam['train_std'], use_cuda=self.use_cuda, shuffle=False)
def ensemble(model_name, file_name, tta=False): preds_raw = [] model = create_model(model_name) test_set = data_loader.get_test_set() rounds = 1 if tta: rounds = 20 loader = data_loader.get_test_loader(model, test_set, tta=True) for index in range(rounds): predictions = np.array(make_preds(model, loader)) preds_raw.append(predictions) preds = np.mean(preds_raw, axis=0) save_array(settings.PREDICT_DIR + os.sep + file_name, preds)
def main(config): # ensure directories are setup prepare_dirs(config) # ensure reproducibility #torch.manual_seed(config.random_seed) kwargs = {} if config.use_gpu: #torch.cuda.manual_seed_all(config.random_seed) kwargs = { 'num_workers': config.num_workers, 'pin_memory': config.pin_memory } #torch.backends.cudnn.deterministic = True # instantiate data loaders test_data_loader = get_test_loader(config.data_dir, config.batch_size, **kwargs) if config.is_train: train_data_loader = get_train_loader(config.data_dir, config.batch_size, config.random_seed, config.shuffle, **kwargs) data_loader = (train_data_loader, test_data_loader) else: data_loader = test_data_loader # instantiate trainer trainer = Trainer(config, data_loader) # either train if config.is_train: save_config(config) trainer.train() # or load a pretrained model and test else: trainer.test()
def main(): wandb.init("AVA") config, unparsed = get_config() # ensure reproducibility torch.manual_seed(config.random_seed) if config.use_gpu: torch.cuda.manual_seed(config.random_seed) # instantiate data loaders if config.is_train: data_loader = get_train_valid_loader(task='MNIST', batch_size=config.batch_size, random_seed=config.random_seed, valid_size=config.valid_size) else: data_loader = get_test_loader(task='MNIST', batch_size=config.batch_size) wandb.config.update(config) # instantiate trainer trainer = Trainer(config, data_loader) # either train trainer.train()
def main(config): # ensure reproducibility torch.manual_seed(config.random_seed) scores = [] # instantiate data loaders count = 0 times = [] for i in range(1, 4): start = time.time() count = i train_data, test_data = load_dataset(config.data_dir, str(count)) # instantiate data loaders data_loader = get_train_loader(train_data, config.batch_size, config.random_seed, config.shuffle) test_loader = get_test_loader(test_data, config.batch_size) # instantiate trainer trainer = Trainer(config, count, data_loader, test_loader) trainer.train() result = trainer.test() scores.append(result) elapsed = time.time() - start times.append(elapsed) scores = np.array(scores) times = np.array(times) print('aver time', times.mean()) # print('avg\tacc\tf1\tprec\trec\tauc') print('acc:', scores.mean(axis=0)[0], '\nf1', scores.mean(axis=0)[1], '\nprec', scores.mean(axis=0)[2], '\nrec', scores.mean(axis=0)[3])
def eval_(model_): test_loader = data_loader.get_test_loader(data_dir='./data', batch_size=16, num_workers=4, pin_memory=False) net = model_ net = net.cuda() optimizer, criterion = optimizer_(net) correct = 0 total = 0 with torch.no_grad(): for data in test_loader: images, labels = data images, labels = images.cuda(), labels.cuda() outputs = net(images) _, predicted = torch.max(outputs.data, 1) total += labels.size(0) correct += (predicted == labels).sum().item() print('Accuracy of the network on the 10000 test images: %d %%' % (100 * correct / total))
def init_loaders(train_batch_size, test_batch_size): import json import h5py from gensim.models.keyedvectors import KeyedVectors with open('./data/datainfo-v1.1.json', 'r') as f: data = json.load(f) f = h5py.File('./data/resnet_features.hdf5', 'r') img_features = f['resnet152_features'][()] f.close() wordvectors_file_vec = './data/fasttext-sbwc.vec' # count = 1000 wordvectors = KeyedVectors.load_word2vec_format( wordvectors_file_vec) #, limit=count) train_loader = get_train_loader(wordvectors, data, img_features, train_batch_size) test_loader = get_test_loader(wordvectors, data, img_features, test_batch_size) return train_loader, test_loader
def main(**kwargs): global args lowest_error1 = 100 for arg, v in kwargs.items(): args.__setattr__(arg, v) program_start_time = time.time() instanceName = "Classification_Accuracy" folder_path = os.path.dirname( os.path.abspath(__file__)) + os.sep + args.model timestamp = datetime.datetime.now() ts_str = timestamp.strftime('%Y-%m-%d-%H-%M-%S') path = folder_path + os.sep + instanceName + os.sep + args.model_name + os.sep + ts_str tensorboard_folder = path + os.sep + "Graph" os.makedirs(path) args.savedir = path writer = SummaryWriter(tensorboard_folder) global logFile logFile = path + os.sep + "log.txt" args.filename = logFile global num_outputs print(args) global device device = 'cuda' if torch.cuda.is_available() else 'cpu' if args.data == "cifar100" or args.data == "CIFAR100": fig_title_str = " on CIFAR-100" elif args.data == "cifar10" or args.data == "CIFAR10": fig_title_str = " on CIFAR-10" elif args.data == "tiny_imagenet": fig_title_str = " on tiny_imagenet" else: LOG( "ERROR =============================dataset should be CIFAR10 or CIFAR100", logFile) NotImplementedError captionStrDict = { "fig_title": fig_title_str, "x_label": "epoch", 'elastic_final_layer_label': "Final_Layer_Output_Classifier", "elastic_intermediate_layer_label": "Intermediate_Layer_Classifier_" } # save input parameters into log file LOG("program start time: " + ts_str + "\n", logFile) # if args.layers_weight_change == 1: # LOG("weights for intermediate layers: 1/(34-Depth), giving different weights for different intermediate layers output, using the formula weigh = 1/(34-Depth)", logFile) # elif args.layers_weight_change == 0: # LOG("weights for intermediate layers: 1, giving same weights for different intermediate layers output as 1", logFile) # else: # print("Parameter --layers_weight_change, Error") # sys.exit() if args.model == "Elastic_ResNet18" or args.model == "Elastic_ResNet34" or args.model == "Elastic_ResNet50" or args.model == "Elastic_ResNet101" or args.model == "Elastic_ResNet152": model = Elastic_ResNet(args, logFile) elif args.model == "Elastic_InceptionV3": args.target_size = ( 299, 299, 3 ) # since pytorch inceptionv3 pretrained accepts image size (299, 299, 3) instead of (224, 224, 3) model = Elastic_InceptionV3(args, logFile) elif args.model == "Elastic_MobileNet": model = Elastic_MobileNet(args, logFile) elif args.model == "Elastic_VGG16": model = Elastic_VGG16_bn(args, logFile) elif args.model == "Elastic_SqueezeNet": model = Elastic_SqueezeNet(args, logFile) elif args.model == "Elastic_DenseNet121" or args.model == "Elastic_DenseNet169" or args.model == "Elastic_DenseNet201": model = Elastic_DenseNet(args, logFile) else: LOG( "--model parameter should be in ResNet, InceptionV3, MobileNet, VGG16, SqueezeNet, DenseNet", logFile) exit() num_outputs = model.num_outputs # num_outputs = 1 LOG("num_outputs: " + str(num_outputs), logFile) LOG("successfully create model: " + args.model, logFile) args_str = str(args) LOG(args_str, logFile) model = model.to(device) if device == 'cuda': model = torch.nn.DataParallel(model).cuda() cudnn.benchmark = True # TUT thinkstation data folder path data_folder = "/media/yi/e7036176-287c-4b18-9609-9811b8e33769/tiny_imagenet/tiny-imagenet-200" # narvi data folder path # data_folder = "/home/zhouy/data/tiny-imagenet-200" # XPS 15 laptop data folder path # data_folder = "D:\Elastic\data" # args.batch_size = 1 summary(model, (3, 224, 224)) if args.data == "tiny_imagenet": train_loader, test_loader = tiny_image_data_loader(data_folder, args) else: train_loader = get_train_loader(args.data, data_dir=data_folder, batch_size=args.batch_size, augment=False, target_size=args.target_size, random_seed=20180614, valid_size=0.2, shuffle=True, show_sample=False, num_workers=4, pin_memory=True, debug=args.debug) test_loader = get_test_loader(args.data, data_dir=data_folder, batch_size=args.batch_size, shuffle=True, target_size=args.target_size, num_workers=4, pin_memory=True, debug=args.debug) criterion = nn.CrossEntropyLoss().cuda() if args.data != "tiny_imagenet": pretrain_optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad, model.parameters()), args.pretrain_learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) LOG("==> Pretraining for **1** epoches \n", logFile) for pretrain_epoch in range(0, 1): accs, losses, lr = train(train_loader, model, criterion, pretrain_optimizer, pretrain_epoch) epoch_result = " pretrain epoch: " + str( pretrain_epoch) + ", pretrain error: " + str( accs) + ", pretrain loss: " + str( losses) + ", pretrain learning rate: " + str( lr) + ", pretrain total train sum loss: " + str( sum(losses)) LOG(epoch_result, logFile) summary(model, (3, 224, 224)) LOG("==> Full training \n", logFile) for param in model.parameters(): param.requires_grad = True optimizers = [] childs = [] k = 0 for child in model.parameters(): childs.append(child) k += 1 # childs_params = [childs[:9], childs[:15], childs[:21], childs[:27], # childs[:33], childs[:39], childs[:45], childs[:51], # childs[:57], childs[:63], childs[:69], childs[:75], childs] childs_params = [childs[:25], childs[:43], childs[:61], childs] for i in range(num_outputs): optimizer = torch.optim.SGD(childs_params[i], args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) optimizers.append(optimizer) # optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate, betas=(0.9, 0.999), eps=1e-08, weight_decay=args.weight_decay) # summary(model, (3,224,224)) scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', threshold=1e-4, patience=10) # implement early stop by own EarlyStopping_epoch_count = 0 epochs_train_accs = [] epochs_train_top5_accs = [] epochs_train_losses = [] epochs_test_accs = [] epochs_test_losses = [] epochs_lr = [] epochs_test_top5_accs = [] for epoch in range(0, args.epochs): epoch_str = "==================================== epoch %d ==============================" % epoch LOG(epoch_str, logFile) # Train for one epoch accs, losses, lr, accs_top5 = train(train_loader, model, criterion, optimizers, epoch) epochs_train_accs.append(accs) epochs_train_losses.append(losses) epochs_lr.append(lr) epochs_train_top5_accs.append(accs_top5) writer.add_scalar(tensorboard_folder + os.sep + "data" + os.sep + 'lr', lr, epoch) for i, a, l, k in zip(range(len(accs)), accs, losses, accs_top5): writer.add_scalar( tensorboard_folder + os.sep + "data" + os.sep + 'train_error_' + str(i), a, epoch) writer.add_scalar( tensorboard_folder + os.sep + "data" + os.sep + 'train_losses_' + str(i), l, epoch) writer.add_scalar( tensorboard_folder + os.sep + "data" + os.sep + 'train_top5_error_' + str(i), k, epoch) epoch_result = "\ntrain error: " + str(accs) + "top 5 error: " + str( accs_top5) + ", \nloss: " + str( losses) + ", \nlearning rate " + str( lr) + ", \ntotal train sum loss " + str(sum(losses)) LOG(epoch_result, logFile) if num_outputs > 1: writer.add_scalar( tensorboard_folder + os.sep + "data" + os.sep + 'train_total_sum_losses', sum(losses), epoch) losses.append(sum(losses)) # add the total sum loss LOG("train_total_sum_losses: " + str(sum(losses)), logFile) # run on test dataset LOG("==> test \n", logFile) test_accs, test_losses, test_top5_accs = validate( test_loader, model, criterion) epochs_test_accs.append(test_accs) epochs_test_losses.append(test_losses) epochs_test_top5_accs.append(test_top5_accs) for i, a, l, k in zip(range(len(test_accs)), test_accs, test_losses, test_top5_accs): writer.add_scalar( tensorboard_folder + os.sep + "data" + os.sep + 'test_error_' + str(i), a, epoch) writer.add_scalar( tensorboard_folder + os.sep + "data" + os.sep + 'test_losses_' + str(i), l, epoch) writer.add_scalar( tensorboard_folder + os.sep + "data" + os.sep + 'test_top5_losses_' + str(i), k, epoch) test_result_str = "==> Test epoch: \nfinal output classifier error: " + str( test_accs ) + "test top 5 error: " + str(test_top5_accs) + ", \ntest_loss" + str( test_losses) + ", \ntotal test sum loss " + str(sum(test_losses)) LOG(test_result_str, logFile) total_loss = sum(test_losses) if num_outputs > 1: writer.add_scalar( tensorboard_folder + os.sep + "data" + os.sep + 'test_total_sum_losses', total_loss, epoch) test_losses.append(total_loss) # add the total sum loss LOG("test_total_sum_losses: " + str(total_loss), logFile) log_stats(path, accs, losses, lr, test_accs, test_losses, accs_top5, test_top5_accs) # Remember best prec@1 and save checkpoint is_best = test_accs[ -1] < lowest_error1 #error not accuracy, but i don't want to change variable names if is_best: lowest_error1 = test_accs[-1] #但是有个问题,有时是倒数第二个CLF取得更好的结果 save_checkpoint( { 'epoch': epoch, 'model': args.model_name, 'state_dict': model.state_dict(), 'best_prec1': lowest_error1, 'optimizer': optimizer.state_dict(), }, args) # apply early_stop with monitoring val_loss # EarlyStopping(patience=15, score_function=score_function(val_loss), trainer=model) scheduler.step(total_loss) # adjust learning rate with test_loss if epoch == 0: prev_epoch_loss = total_loss # use all intemediate classifiers sum loss instead of only one classifier loss else: if total_loss >= prev_epoch_loss: # means this current epoch doesn't reduce test losses EarlyStopping_epoch_count += 1 if EarlyStopping_epoch_count > 20: LOG( "No improving test_loss for more than 10 epochs, stop running model", logFile) break # n_flops, n_params = measure_model(model, IMAGE_SIZE, IMAGE_SIZE) # FLOPS_result = 'Finished training! FLOPs: %.2fM, Params: %.2fM' % (n_flops / 1e6, n_params / 1e6) # LOG(FLOPS_result, logFile) # print(FLOPS_result) writer.close() end_timestamp = datetime.datetime.now() end_ts_str = end_timestamp.strftime('%Y-%m-%d-%H-%M-%S') LOG("program end time: " + end_ts_str + "\n", logFile) # here plot figures plot_figs(epochs_train_accs, epochs_train_losses, epochs_test_accs, epochs_test_losses, args, captionStrDict) LOG("============Finish============", logFile)
def test(model, savepath): model.load_state_dict(torch.load(savepath)) test_data_iter = data_loader.get_test_loader(batch_size=args.batch_size) valid_loss = evaluate(model, test_data_iter)
args.span_range_height = args.span_range_width = args.span_range args.grid_height = args.grid_width = args.grid_size args.image_height = args.image_width = 28 torch.manual_seed(args.seed) if args.cuda: torch.cuda.manual_seed(args.seed) model = mnist_model.get_model(args) if args.cuda: model.cuda() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum) train_loader = data_loader.get_train_loader(args) test_loader = data_loader.get_test_loader(args) def train(epoch): model.train() for batch_idx, (data, target) in enumerate(train_loader): if args.cuda: data, target = data.cuda(), target.cuda() # print(data.shape) data, target = Variable(data), Variable(target) optimizer.zero_grad() output = model(data) loss = F.nll_loss(output, target) loss.backward() optimizer.step() if batch_idx % args.log_interval == 0:
if args.is_train: train_dataset, val_dataset = get_MNIST_train_val_dataset(args.data_dir) # train_dataset, val_dataset = get_MNIST_train_val_dataset('./data/MNIST') train_loader, val_loader = get_train_val_loader( train_dataset, val_dataset, val_split=args.val_split, random_split=args.random_split, batch_size=args.batch_size, **kwargs) args.num_class = train_loader.dataset.num_class args.num_channels = train_loader.dataset.num_channels else: test_dataset = get_MNIST_test_dataset(args.data_dir) test_loader = get_test_loader(test_dataset, args.batch_size, **kwargs) args.num_class = test_loader.dataset.num_class args.num_channels = test_loader.dataset.num_channels # build RAM model model = RecurrentAttention(args) if args.use_gpu: model.cuda() optimizer = torch.optim.SGD(model.parameters(), lr=args.init_lr, momentum=args.momentum) logger.info('Number of model parameters: {:,}'.format( sum([p.data.nelement() for p in model.parameters()]))) trainer = Trainer(model, optimizer, watch=['acc'], val_watch=['acc'])
# Initialize block_probs to be an even distribution(1/n) block_probs = [[] for i in range(n_blocks)] for i in range(n_blocks): block_probs[i] = [(1/len(blocks[i])) for j in range(len(blocks[i]))] # Initialize block_weights to all start at 0(used to be 1/n) block_weights = [[] for i in range(n_blocks)] for i in range(n_blocks): block_weights[i] = [0 for j in range(len(blocks[i]))] train_loader, valid_loader = custom_dl.get_train_valid_loader(data_dir=root+'/data/cifar10/', batch_size=16, augment=False, random_seed=1) test_loader = custom_dl.get_test_loader(data_dir=root+'/data/cifar10/', batch_size=16) def save_grad(module, gradInput, gradOutput): module.block_grad = gradInput[0] def save_forward(module, forwardInput, output): module.block_forward = output[0] def train_layer(data, i, blocks): net_blocks = [] # For each block layer, chose a random block to train for block_layer in blocks: block_i = np.random.choice(len(block_layer)) block = block_layer[block_i] net_blocks.append(block) net = ProxylessNasNet(first_conv, net_blocks, feature_mix_layer, classifier, ngpu=ngpu).to(device)
def main(config): # ensure directories are setup prepare_dirs(config) # create Omniglot data loaders torch.manual_seed(config.random_seed) kwargs = {} if config.use_gpu: torch.cuda.manual_seed(config.random_seed) kwargs = {'num_workers': 1, 'pin_memory': True} if config.is_train: data_loader = get_train_valid_loader( config.data_dir, config.batch_size, config.num_train, config.augment, config.way, config.valid_trials, config.shuffle, config.random_seed, **kwargs ) else: data_loader = get_test_loader( config.data_dir, config.way, config.test_trials, config.random_seed, **kwargs ) # sample 3 layer wise hyperparams if first time training if config.is_train and not config.resume: print("[*] Sampling layer hyperparameters.") layer_hyperparams = { 'layer_init_lrs': [], 'layer_end_momentums': [], 'layer_l2_regs': [] } for i in range(6): # sample lr = random.uniform(1e-4, 1e-1) mom = random.uniform(0, 1) reg = random.uniform(0, 0.1) # store layer_hyperparams['layer_init_lrs'].append(lr) layer_hyperparams['layer_end_momentums'].append(mom) layer_hyperparams['layer_l2_regs'].append(reg) try: save_config(config, layer_hyperparams) except ValueError: print( "[!] Samples already exist. Either change the model number,", "or delete the json file and rerun.", sep=' ' ) return # else load it from config file else: try: layer_hyperparams = load_config(config) except FileNotFoundError: print("[!] No previously saved config. Set resume to False.") return trainer = Trainer(config, data_loader, layer_hyperparams) if config.is_train: trainer.train() else: trainer.test()
img_var = Variable(img) img_var = img_var.cuda() # send to GPU op = model(img_var) result = op.data[0].cpu().numpy() value = np.amax(result) index = np.argmax(result) print('The image is: ', labels[index]) #%% TESTING ON LANDMASS-2 IMAGES: landmass2_dir = '/home/yazeed/Documents/datasets/landmass-2/' test_loader = get_test_loader(landmass2_dir, batch_size, shuffle=False, num_workers=n_threads, pin_memory=pin_memory) # this only needs to be created once -- then reused: target_onehot = torch.FloatTensor(test_loader.batch_size,len(test_loader.dataset.classes)).zero_() if use_gpu: target_onehot = target_onehot.cuda() for batch_idx, (data, target) in enumerate(test_loader): if use_gpu: data, target = data.cuda(async=True), target.cuda(async=True) # On GPU data, target = Variable(data), Variable(target)
def main(data_dir): siamese_model = Siamese() batch_size = 4 num_train = 30000 augment = True way = 20 trials = 300 epochs = 50 train_loader, val_loader = get_train_valid_loader(data_dir, batch_size, num_train, augment, way, trials, pin_memory=True) criterion = torch.nn.BCELoss() optimizer = torch.optim.SGD(siamese_model.parameters(), lr=1e-3, momentum=0.9) lambda1 = lambda epoch: 0.99**epoch #scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda1) writer = SummaryWriter() siamese_model.cuda() best_accuracy = 0.0 for i in range(epochs): siamese_model.train() #batch_count = 0 avg_train_loss = 0.0 for it, (img_1, img_2, labels) in enumerate(train_loader): optimizer.zero_grad() img_1 = img_1.cuda() img_2 = img_2.cuda() labels = labels.cuda() preds = siamese_model(img_1, img_2) loss = criterion(preds, labels) avg_train_loss += loss.item() writer.add_scalar('Loss_train', loss.item(), len(train_loader) * i + it) loss.backward() optimizer.step() #batch_count+=1 #print(batch_count) siamese_model.eval() count = 0 with torch.no_grad(): for ref_images, candidates in val_loader: ref_images = ref_images.cuda() candidates = candidates.cuda() preds = siamese_model(ref_images, candidates) if torch.argmax(preds) == 0: count += 1 if count / len(val_loader) > best_accuracy: best_accuracy = count / len(val_loader) torch.save(siamese_model.state_dict(), 'best_model.pth') writer.add_scalar('Accuracy_validation', count / trials, i) print('Epoch {} | Train loss {} | Val accuracy {}'.format( i, avg_train_loss / len(train_loader), count / trials)) #scheduler.step() writer.flush() best_model = Siamese().cuda() best_model.load_state_dict(torch.load('best_model.pth')) best_model.eval() trials = 400 test_loader = get_test_loader(data_dir, way, trials) test_count = 0 with torch.no_grad(): for ref_images, candidates in test_loader: ref_images = ref_images.cuda() candidates = candidates.cuda() preds = best_model(ref_images, candidates) if torch.argmax(preds) == 0: test_count += 1 print('Test Accuracy {}'.format(test_count / len(test_loader)))
def train(opt): # Load models print('----------- Network Initialization --------------') teacher = select_model(dataset=opt.data_name, model_name=opt.t_name, pretrained=True, pretrained_models_path=opt.t_model, n_classes=opt.num_class).to(opt.device) print('finished teacher model init...') student = select_model(dataset=opt.data_name, model_name=opt.s_name, pretrained=True, pretrained_models_path=opt.s_model, n_classes=opt.num_class).to(opt.device) print('finished student model init...') teacher.eval() nets = {'snet': student, 'tnet': teacher} for param in teacher.parameters(): param.requires_grad = False # initialize optimizer optimizer = torch.optim.SGD(student.parameters(), lr=opt.lr, momentum=opt.momentum, weight_decay=opt.weight_decay, nesterov=True) # define loss functions if opt.cuda: criterionCls = nn.CrossEntropyLoss().cuda() criterionAT = AT(opt.p) else: criterionCls = nn.CrossEntropyLoss() criterionAT = AT(opt.p) print('----------- DATA Initialization --------------') train_loader = get_train_loader(opt) test_clean_loader, test_bad_loader = get_test_loader(opt) print('----------- Train Initialization --------------') for epoch in range(0, opt.epochs): adjust_learning_rate(optimizer, epoch, opt.lr) # train every epoch criterions = {'criterionCls': criterionCls, 'criterionAT': criterionAT} if epoch == 0: # before training test firstly test(opt, test_clean_loader, test_bad_loader, nets, criterions, epoch) train_step(opt, train_loader, nets, optimizer, criterions, epoch + 1) # evaluate on testing set print('testing the models......') acc_clean, acc_bad = test(opt, test_clean_loader, test_bad_loader, nets, criterions, epoch + 1) # remember best precision and save checkpoint # save_root = opt.checkpoint_root + '/' + opt.s_name if opt.save: is_best = acc_clean[0] > opt.threshold_clean opt.threshold_clean = min(acc_bad[0], opt.threshold_clean) best_clean_acc = acc_clean[0] best_bad_acc = acc_bad[0] save_checkpoint( { 'epoch': epoch, 'state_dict': student.state_dict(), 'best_clean_acc': best_clean_acc, 'best_bad_acc': best_bad_acc, 'optimizer': optimizer.state_dict(), }, is_best, opt.checkpoint_root, opt.s_name)
traintransformSequence) # train_dataset, val_dataset = get_MNIST_train_val_dataset('./data/MNIST') train_loader, val_loader = get_train_val_loader( train_dataset, val_dataset, val_split=args.val_split, random_split=args.random_split, batch_size=args.batch_size, **kwargs) args.num_class = train_loader.dataset.num_class args.num_channels = train_loader.dataset.num_channels else: test_dataset = ImageFolder(os.path.join(args.data_dir, 'test'), testtransformSequence) test_loader = get_test_loader(test_dataset, args.batch_size, **kwargs) args.num_class = test_loader.dataset.num_class args.num_channels = test_loader.dataset.num_channels # build RAM model model = RecurrentAttention(args) if args.use_gpu: model.cuda() # optimizer = torch.optim.SGD(model.parameters(), lr=args.init_lr, momentum=args.momentum) optimizer = torch.optim.Adam(model.parameters(), lr=0.0001, betas=(0.9, 0.999), eps=1e-08, weight_decay=1e-5) logger.info('Number of model parameters: {:,}'.format(