def main(batch_size, test_batch_size, epochs, lr, cuda, seed, log_interval): """ Runs data processing scripts to turn raw data from (../raw) into cleaned data ready to be analyzed (saved in ../processed). """ logger = logging.getLogger(__name__) logger.info('making final data set from raw data') TRAIN_DATA = os.path.join(Path(__file__).resolve().parents[2], 'data', 'processed', 'training.pt') TEST_DATA = os.path.join(Path(__file__).resolve().parents[2], 'data', 'processed', 'test.pt') torch.manual_seed(seed) device = torch.device("cuda" if (cuda and torch.cuda.is_available()) else "cpu") x_train, y_train = torch.load(TRAIN_DATA) x_test, y_test = torch.load(TEST_DATA) train = data_utils.TensorDataset(x_train, y_train) train_loader = data_utils.DataLoader(train, batch_size=batch_size, shuffle=True) test = data_utils.TensorDataset(x_test, y_test) test_loader = data_utils.DataLoader(test, batch_size=test_batch_size, shuffle=True) model = LeNet().to(device) optimizer = optim.Adam(model.parameters(), lr=lr) for epoch in range(1, epochs + 1): train(args, model, device, train_loader, optimizer, epoch) test(args, model, device, test_loader)
def get_model(model): model_path = '../saved' if model == 'LeNet-5': net = LeNet() model_name = 'lenet.pth' elif model == 'VGG-16': net = Vgg16_Net() model_name = 'vgg16.pth' elif model == 'ResNet18': net = ResNet18() model_name = 'resnet18.pth' elif model == 'ResNet34': net = ResNet34() model_name = 'resnet34.pth' elif model == 'ResNet50': net = ResNet50() model_name = 'resnet50.pth' else: net = ResNet101() model_name = 'resnet101.pth' return net, os.path.join(model_path, model_name)
def main(epochs, training=True): results = defaultdict(list) model = LeNet() print(model) if USE_CUDA: # GPU optimization model.cuda() model = torch.nn.DataParallel(model, device_ids=range( torch.cuda.device_count())) cudnn.benchmark = True dataloader = load_data() criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters()) for epoch in range(epochs): steps, losses, acc = train( model, # the model dataloader, # the data provider criterion, # the loss function optimizer, # the optimization algorithm epoch + 1, # current epoch ) # add observations to the dictionary results['step'] += steps results['loss_scores'] += losses results['acc_scores'] += acc results['epoch'] += [epoch + 1] * len(steps) if save: save_checkpoint({ 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), }) return results
ds = tfds.load(name='mnist', as_supervised=True) def preprocess(x, y): x = tf.cast(x, tf.float32) x = tf.reshape(x, shape=[-1]) x /= 255 return x, y ds['train'] = ds['train'].repeat().shuffle(20000).map(preprocess).batch(128) train_iter = iter(ds['train']) ds['test'] = ds['test'].map(preprocess).batch(128) model = LeNet(input_shape=[784], n_classes=10, l2_reg=1e-5, layer_sizes=[400]) model.build(input_shape=[1, 784]) model.summary() model.step_idx = tf.Variable(0, trainable=False) loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) idx = np.random.randint(100, 1000) writer = tf.summary.create_file_writer(logdir=f'logs/test{idx}') # optimizer = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9) optimizer = tf.keras.optimizers.Adam() print(f'logging index is {idx}') # %%
def test_lenet(): data = Tensor(np.ones([32, 1, 32, 32]).astype(np.float32) * 0.01) label = Tensor(np.ones([32]).astype(np.int32)) net = LeNet() train(net, data, label)
import torch from models.lenet import LeNet import torch.nn as nn import torch.optim as optim net = LeNet() print(net) input = torch.randn(1, 1, 32, 32) out = net(input) net.zero_grad() target = torch.randn(10) target = target.view(1, -1) criterion = nn.MSELoss() # create your optimizer optimizer = optim.SGD(net.parameters(), lr=0.01) # in your training loop: optimizer.zero_grad() # zero the gradient buffers output = net(input) loss = criterion(output, target) loss.backward() optimizer.step()
def main(): # Training settings parser = argparse.ArgumentParser(description='PyTorch MNIST Example') parser.add_argument('--batch-size', type=int, default=64, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--world_size', type=int, default=1, help='number of GPUs to use') parser.add_argument('--epochs', type=int, default=10, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.01, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--wd', type=float, default=1e-4, help='weight decay (default: 5e-4)') parser.add_argument('--lr-decay-every', type=int, default=100, help='learning rate decay by 10 every X epochs') parser.add_argument('--lr-decay-scalar', type=float, default=0.1, help='--') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') parser.add_argument( '--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--run_test', default=False, type=str2bool, nargs='?', help='run test only') parser.add_argument( '--limit_training_batches', type=int, default=-1, help='how many batches to do per training, -1 means as many as possible' ) parser.add_argument('--no_grad_clip', default=False, type=str2bool, nargs='?', help='turn off gradient clipping') parser.add_argument('--get_flops', default=False, type=str2bool, nargs='?', help='add hooks to compute flops') parser.add_argument( '--get_inference_time', default=False, type=str2bool, nargs='?', help='runs valid multiple times and reports the result') parser.add_argument('--mgpu', default=False, type=str2bool, nargs='?', help='use data paralization via multiple GPUs') parser.add_argument('--dataset', default="MNIST", type=str, help='dataset for experiment, choice: MNIST, CIFAR10', choices=["MNIST", "CIFAR10", "Imagenet"]) parser.add_argument('--data', metavar='DIR', default='/imagenet', help='path to imagenet dataset') parser.add_argument( '--model', default="lenet3", type=str, help='model selection, choices: lenet3, vgg, mobilenetv2, resnet18', choices=[ "lenet3", "vgg", "mobilenetv2", "resnet18", "resnet152", "resnet50", "resnet50_noskip", "resnet20", "resnet34", "resnet101", "resnet101_noskip", "densenet201_imagenet", 'densenet121_imagenet' ]) parser.add_argument('--tensorboard', type=str2bool, nargs='?', help='Log progress to TensorBoard') parser.add_argument( '--save_models', default=True, type=str2bool, nargs='?', help='if True, models will be saved to the local folder') # ============================PRUNING added parser.add_argument( '--pruning_config', default=None, type=str, help= 'path to pruning configuration file, will overwrite all pruning parameters in arguments' ) parser.add_argument('--group_wd_coeff', type=float, default=0.0, help='group weight decay') parser.add_argument('--name', default='test', type=str, help='experiment name(folder) to store logs') parser.add_argument( '--augment', default=False, type=str2bool, nargs='?', help= 'enable or not augmentation of training dataset, only for CIFAR, def False' ) parser.add_argument('--load_model', default='', type=str, help='path to model weights') parser.add_argument('--pruning', default=False, type=str2bool, nargs='?', help='enable or not pruning, def False') parser.add_argument( '--pruning-threshold', '--pt', default=100.0, type=float, help= 'Max error perc on validation set while pruning (default: 100.0 means always prune)' ) parser.add_argument( '--pruning-momentum', default=0.0, type=float, help= 'Use momentum on criteria between pruning iterations, def 0.0 means no momentum' ) parser.add_argument('--pruning-step', default=15, type=int, help='How often to check loss and do pruning step') parser.add_argument('--prune_per_iteration', default=10, type=int, help='How many neurons to remove at each iteration') parser.add_argument( '--fixed_layer', default=-1, type=int, help='Prune only a given layer with index, use -1 to prune all') parser.add_argument('--start_pruning_after_n_iterations', default=0, type=int, help='from which iteration to start pruning') parser.add_argument('--maximum_pruning_iterations', default=1e8, type=int, help='maximum pruning iterations') parser.add_argument('--starting_neuron', default=0, type=int, help='starting position for oracle pruning') parser.add_argument('--prune_neurons_max', default=-1, type=int, help='prune_neurons_max') parser.add_argument('--pruning-method', default=0, type=int, help='pruning method to be used, see readme.md') parser.add_argument('--pruning_fixed_criteria', default=False, type=str2bool, nargs='?', help='enable or not criteria reevaluation, def False') parser.add_argument('--fixed_network', default=False, type=str2bool, nargs='?', help='fix network for oracle or criteria computation') parser.add_argument( '--zero_lr_for_epochs', default=-1, type=int, help='Learning rate will be set to 0 for given number of updates') parser.add_argument( '--dynamic_network', default=False, type=str2bool, nargs='?', help= 'Creates a new network graph from pruned model, works with ResNet-101 only' ) parser.add_argument('--use_test_as_train', default=False, type=str2bool, nargs='?', help='use testing dataset instead of training') parser.add_argument('--pruning_mask_from', default='', type=str, help='path to mask file precomputed') parser.add_argument( '--compute_flops', default=True, type=str2bool, nargs='?', help= 'if True, will run dummy inference of batch 1 before training to get conv sizes' ) # ============================END pruning added best_prec1 = 0 global global_iteration global group_wd_optimizer global_iteration = 0 args = parser.parse_args() use_cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) args.distributed = args.world_size > 1 if args.distributed: dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=0) device = torch.device("cuda" if use_cuda else "cpu") if args.model == "lenet3": model = LeNet(dataset=args.dataset) elif args.model == "vgg": model = vgg11_bn(pretrained=True) elif args.model == "resnet18": model = PreActResNet18() elif (args.model == "resnet50") or (args.model == "resnet50_noskip"): if args.dataset == "CIFAR10": model = PreActResNet50(dataset=args.dataset) else: from models.resnet import resnet50 skip_gate = True if "noskip" in args.model: skip_gate = False if args.pruning_method not in [22, 40]: skip_gate = False model = resnet50(skip_gate=skip_gate) elif args.model == "resnet34": if not (args.dataset == "CIFAR10"): from models.resnet import resnet34 model = resnet34() elif "resnet101" in args.model: if not (args.dataset == "CIFAR10"): from models.resnet import resnet101 if args.dataset == "Imagenet": classes = 1000 if "noskip" in args.model: model = resnet101(num_classes=classes, skip_gate=False) else: model = resnet101(num_classes=classes) elif args.model == "resnet20": if args.dataset == "CIFAR10": NotImplementedError( "resnet20 is not implemented in the current project") # from models.resnet_cifar import resnet20 # model = resnet20() elif args.model == "resnet152": model = PreActResNet152() elif args.model == "densenet201_imagenet": from models.densenet_imagenet import DenseNet201 model = DenseNet201(gate_types=['output_bn'], pretrained=True) elif args.model == "densenet121_imagenet": from models.densenet_imagenet import DenseNet121 model = DenseNet121(gate_types=['output_bn'], pretrained=True) else: print(args.model, "model is not supported") # dataset loading section if args.dataset == "MNIST": kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} train_loader = torch.utils.data.DataLoader(datasets.MNIST( '../data', train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])), batch_size=args.batch_size, shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader( datasets.MNIST('../data', train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])), batch_size=args.test_batch_size, shuffle=True, **kwargs) elif args.dataset == "CIFAR10": # Data loading code normalize = transforms.Normalize( mean=[x / 255.0 for x in [125.3, 123.0, 113.9]], std=[x / 255.0 for x in [63.0, 62.1, 66.7]]) if args.augment: transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ]) else: transform_train = transforms.Compose([ transforms.ToTensor(), normalize, ]) transform_test = transforms.Compose([transforms.ToTensor(), normalize]) kwargs = {'num_workers': 8, 'pin_memory': True} train_loader = torch.utils.data.DataLoader(datasets.CIFAR10( '../data', train=True, download=True, transform=transform_train), batch_size=args.batch_size, shuffle=True, drop_last=True, **kwargs) test_loader = torch.utils.data.DataLoader( datasets.CIFAR10('../data', train=False, transform=transform_test), batch_size=args.test_batch_size, shuffle=True, **kwargs) elif args.dataset == "Imagenet": traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_dataset = datasets.ImageFolder( traindir, transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) if args.distributed: train_sampler = torch.utils.data.distributed.DistributedSampler( train_dataset) else: train_sampler = None kwargs = {'num_workers': 16} train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), sampler=train_sampler, pin_memory=True, **kwargs) if args.use_test_as_train: train_loader = torch.utils.data.DataLoader( datasets.ImageFolder( valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.batch_size, shuffle=(train_sampler is None), **kwargs) test_loader = torch.utils.data.DataLoader(datasets.ImageFolder( valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.batch_size, shuffle=False, pin_memory=True, **kwargs) ####end dataset preparation if args.dynamic_network: # attempts to load pruned model and modify it be removing pruned channels # works for resnet101 only if (len(args.load_model) > 0) and (args.dynamic_network): if os.path.isfile(args.load_model): load_model_pytorch(model, args.load_model, args.model) else: print("=> no checkpoint found at '{}'".format(args.load_model)) exit() dynamic_network_change_local(model) # save the model log_save_folder = "%s" % args.name if not os.path.exists(log_save_folder): os.makedirs(log_save_folder) if not os.path.exists("%s/models" % (log_save_folder)): os.makedirs("%s/models" % (log_save_folder)) model_save_path = "%s/models/pruned.weights" % (log_save_folder) model_state_dict = model.state_dict() if args.save_models: save_checkpoint({'state_dict': model_state_dict}, False, filename=model_save_path) print("model is defined") # aux function to get size of feature maps # First it adds hooks for each conv layer # Then runs inference with 1 image output_sizes = get_conv_sizes(args, model) if use_cuda and not args.mgpu: model = model.to(device) elif args.distributed: model.cuda() print( "\n\n WARNING: distributed pruning was not verified and might not work correctly" ) model = torch.nn.parallel.DistributedDataParallel(model) elif args.mgpu: model = torch.nn.DataParallel(model).cuda() else: model = model.to(device) print( "model is set to device: use_cuda {}, args.mgpu {}, agrs.distributed {}" .format(use_cuda, args.mgpu, args.distributed)) weight_decay = args.wd if args.fixed_network: weight_decay = 0.0 # remove updates from gate layers, because we want them to be 0 or 1 constantly if 1: parameters_for_update = [] parameters_for_update_named = [] for name, m in model.named_parameters(): if "gate" not in name: parameters_for_update.append(m) parameters_for_update_named.append((name, m)) else: print("skipping parameter", name, "shape:", m.shape) total_size_params = sum( [np.prod(par.shape) for par in parameters_for_update]) print("Total number of parameters, w/o usage of bn consts: ", total_size_params) optimizer = optim.SGD(parameters_for_update, lr=args.lr, momentum=args.momentum, weight_decay=weight_decay) if 1: # helping optimizer to implement group lasso (with very small weight that doesn't affect training) # will be used to calculate number of remaining flops and parameters in the network group_wd_optimizer = group_lasso_decay( parameters_for_update, group_lasso_weight=args.group_wd_coeff, named_parameters=parameters_for_update_named, output_sizes=output_sizes) cudnn.benchmark = True # define objective criterion = nn.CrossEntropyLoss() ###=======================added for pruning # logging part log_save_folder = "%s" % args.name if not os.path.exists(log_save_folder): os.makedirs(log_save_folder) if not os.path.exists("%s/models" % (log_save_folder)): os.makedirs("%s/models" % (log_save_folder)) train_writer = None if args.tensorboard: try: # tensorboardX v1.6 train_writer = SummaryWriter(log_dir="%s" % (log_save_folder)) except: # tensorboardX v1.7 train_writer = SummaryWriter(logdir="%s" % (log_save_folder)) time_point = time.strftime("%Y_%m_%d_%H_%M_%S", time.localtime()) textfile = "%s/log_%s.txt" % (log_save_folder, time_point) stdout = Logger(textfile) sys.stdout = stdout print(" ".join(sys.argv)) # initializing parameters for pruning # we can add weights of different layers or we can add gates (multiplies output with 1, useful only for gradient computation) pruning_engine = None if args.pruning: pruning_settings = dict() if not (args.pruning_config is None): pruning_settings_reader = PruningConfigReader() pruning_settings_reader.read_config(args.pruning_config) pruning_settings = pruning_settings_reader.get_parameters() # overwrite parameters from config file with those from command line # needs manual entry here # user_specified = [key for key in vars(default_args).keys() if not (vars(default_args)[key]==vars(args)[key])] # argv_of_interest = ['pruning_threshold', 'pruning-momentum', 'pruning_step', 'prune_per_iteration', # 'fixed_layer', 'start_pruning_after_n_iterations', 'maximum_pruning_iterations', # 'starting_neuron', 'prune_neurons_max', 'pruning_method'] has_attribute = lambda x: any([x in a for a in sys.argv]) if has_attribute('pruning-momentum'): pruning_settings['pruning_momentum'] = vars( args)['pruning_momentum'] if has_attribute('pruning-method'): pruning_settings['method'] = vars(args)['pruning_method'] pruning_parameters_list = prepare_pruning_list( pruning_settings, model, model_name=args.model, pruning_mask_from=args.pruning_mask_from, name=args.name) print("Total pruning layers:", len(pruning_parameters_list)) folder_to_write = "%s" % log_save_folder + "/" log_folder = folder_to_write pruning_engine = pytorch_pruning(pruning_parameters_list, pruning_settings=pruning_settings, log_folder=log_folder) pruning_engine.connect_tensorboard(train_writer) pruning_engine.dataset = args.dataset pruning_engine.model = args.model pruning_engine.pruning_mask_from = args.pruning_mask_from pruning_engine.load_mask() gates_to_params = connect_gates_with_parameters_for_flops( args.model, parameters_for_update_named) pruning_engine.gates_to_params = gates_to_params ###=======================end for pruning # loading model file if (len(args.load_model) > 0) and (not args.dynamic_network): if os.path.isfile(args.load_model): load_model_pytorch(model, args.load_model, args.model) else: print("=> no checkpoint found at '{}'".format(args.load_model)) exit() if args.tensorboard and 0: if args.dataset == "CIFAR10": dummy_input = torch.rand(1, 3, 32, 32).to(device) elif args.dataset == "Imagenet": dummy_input = torch.rand(1, 3, 224, 224).to(device) train_writer.add_graph(model, dummy_input) for epoch in range(1, args.epochs + 1): if args.distributed: train_sampler.set_epoch(epoch) adjust_learning_rate(args, optimizer, epoch, args.zero_lr_for_epochs, train_writer) if not args.run_test and not args.get_inference_time: train(args, model, device, train_loader, optimizer, epoch, criterion, train_writer=train_writer, pruning_engine=pruning_engine) if args.pruning: # skip validation error calculation and model saving if pruning_engine.method == 50: continue # evaluate on validation set prec1, _ = validate(args, test_loader, model, device, criterion, epoch, train_writer=train_writer) # remember best prec@1 and save checkpoint is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) model_save_path = "%s/models/checkpoint.weights" % (log_save_folder) model_state_dict = model.state_dict() if args.save_models: save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model_state_dict, 'best_prec1': best_prec1, }, is_best, filename=model_save_path)
train_loader = cfg.dataset_loader(root=cfg.mnist_dir, train=True, data_preprocess=train_data_preprocess) test_loader = cfg.dataset_loader(root=cfg.mnist_dir, train=False, data_preprocess=valid_data_preprocess) # test_loader = cfg.dataset_loader(root=cfg.cat_dog_test, train=False, shuffle=False, # data_preprocess=valid_data_preprocess) # ---------------构建网络、定义损失函数、优化器-------------------------- # 构建网络结构 # net = resnet() # net = AlexNet(num_classes=cfg.num_classes) # net = resnet50() # net = resnet18() net = LeNet(num_classes=cfg.num_classes) # net = VGG('VGG11', num_classes=10, dataset='cifar-10') # 重写网络最后一层 # fc_in_features = net.fc.in_features # 网络最后一层的输入通道 # net.fc = nn.Linear(in_features=fc_in_features, out_features=cfg.num_classes) # 将网络结构、损失函数放置在GPU上;配置优化器 net = net.to(cfg.device) # net = nn.DataParallel(net, device_ids=[0, 1]) # criterion = nn.BCEWithLogitsLoss().cuda(device=cfg.device) criterion = nn.CrossEntropyLoss().cuda(device=cfg.device) # 常规优化器:随机梯度下降和Adam #optimizer = optim.SGD(params=net.parameters(), lr=cfg.learning_rate, # weight_decay=cfg.weight_decay, momentum=cfg.momentum) optimizer = optim.Adam(params=net.parameters(), lr=cfg.learning_rate,
transform=transforms.Compose( [transforms.Resize((32, 32)), transforms.ToTensor()])) data_test = MNIST('./data/mnist', train=False, download=True, transform=transforms.Compose( [transforms.Resize((32, 32)), transforms.ToTensor()])) data_train_loader = DataLoader(data_train, batch_size=256, shuffle=True, num_workers=8) data_test_loader = DataLoader(data_test, batch_size=1024, num_workers=8) net = LeNet() criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(net.parameters(), lr=2e-3) cur_batch_win = None cur_batch_win_opts = { 'title': 'Epoch Loss Trace', 'xlabel': 'Batch Number', 'ylabel': 'Loss', 'width': 1200, 'height': 600, } def train(epoch): global cur_batch_win
# ============================================================================= # model = CNN(num_classes, # compile_model=False) # model.load_weights('checkpoints/cnn_best_weights/' + \ # 'epoch.152_val_loss.0.600990.h5') # model.compile(loss="categorical_crossentropy", # optimizer='adam', # metrics=["accuracy"]) # loss,acc = model.evaluate_generator(test, steps=test_steps, verbose=2) # print("Restored model, accuracy: {:5.2f}%".format(100*acc)) # ============================================================================= model = LeNet(num_classes, lr=3e-4, optimizer_type="adam", reg=1e-3) #model = Simple_NN(num_classes, lr=.01, reg=0.0) start = time.time() model.fit(train, epochs=10, steps_per_epoch=steps_per_epoch, validation_data=valid, validation_steps=validation_steps, verbose=1) end = time.time() print(end - start) # Experiments experiment_notes = "Experiment 7: Modified LeNet
def create_net(task_config, projector_config): model = task_config['model'] output_size = task_config['output_size'] context_size = projector_config.get('context_size', None) block_in = projector_config.get('block_in', None) block_out = projector_config.get('block_out', None) print("Creating", model) if context_size > 0: hyper = True hyperlayers = ['conv2'] else: hyper = False hyperlayers = [] if model == 'deepbind': num_filters = task_config.get('num_filters', 16) hidden_dim = task_config.get('hidden_dim', 32) net = DeepBind(context_size, block_in, block_out, {'context_size': 100}, hyper, filters=num_filters, hidden_units=hidden_dim) elif model == 'linear_model': input_size = task_config.get('input_dim', 20) net = LinearModel(context_size, block_in, block_out, input_dim=input_size, output_dim=output_size, hyper=hyper) elif model == 'lstm_language_model': layer_size = task_config.get('layer_size', 32) net = LSTMLanguageModel(context_size, block_in, block_out, ninp=layer_size, nhid=layer_size, hyper=hyper) elif model == 'wide_resnet': N = task_config.get('N', 6) k = task_config.get('k', 1) num_classes = output_size net = WideResnet(context_size, block_in, block_out, N, k, num_classes, hyper) elif model == 'lenet': if context_size > 0: hyperlayers = ['conv2', 'fc1', 'fc2'] net = LeNet(context_size, block_in, block_out, hyperlayers) else: print("Please select a valid model kind") sys.exit(0) return net
cnn = PreActResNet34(channels=num_channels, num_classes=num_classes) elif args.model == 'resnet50': cnn = PreActResNet50(channels=num_channels, num_classes=num_classes) elif args.model == 'resnet101': cnn = PreActResNet101(channels=num_channels, num_classes=num_classes) elif args.model == 'resnet152': cnn = PreActResNet152(channels=num_channels, num_classes=num_classes) elif args.model == 'vgg': cnn = VGG(depth=16, num_classes=num_classes, channels=num_channels) elif args.model == 'wideresnet': if args.dataset == 'svhn': cnn = Wide_ResNet(depth=16, num_classes=num_classes, widen_factor=8, dropout_rate=args.dropout_rate) else: cnn = Wide_ResNet(depth=28, num_classes=num_classes, widen_factor=10, dropout_rate=args.dropout_rate) elif args.model == 'lenet': cnn = LeNet() elif args.model == 'magnetlenet': cnn = MagnetLeNet(num_classes) elif args.model == 'magnetfashion': cnn = FashionSimpleNet(num_classes) if args.spladvise or args.magnet: if args.shallow_model == 'resnet18': num_classes = 2 shallow_net = PreActResNet18(channels=num_channels, num_classes=num_classes) elif args.shallow_model == 'resnet34': num_classes = 2 shallow_net = PreActResNet34(channels=num_channels, num_classes=num_classes) elif args.shallow_model == 'resnet50': num_classes = 2 shallow_net = PreActResNet50(channels=num_channels, num_classes=num_classes)
def main(): ##Defining the parser parser = argparse.ArgumentParser(description="Tensorflow Trainer") parser.add_argument("--resume", type=str, help="resume from checkpoint: ./path/model.ckpt") parser.add_argument("--start_iteration", default=0, type=int, help="starting iterations") parser.add_argument("--stop_iteration", default=1000, type=int, help="starting iterations") parser.add_argument("--epochs", default=100, type=int, help="total epochs") parser.add_argument("--gpu", default=0, type=int, help="GPU index") parser.add_argument("--arch", default="yae", type=str, help="architecture to use for training: yae, cae") parser.add_argument("--implicit_units", default=32, type=int, help="implicit units in the code") parser.add_argument("--wdecay", default=0.0, type=float, help="Define the weight decay") parser.add_argument("--lrate", default= 0.0001, type=float, help="Learning rate for Adam") parser.add_argument("--mini_batch", default=128, type=int, help="mini-batch size") parser.add_argument("--lambda_e", default=1.0, type=float, help="Explicit loss mixing coefficient") parser.add_argument("--lambda_i", default=1.0, type=float, help="Implicit loss mixing coefficient") parser.add_argument("--beta", default=1.0, type=float, help="beta hyperparameter used in beta-VAE") args = parser.parse_args() #Set the GPU os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"]=str(args.gpu) import tensorflow as tf #Set global hyperparameters learning_rate = args.lrate mini_batch_size = args.mini_batch tot_epochs = args.epochs tot_labels = 10 dataset_size = 60000 tot_iterations = int((dataset_size / mini_batch_size) * tot_epochs) save_every_iteration = tot_iterations-1 print_every_iteration = 25 features_path = "./datasets/mnist/train/features.npy" labels_path = "./datasets/mnist/train/labels.npy" dataset_train = Dataset() dataset_train.load(features_path, labels_path, tot_labels, normalizer=255.0, shuffle=True, verbose=True) ##Set local hyperparameters if(args.arch=="yae"): simulation_path = "./results/yae" + "_ep" + str(args.epochs) +"_lambdae" + str(args.lambda_e) + "_lambdai" + str(args.lambda_i) from models.yae import Autoencoder my_net = Autoencoder(batch_size=mini_batch_size, channels=1, conv_filters=8, style_size=args.implicit_units, content_size=tot_labels, ksize=(3,3), start_iteration=args.start_iteration, dir_header=simulation_path) elif(args.arch=="cae"): simulation_path = "./results/cae" + "_ep" + str(args.epochs) + "_wdecay" + str(args.wdecay) + "_units" + str(args.implicit_units) from models.cae import Autoencoder my_net = Autoencoder(batch_size=mini_batch_size, channels=1, conv_filters=8, style_size=args.implicit_units, content_size=tot_labels, ksize=(3,3), start_iteration=args.start_iteration, dir_header=simulation_path) elif(args.arch=="cvae"): from models.cvae import Autoencoder simulation_path = "./results/cvae" + "_ep" + str(args.epochs) + "_wdecay" + str(args.wdecay) + "_units" + str(args.implicit_units) + "_beta" + str(args.beta) my_net = Autoencoder(batch_size=mini_batch_size, channels=1, conv_filters=8, style_size=args.implicit_units, content_size=tot_labels, ksize=(3,3), start_iteration=args.start_iteration, dir_header=simulation_path, beta=args.beta) elif(args.arch=="aae"): from models.aae import Autoencoder simulation_path = "./results/aae" + "_ep" + str(args.epochs) + "_wdecay" + str(args.wdecay) + "_units" + str(args.implicit_units) my_net = Autoencoder(batch_size=mini_batch_size, channels=1, conv_filters=4, style_size=args.implicit_units, content_size=tot_labels, ksize=(3,3), start_iteration=args.start_iteration, dir_header=simulation_path) elif(args.arch=="lenet"): simulation_path = "./results/lenet" + "_ep" + str(args.epochs) + "_wdecay" + str(args.wdecay) + "_lr" + str(args.lrate) from models.lenet import LeNet my_net = LeNet(batch_size=mini_batch_size, channels=1, conv_filters=8, tot_labels=10, ksize=(5,5), start_iteration=args.start_iteration, dir_header=simulation_path) else: raise ValueError("[ERROR] The architecture '" + args.arch + "' does not exist!") #Init the session sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) my_net.init_summary(sess) if args.resume: print("[INFO] Resuming from checkpoint: " + str(args.resume)) my_net.load(sess, args.resume) else: sess.run(tf.global_variables_initializer()) #WARNING: do not call it when the load() method is used print("[INFO] Starting training...") for iteration in range(args.start_iteration, tot_iterations): if(args.arch=="yae"): input_features, input_labels = dataset_train.return_features_labels(mini_batch_size, onehot=False) local_loss = my_net.train(sess, input_features, input_labels, learning_rate, args.lambda_e, args.lambda_i, iteration, print_every_iteration) elif(args.arch=="cae"): input_features, input_labels = dataset_train.return_features_labels(mini_batch_size, onehot=False) local_loss = my_net.train(sess, input_features, input_labels, learning_rate, iteration, print_every_iteration) elif(args.arch=="cvae" or args.arch=="aae"): input_features, input_labels = dataset_train.return_features_labels(mini_batch_size, onehot=False) local_loss = my_net.train(sess, input_features, input_labels, learning_rate, iteration, print_every_iteration) elif(args.arch=="lenet"): input_features, input_labels = dataset_train.return_features_labels(mini_batch_size, onehot=False) local_loss = my_net.train(sess, input_features, input_labels, learning_rate, iteration, print_every_iteration) else: raise ValueError("[ERROR] The architecture '" + args.arch + "' does not exist!") if(iteration % print_every_iteration == 0): print("Iteration: " + str(iteration) + "/" + str(tot_iterations) + " [" + str(round((iteration/float(tot_iterations))*100.0, 1)) + "%]") print("Loss: " + str(local_loss)) print("") if(iteration % save_every_iteration == 0 and iteration!=0): my_net.save(sess, verbose=True)
def main(args): check_path(args) # CIFAR-10的全部类别,一共10类 classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck') # 数据集 data_builder = DataBuilder(args) dataSet = DataSet(data_builder.train_builder(), data_builder.test_builder(), classes) # 选择模型 if args.lenet: net = LeNet() model_name = args.name_le elif args.vgg: net = Vgg16_Net() model_name = args.name_vgg elif args.resnet18: net = ResNet18() model_name = args.name_res18 elif args.resnet34: net = ResNet34() model_name = args.name_res34 elif args.resnet50: net = ResNet50() model_name = args.name_res50 elif args.resnet101: net = ResNet101() model_name = args.name_res101 elif args.resnet152: net = ResNet152() model_name = args.name_res152 # 交叉熵损失函数 criterion = nn.CrossEntropyLoss() # SGD优化器 optimizer = optim.SGD(net.parameters(), lr=args.learning_rate, momentum=args.sgd_momentum, weight_decay=args.weight_decay) # 余弦退火调整学习率 scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=150) # 模型的参数保存路径 model_path = os.path.join(args.model_path, model_name) # 启动训练 if args.do_train: print("Training...") trainer = Trainer(net, criterion, optimizer, scheduler, dataSet.train_loader, dataSet.test_loader, model_path, args) trainer.train(epochs=args.epoch) # t.save(net.state_dict(), model_path) # 启动测试,如果--do_train也出现,则用刚刚训练的模型进行测试 # 否则就使用已保存的模型进行测试 if args.do_eval: if not args.do_train and not os.path.exists(model_path): print( "Sorry, there's no saved model yet, you need to train first.") return # --do_eval if not args.do_train: checkpoint = t.load(model_path) net.load_state_dict(checkpoint['net']) accuracy = checkpoint['acc'] epoch = checkpoint['epoch'] print("Using saved model, accuracy : %f epoch: %d" % (accuracy, epoch)) tester = Tester(dataSet.test_loader, net, args) tester.test() if args.show_model: if not os.path.exists(model_path): print( "Sorry, there's no saved model yet, you need to train first.") return show_model(args) if args.do_predict: device = t.device("cuda" if t.cuda.is_available() else "cpu") checkpoint = t.load(model_path, map_location=device) net.load_state_dict(checkpoint['net']) predictor = Predictor(net, classes) img_path = 'test' img_name = [os.path.join(img_path, x) for x in os.listdir(img_path)] for img in img_name: predictor.predict(img)
def run_training(): # A simple transform which we will apply to the MNIST images simple_transform = transforms.Compose( [transforms.ToTensor(), transforms.Normalize((0.5, ), (1.0, ))]) train_set = datasets.MNIST(root=config.DATA_DIR, train=True, transform=simple_transform, download=True) test_set = datasets.MNIST(root=config.DATA_DIR, train=False, transform=simple_transform, download=True) # train_dataset = dataset.ClassificationDataset(image_paths=train_imgs, targets=train_targets, # resize=(config.IMAGE_HEIGHT, config.IMAGE_WIDTH)) train_loader = torch.utils.data.DataLoader(dataset=train_set, batch_size=config.BATCH_SIZE, num_workers=config.NUM_WORKERS, shuffle=True) # test_dataset = dataset.ClassificationDataset(image_paths=test_imgs, targets=test_targets, # resize=(config.IMAGE_HEIGHT, config.IMAGE_WIDTH)) test_loader = torch.utils.data.DataLoader(dataset=test_set, batch_size=config.BATCH_SIZE, num_workers=config.NUM_WORKERS, shuffle=False) model = LeNet() if torch.cuda.is_available(): print('GPU available... using GPU') torch.cuda.manual_seed_all(42) # device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu') model.to(config.DEVICE) optimizer = torch.optim.Adam(model.parameters(), lr=3e-4) # scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( # optimizer, factor=0.8, patience=5, verbose=True # ) criterion = nn.CrossEntropyLoss() train_loss_data = [] test_loss_data = [] for epoch in range(config.EPOCHS): # training train_loss = engine.train_fn(model, train_loader, optimizer, criterion, save_model=True) # validation eval_preds, eval_loss = engine.eval_fn(model, test_loader, criterion) print( f"Epoch {epoch} => Training Loss: {train_loss}, Val Loss: {eval_loss}" ) train_loss_data.append(train_loss) test_loss_data.append(eval_loss) # print(train_dataset[0]) plot_loss(train_loss_data, test_loss_data, plot_path=config.PLOT_PATH) print("done")
checkpoint_dir='./checkpoints' # Placeholders for the input and labels X_source = tf.placeholder(tf.float32, shape=[batch_size, xydim, xydim, cdim]) X_target = tf.placeholder(tf.float32, shape=[batch_size, xydim, xydim, cdim]) Y_source = tf.placeholder(tf.float32, shape=[batch_size, n_classes]) Y_target = tf.placeholder(tf.float32, shape=[batch_size, n_classes]) # Placeholders for lambdas lambda_r = tf.placeholder(tf.float32) lambda_s = tf.placeholder(tf.float32) lambda_p = tf.placeholder(tf.float32) opt_lr = tf.placeholder(tf.float32) # Intialize source/residual streams lenet_model = LeNet(only_features=True, feature_dim=feature_dim).build(X_source) resid_model = ResidualNet().build(lenet_model) ''' Weights for the classifier and discriminator ''' ''' Weights for the classifier and discriminator ''' wC = { "classifier": tf.get_variable('classifier', shape=[500, n_classes], initializer=tf.contrib.layers.xavier_initializer()) } wD = { "fc1": tf.get_variable('fc1_disc', shape=[n_classes, 500], initializer=tf.contrib.layers.xavier_initializer()), "fc2": tf.get_variable('fc2_disc', shape=[500, 500], initializer=tf.contrib.layers.xavier_initializer()), "logits": tf.get_variable('logits_disc', shape=[500, 2], initializer=tf.contrib.layers.xavier_initializer()) }
import os from config import args import tensorflow as tf if args.model == 'lenet': if args.run_name == 'none': from models.lenet import LeNet as Model elif args.run_name == 'dropout': from models.lenet_dropout import LeNet as Model elif args.run_name == 'dropconnect': from models.lenet_dropconnect import LeNet as Model elif args.model == 'fcnet': if args.run_name == "dropout": from models.FCNet_dropout import FCNet as Model elif args.run_name == "dropconnect": from models.FCNet_dropconnect import FCNet as Model elif args.run_name == "none": from models.FCNet import FCNet as Model if __name__ == '__main__': os.environ['CUDA_VISIBLE_DEVICES'] = '1' model = Model(tf.Session(), args) if not os.path.exists(args.modeldir + args.run_name): os.makedirs(args.modeldir + args.run_name) if not os.path.exists(args.logdir + args.run_name): os.makedirs(args.logdir + args.run_name) if args.mode == 'train': model.train() elif args.mode == 'test': model.test(step_num=args.reload_step)
def main_attack(args): # Use CUDA os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id use_cuda = torch.cuda.is_available() device = 'cuda' if use_cuda else 'cpu' # Random seed random.seed(args.seed) torch.manual_seed(args.seed) if use_cuda: torch.cuda.manual_seed_all(args.seed) # Data print(f'==> Preparing dataset {args.dataset}') if args.dataset in ['cifar10', 'cifar100']: detph = 28 widen_factor = 10 dropout = 0.3 transform_test = transforms.Compose([transforms.ToTensor()]) elif args.dataset == 'tiny-imagenet': transform_test = transforms.Compose([transforms.ToTensor()]) elif args.dataset == 'imagenet': transform_test = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor() ]) elif args.dataset == 'mnist': transform_test = transforms.Compose([transforms.ToTensor()]) elif args.dataset == 'SVHN': detph = 16 widen_factor = 4 dropout = 0.4 transform_train = transforms.Compose( [transforms.RandomCrop(32, padding=4), transforms.ToTensor()]) transform_test = transforms.Compose([transforms.ToTensor()]) print(f'Running on dataset {args.dataset}') if args.dataset in ['cifar10', 'cifar100', 'mnist']: if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 num_classes = 10 elif args.dataset == 'cifar100': dataloader = datasets.CIFAR100 num_classes = 100 elif args.dataset == 'mnist': dataloader = datasets.MNIST num_classes = 10 testset = dataloader(root='.data', train=False, download=False, transform=transform_test) elif args.dataset == 'tiny-imagenet': testset = datasets.ImageFolder('tiny-imagenet-200/val', transform=transform_test) num_classes = 200 elif args.dataset == 'imagenet': testset = datasets.ImageFolder('imagenet/val', transform=transform_test) num_classes = 1000 elif args.dataset == 'SVHN': testset = datasets.SVHN('data', split='test', transform=transform_test, download=True) num_classes = 10 testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model if args.arch == 'vgg16': model = VGG16(dataset=args.dataset, num_classes=num_classes, kernels1=args.kernels1, kernels2=args.kernels2, kernels3=args.kernels3, orientations=args.orientations, learn_theta=args.learn_theta) elif args.arch == 'resnet18': model = ResNet18(dataset=args.dataset, num_classes=num_classes, kernels1=args.kernels1, kernels2=args.kernels2, kernels3=args.kernels3, orientations=args.orientations, learn_theta=args.learn_theta) elif args.arch == 'madry': model = MadryNet(kernels1=args.kernels1, kernels2=args.kernels2, kernels3=args.kernels3, orientations=args.orientations, learn_theta=args.learn_theta) elif args.arch == 'lenet': model = LeNet(kernels1=args.kernels1, kernels2=args.kernels2, kernels3=args.kernels3, orientations=args.orientations, learn_theta=args.learn_theta) elif args.arch == 'alexnet': model = AlexNet(dataset=args.dataset, num_classes=num_classes, kernels1=args.kernels1, kernels2=args.kernels2, kernels3=args.kernels3, orientations=args.orientations, learn_theta=args.learn_theta) elif args.arch == 'wide-resnet': model = Wide_ResNet(dataset=args.dataset, num_classes=num_classes, kernels1=args.kernels1, kernels2=args.kernels2, kernels3=args.kernels3, orientations=args.orientations, learn_theta=args.learn_theta, finetune=False, depth=detph, widen_factor=widen_factor, dropout_rate=dropout, use_7x7=args.use_7x7) print('Model:') print(model) if use_cuda: model = torch.nn.DataParallel(model).cuda() # Compute number of parameters and print them param_num = sum(p.numel() for p in model.parameters() if p.requires_grad) param_txt = f' Total trainable params: {param_num:d}' print(param_txt) criterion = nn.CrossEntropyLoss() # Resume # Load checkpoint. print('==> Resuming from checkpoint...') checkpoint_filename = osp.join(args.checkpoint, 'model_best.pth.tar') assert osp.isfile( checkpoint_filename), 'Error: no checkpoint directory found!' checkpoint = torch.load(checkpoint_filename) start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, use_cuda) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) print(f'Running {args.attack} attack!') if args.attack == 'cw': c_vals = torch.logspace(start=-2, end=2, steps=9) for c in c_vals: print(f'Running attack with c = {c:5.3f}') attack_cw(model, testloader, device=device, c=c) print('\n') else: if args.dataset == 'mnist': epsilons = [.1, .2, .3, .4] else: epsilons = [2 / 255, 8 / 255, 16 / 255, .1] print(f'Epsilons are: {epsilons}') minimum = 0. maximum = 1. print(f'Images maxima: {maximum} -- minima: {minimum}') df = { 'epsilons': [ 0., ], 'test_set_accs': [ test_acc, ], 'flip_rates': [ 0., ], } for eps in epsilons: print(f'Running attack with epsilon = {eps:5.3f}') acc_test_set, flip_rate = attack_pgd(model, testloader, device=device, minimum=minimum, maximum=maximum, eps=eps) df['epsilons'].append(eps) df['test_set_accs'].append(acc_test_set) df['flip_rates'].append(flip_rate) print('\n') df = pd.DataFrame.from_dict(df) print('Overall results: \n', df) filename = osp.join(args.checkpoint, 'attack_results.csv') df.to_csv(filename, index=False)
test_loader = torch.utils.data.DataLoader(datasets.MNIST( '../Embedding_data', train=False, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])), batch_size=args.batch_size, shuffle=True, **kwargs) # Model print('==> Building model..') net = LeNet(5) net = net.to(device) if device == 'cuda': net = torch.nn.DataParallel(net) cudnn.benchmark = True if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isdir( 'checkpoint'), 'Error: no checkpoint directory found!' checkpoint = torch.load('./checkpoint/classification_ckpt.t7') net.load_state_dict(checkpoint['net']) best_acc = checkpoint['acc'] start_epoch = checkpoint['epoch'] print('the current best acc is %.3f on epoch %d' %
import init_paths from utils.helpers import read_json_config, get_args from data_loaders.mnist_loader import MNISTDataLoader from models.lenet import LeNet from trainers.mnist_trainer import MNISTTrainer try: args = get_args() config = read_json_config(args.config) except: print('Error on reading config file.') print(config.experiment_name) data_loader = MNISTDataLoader(config) model = LeNet() trainer = MNISTTrainer(model, config, data_loader) trainer.train()
# 配置实例化 # cfg = Cifar10Config() cfg = MnistConfig() # 数据预处理 test_data_preprocess = transforms.Compose([transforms.ToTensor()]) # 获取测试集的加载器 test_loader = cfg.dataset_loader(root=cfg.mnist_dir, train=False, shuffle=False, data_preprocess=test_data_preprocess) # ---------------构建网络、定义损失函数、优化器-------------------------- # net = resnet18() # net = resnet_v2.resnet18(num_classes=cfg.num_classes, type_dataset='cifar-10') # net = vggnet.VGG(vgg_name='VGG11', num_classes=10, dataset='cifar-10') net = LeNet(num_classes=10) # 重写网络最后一层 # fc_in_features = net.fc.in_features # 网络最后一层的输入通道 # net.fc = nn.Linear(in_features=fc_in_features, out_features=cfg.num_classes) # 加载模型权重、将网络放入GPU if os.path.exists(cfg.checkpoints): net.load_state_dict(torch.load(cfg.checkpoints)) print('load model argument...') # -------------进行测试----------------- print('进行测试.....') # 测试函数 print('test stage...\n') # 将网络结构调成验证模式、定义准确率、标签列表和预测列表 net.eval()
def main(): global best_acc start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch if not os.path.isdir(args.checkpoint): mkdir_p(args.checkpoint) print_training_params(args=args, txt_file_path=txt_file_path) # Data print(f'==> Preparing dataset {args.dataset}') if args.dataset in ['cifar10', 'cifar100']: detph = 28 widen_factor = 10 dropout = 0.3 transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor() ]) transform_test = transforms.Compose([ transforms.ToTensor() ]) elif args.dataset == 'tiny-imagenet': transform_train = transforms.Compose([ transforms.ToTensor() ]) transform_test = transforms.Compose([ transforms.ToTensor() ]) elif args.dataset == 'imagenet': transform_train = transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor() ]) transform_test = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor() ]) elif args.dataset == 'mnist': transform_train = transforms.Compose([ transforms.ToTensor() ]) transform_test = transforms.Compose([ transforms.ToTensor() ]) elif args.dataset == 'SVHN': detph = 16 widen_factor = 4 dropout = 0.4 transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.ToTensor() ]) transform_test = transforms.Compose([ transforms.ToTensor() ]) print(f'Running on dataset {args.dataset}') if args.dataset in ['cifar10', 'cifar100', 'mnist']: if args.dataset == 'cifar10': dataloader = datasets.CIFAR10 num_classes = 10 elif args.dataset == 'cifar100': dataloader = datasets.CIFAR100 num_classes = 100 elif args.dataset == 'mnist': dataloader = datasets.MNIST num_classes = 10 trainset = dataloader(root='.data', train=True, download=True, transform=transform_train) testset = dataloader(root='.data', train=False, download=False, transform=transform_test) elif args.dataset == 'imagenet': trainset = datasets.ImageFolder('imagenet/train', transform=transform_train) testset = datasets.ImageFolder('imagenet/val', transform=transform_test) num_classes = 1000 elif args.dataset == 'SVHN': trainset = datasets.SVHN('data', split='train', transform=transform_train, download=True) testset = datasets.SVHN('data', split='test', transform=transform_test, download=True) num_classes = 10 trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers) testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers) # Model print("==> creating model '{}'".format(args.arch)) if args.arch == 'vgg16': model = VGG16( dataset=args.dataset, num_classes=num_classes, kernels1=args.kernels1, kernels2=args.kernels2, kernels3=args.kernels3, orientations=args.orientations, learn_theta=args.learn_theta, finetune=args.finetune ) elif args.arch == 'resnet18': model = ResNet18( dataset=args.dataset, num_classes=num_classes, kernels1=args.kernels1, kernels2=args.kernels2, kernels3=args.kernels3, orientations=args.orientations, learn_theta=args.learn_theta, finetune=args.finetune ) elif args.arch == 'madry': model = MadryNet( kernels1=args.kernels1, kernels2=args.kernels2, kernels3=args.kernels3, orientations=args.orientations, learn_theta=args.learn_theta ) elif args.arch == 'lenet': model = LeNet( kernels1=args.kernels1, kernels2=args.kernels2, kernels3=args.kernels3, orientations=args.orientations, learn_theta=args.learn_theta ) elif args.arch == 'alexnet': model = AlexNet( dataset=args.dataset, num_classes=num_classes, kernels1=args.kernels1, kernels2=args.kernels2, kernels3=args.kernels3, orientations=args.orientations, learn_theta=args.learn_theta ) elif args.arch == 'wide-resnet': model = Wide_ResNet( dataset=args.dataset, num_classes=num_classes, kernels1=args.kernels1, kernels2=args.kernels2, kernels3=args.kernels3, orientations=args.orientations, learn_theta=args.learn_theta, finetune=args.finetune, depth=detph, widen_factor=widen_factor, dropout_rate=dropout, use_7x7=args.use_7x7 ) print('Model:') print(model) print_to_log(text=repr(model), txt_file_path=txt_file_path) if device == 'cuda': model = torch.nn.DataParallel(model).to(device) # Compute number of parameters and print them param_num = sum(p.numel() for p in model.parameters() if p.requires_grad) param_txt = f' Total trainable params: {param_num:d}' print_to_log(text=param_txt, txt_file_path=txt_file_path) print(param_txt) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # Resume title = f'{args.dataset}-' + args.arch if args.resume: # Load checkpoint. print('==> Resuming from checkpoint...') assert osp.isfile(args.resume), 'Error: no checkpoint directory found!' args.checkpoint = osp.dirname(args.resume) checkpoint = torch.load(args.resume) best_acc = checkpoint['best_acc'] start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True) else: logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title) logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.']) if args.evaluate: print('\nEvaluation only') test_loss, test_acc = test(testloader, model, criterion, start_epoch, device) print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc)) return # Train and val for epoch in range(start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr'])) train_loss, train_acc = train( trainloader, model, criterion, optimizer, epoch, device, train_adv=args.train_adv, args=args) test_loss, test_acc = test( testloader, model, criterion, epoch, device) # append logger file logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc]) # save model is_best = test_acc > best_acc best_acc = max(test_acc, best_acc) save_checkpoint({ 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'acc': test_acc, 'best_acc': best_acc, 'optimizer' : optimizer.state_dict(), }, is_best, checkpoint=args.checkpoint) if args.kernels1 is not None: plot_kernels(model, args.checkpoint, epoch, device) logger.close() logger.plot() savefig(os.path.join(args.checkpoint, 'log.eps')) print('Best acc:') print(best_acc) print('Training finished. Running attack') main_attack(args) print('Running SVD computation') main_svs_computation(args)
test_loader = torch.utils.data.DataLoader(datasets.MNIST( '../Embedding_data', train=False, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])), batch_size=bs, shuffle=True, **kwargs) # Model print('==> Building model..') device = 'cuda' if torch.cuda.is_available() else 'cpu' net = LeNet(5) net = net.to(device) if device == 'cuda': net = torch.nn.DataParallel(net) cudnn.benchmark = True # fileList=file_name('checkpoint','.t7') fileList = ['checkpoint/Fisher2_ckpt.t7'] for ckpt_file in fileList: print('==> Resuming from checkpoint ' + ckpt_file) assert os.path.isdir( 'checkpoint'), 'Error: no checkpoint directory found!' checkpoint = torch.load(ckpt_file) net.load_state_dict(checkpoint['net']) best_acc = checkpoint['acc'] net.eval()
def run_magnet_loss(): ''' Test function for the magnet loss ''' m = 8 d = 8 k = 8 alpha = 1.0 batch_size = m * d global plotter plotter = VisdomLinePlotter(env_name=args.name) trainloader, testloader, trainset, testset, n_train = load_dataset(args) emb_dim = 2 n_epochs = 15 epoch_steps = len(trainloader) n_steps = epoch_steps * 15 cluster_refresh_interval = epoch_steps if args.mnist: model = torch.nn.DataParallel(LeNet(emb_dim)).cuda() if args.cifar10: model = torch.nn.DataParallel(VGG(depth=16, num_classes=emb_dim)) print(model) optimizer = optim.Adam(model.parameters(), lr=args.lr) minibatch_magnet_loss = MagnetLoss() images = getattr(trainset, 'train_data') labels = getattr(trainset, 'train_labels') # Get initial embedding initial_reps = compute_reps(model, trainset, 400) if args.cifar10: labels = np.array(labels, dtype=np.float32) # Create batcher batch_builder = ClusterBatchBuilder(labels, k, m, d) batch_builder.update_clusters(initial_reps) batch_losses = [] batch_example_inds, batch_class_inds = batch_builder.gen_batch() trainloader.sampler.batch_indices = batch_example_inds _ = model.train() losses = AverageMeter() for i in tqdm(range(n_steps)): for batch_idx, (img, target) in enumerate(trainloader): img = Variable(img).cuda() target = Variable(target).cuda() optimizer.zero_grad() output, features = model(img) batch_loss, batch_example_losses = minibatch_magnet_loss( output, batch_class_inds, m, d, alpha) batch_loss.backward() optimizer.step() # Update loss index batch_builder.update_losses(batch_example_inds, batch_example_losses) batch_losses.append(batch_loss.data[0]) if not i % 1000: print(i, batch_loss) if not i % cluster_refresh_interval: print("Refreshing clusters") reps = compute_reps(model, trainset, 400) batch_builder.update_clusters(reps) if not i % 2000: n_plot = 10000 plot_embedding(compute_reps(model, trainset, 400)[:n_plot], labels[:n_plot], name=i) batch_example_inds, batch_class_inds = batch_builder.gen_batch() trainloader.sampler.batch_indices = batch_example_inds losses.update(batch_loss, 1) # Log the training loss if args.visdom: plotter.plot('loss', 'train', i, losses.avg.data[0]) # Plot loss curve plot_smooth(batch_losses, "batch-losses")
def main(): ##Defining the parser parser = argparse.ArgumentParser(description="Tensorflow Trainer") parser.add_argument("--resume", type=str, help="the path to the saved network") parser.add_argument( "--path", type=str, help= "the root folder of the experiment (it contains the logs and model)") parser.add_argument("--gendata_path", type=str, help="the folder containing the artificial data") parser.add_argument("--load", type=str, help="load data") parser.add_argument( "--type", default="gendata", type=str, help="type of test: gendata (generate a dataset), loss (return losses)" ) parser.add_argument("--arch", default="yae", type=str, help="architecture to use for training: yae, cae") parser.add_argument("--gpu", default=0, type=int, help="GPU index") parser.add_argument("--lambda_e", default=1.0, type=float, help="Explicit loss mixing coefficient") parser.add_argument("--lambda_i", default=1.0, type=float, help="Implicit loss mixing coefficient") parser.add_argument("--implicit_units", default=32, type=int, help="implicit units in the code") parser.add_argument("--tot_samples", default=16, type=int, help="the total sample generted by measure") parser.add_argument("--batch", default=20000, type=int, help="the batch feed to the LeNet classifier") args = parser.parse_args() #Set the GPU GPU = args.gpu os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = str(GPU) import tensorflow as tf tot_labels = 10 mini_batch_size = 10000 dataset_test = Dataset() features_path = "./datasets/mnist/test/features.npy" labels_path = "./datasets/mnist/test/labels.npy" dataset_test.load(features_path, labels_path, tot_labels, normalizer=255.0, shuffle=False, verbose=True) ##Set the hyper-parameters based on the chosen dataset if (args.arch == "yae"): simulation_path = args.path from models.yae import Autoencoder my_net = Autoencoder(batch_size=mini_batch_size, channels=1, conv_filters=8, style_size=args.implicit_units, content_size=10, ksize=(3, 3), start_iteration=0, dir_header=simulation_path) elif (args.arch == "cae"): simulation_path = args.path from models.cae import Autoencoder my_net = Autoencoder(batch_size=mini_batch_size, channels=1, conv_filters=8, style_size=args.implicit_units, content_size=10, ksize=(3, 3), start_iteration=0, dir_header=simulation_path) elif (args.arch == "cvae"): simulation_path = args.path from models.cvae import Autoencoder my_net = Autoencoder(batch_size=mini_batch_size, channels=1, conv_filters=8, style_size=args.implicit_units, content_size=10, ksize=(3, 3), start_iteration=0, dir_header=simulation_path) elif (args.arch == "aae"): simulation_path = args.path from models.aae import Autoencoder my_net = Autoencoder(batch_size=mini_batch_size, channels=1, conv_filters=4, style_size=args.implicit_units, content_size=10, ksize=(3, 3), start_iteration=0, dir_header=simulation_path) elif (args.arch == "lenet"): simulation_path = args.path from models.lenet import LeNet my_net = LeNet(batch_size=args.batch, channels=1, conv_filters=8, tot_labels=10, ksize=(5, 5), start_iteration=0, dir_header=simulation_path) else: raise ValueError("[ERROR] The dataset does not exist!") #Init the session sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) if args.resume: print("[INFO] Resuming from checkpoint: " + str(args.resume)) my_net.load(sess, args.resume) else: raise ValueError( "[ERROR] To test a model it is necessary to resume from checkpoint..." ) if (args.arch == "yae" and args.type == "loss"): if not os.path.exists(simulation_path + "/test_loss"): os.makedirs(simulation_path + "/test_loss") time_id = strftime("%H%M%S_%d%m%Y", gmtime()) test_loss_path = simulation_path + "/test_loss/" + time_id os.makedirs(test_loss_path) input_features, input_labels = dataset_test.return_features_labels( mini_batch_size, onehot=False) loss, loss_r, loss_c, acc_c, loss_e, loss_i = my_net.test( sess, input_features, input_labels, args.lambda_e, args.lambda_i) with open(test_loss_path + "/test_loss.csv", "w") as text_file: header = "loss, loss_r, loss_c, acc_c, loss_e, loss_i" body = str(loss) + "," + str(loss_r) + "," + str( loss_c) + "," + str(acc_c) + "," + str(loss_e) + "," + str( loss_i) text_file.write(header + '\n' + body) print("====================================") print(header) print(body) print("====================================") elif (args.arch == "cae" and args.type == "loss"): if not os.path.exists(simulation_path + "/test_loss"): os.makedirs(simulation_path + "/test_loss") time_id = strftime("%H%M%S_%d%m%Y", gmtime()) test_loss_path = simulation_path + "/test_loss/" + time_id os.makedirs(test_loss_path) input_features, input_labels = dataset_test.return_features_labels( mini_batch_size, onehot=False) loss = my_net.test(sess, input_features, input_labels) with open(test_loss_path + "/test_loss.csv", "w") as text_file: header = "loss" body = str(loss) text_file.write(header + '\n' + body) print("====================================") print(header) print(body) print("====================================") elif (args.arch == "cvae" and args.type == "loss"): if not os.path.exists(simulation_path + "/test_loss"): os.makedirs(simulation_path + "/test_loss") time_id = strftime("%H%M%S_%d%m%Y", gmtime()) test_loss_path = simulation_path + "/test_loss/" + time_id os.makedirs(test_loss_path) input_features, input_labels = dataset_test.return_features_labels( mini_batch_size, onehot=False) loss = my_net.test(sess, input_features, input_labels) with open(test_loss_path + "/test_loss.csv", "w") as text_file: header = "loss_r" body = str(loss) text_file.write(header + '\n' + body) print("====================================") print(header) print(body) print("====================================") elif ((args.arch == "cae" or args.arch == "cvae" or args.arch == "yae" or args.arch == "aae") and args.type == "gendata"): if not os.path.exists(simulation_path + "/gendata"): os.makedirs(simulation_path + "/gendata") time_id = strftime("%H%M%S_%d%m%Y", gmtime()) gendata_path = simulation_path + "/gendata/" + time_id os.makedirs(gendata_path) print("Starting gendata...") input_features, _ = dataset_test.return_features_labels(10000, onehot=False, shuffle=False) features_list = list() labels_list = list() for i in range(0, 10): print("Input: " + str(i)) print("Input (shape): " + str(input_features.shape)) input_labels = np.ones(10000) * i if (args.arch == "cae" or args.arch == "cvae" or args.arch == "aae"): output = my_net.forward_conditional( sess, input_features, input_labels) #size [10000, 32, 32, 1] elif (args.arch == "yae"): output = my_net.forward_conditional(sess, input_features, input_labels, args.lambda_e, args.lambda_i) print("Output (shape): " + str(output.shape)) output = (output * 255).astype(np.uint8) features_list.append(output) labels_list.append(input_labels) print("Saving data...") features_matrix = np.concatenate(features_list, axis=0) print("Features (shape): " + str(features_matrix.shape)) np.save(gendata_path + "/features", features_matrix) labels_matrix = np.concatenate(labels_list, axis=0) print("Labels (shape): " + str(labels_matrix.shape)) np.save(gendata_path + "/labels", labels_matrix) print("Done!") elif (args.type == "metrics"): print("Iterating the test set...") original_data_matrix = np.load(features_path) data_matrix = np.load(args.gendata_path + "/features.npy") if not os.path.exists(simulation_path + "/sample"): os.makedirs(simulation_path + "/sample") if not os.path.exists(simulation_path + "/metrics"): os.makedirs(simulation_path + "/metrics") time_id = strftime("%H%M%S_%d%m%Y", gmtime()) sample_path = simulation_path + "/sample/" + time_id metrics_path = simulation_path + "/metrics/" + time_id os.makedirs(sample_path) os.makedirs(metrics_path) img_ssim_list = list() img_mse_list = list() for i in range(10000): img_original = original_data_matrix[i, :, :, 0] if (i < args.tot_samples): cv2.imwrite(sample_path + "/" + str(i) + ".png", img_original) for j in range(0, 10): location = (j * 10000) + i img_generated = data_matrix[location, :, :, 0] (score, diff) = compare_ssim(img_original, img_generated, full=True) mse = (img_original - img_generated)**2 img_ssim_list.append(score) img_mse_list.append(mse) if (i < args.tot_samples): cv2.imwrite( sample_path + "/" + str(i) + "_" + str(j) + ".png", img_generated) if (i % 1000 == 0): print(str(i) + "/" + str(10000)) print("====================================") print("INTERNAL SSIM") print("Mean: \t" + str(np.mean(img_ssim_list))) print("Std: \t" + str(np.std(img_ssim_list))) print("------------------------------------") print("INTERNAL MSE") print("Mean: \t" + str(np.mean(img_mse_list))) print("Std: \t" + str(np.std(img_mse_list))) print("====================================") with open(metrics_path + "/test_metrics.csv", "w") as text_file: header = "SSIM, MSE" body = str(np.mean(img_ssim_list)) + ',' + str( np.mean(img_mse_list)) text_file.write(header + '\n' + body) elif (args.arch == "lenet" and args.type == "accuracy"): if not os.path.exists(simulation_path + "/test_gendata_accuracy"): os.makedirs(simulation_path + "/test_gendata_accuracy") time_id = strftime("%H%M%S_%d%m%Y", gmtime()) accuracy_path = simulation_path + "/test_gendata_accuracy/" + time_id os.makedirs(accuracy_path) dataset_test = Dataset() features_path = args.gendata_path + "/features.npy" labels_path = args.gendata_path + "/labels.npy" dataset_test.load(features_path, labels_path, tot_labels, normalizer=255.0, shuffle=True, verbose=True) input_features, input_labels = dataset_test.return_features_labels( args.batch, onehot=False) print(input_features.shape) output = my_net.test(sess, input_features, input_labels) print("==============================") print("Loss:\t\t" + str(output[0])) print("Accuracy:\t" + str(output[1] * 100.0)) print("==============================") with open(accuracy_path + "/test_gendata_accuracy.csv", "w") as text_file: header = "Loss, Accuracy" body = str(output[0]) + ',' + str(output[1] * 100.0) text_file.write(header + '\n' + body) else: raise ValueError( "[ERROR] This test does not exists for the model and dataset.")
test_loader = torch.utils.data.DataLoader(datasets.MNIST( '../Embedding_data', train=False, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])), batch_size=args.batch_size, shuffle=True, **kwargs) # Model print('==> Building model..') net = LeNet(args.partition_proportion) net = net.to(device) if device == 'cuda': net = torch.nn.DataParallel(net) cudnn.benchmark = True if args.resume: # Load checkpoint. print('==> Resuming from checkpoint..') assert os.path.isdir( 'checkpoint'), 'Error: no checkpoint directory found!' if os.path.isfile('./checkpoint/lifted_ckpt.t7'): checkpoint = torch.load('./checkpoint/lifted_ckpt.t7') best_acc = checkpoint['acc'] else: checkpoint = torch.load('./checkpoint/classification_ckpt.t7')
def get_model(model_name): if model_name == "lenet": return LeNet()