def main(batch_size, test_batch_size, epochs, lr, cuda, seed, log_interval):
""" Runs data processing scripts to turn raw data from (../raw) into
cleaned data ready to be analyzed (saved in ../processed).
"""
logger = logging.getLogger(__name__)
logger.info('making final data set from raw data')
TRAIN_DATA = os.path.join(Path(__file__).resolve().parents[2], 'data', 'processed', 'training.pt')
TEST_DATA = os.path.join(Path(__file__).resolve().parents[2], 'data', 'processed', 'test.pt')
torch.manual_seed(seed)
device = torch.device("cuda" if (cuda and torch.cuda.is_available()) else "cpu")
x_train, y_train = torch.load(TRAIN_DATA)
x_test, y_test = torch.load(TEST_DATA)
train = data_utils.TensorDataset(x_train, y_train)
train_loader = data_utils.DataLoader(train, batch_size=batch_size, shuffle=True)
test = data_utils.TensorDataset(x_test, y_test)
test_loader = data_utils.DataLoader(test, batch_size=test_batch_size, shuffle=True)
model = LeNet().to(device)
optimizer = optim.Adam(model.parameters(), lr=lr)
for epoch in range(1, epochs + 1):
train(args, model, device, train_loader, optimizer, epoch)
test(args, model, device, test_loader)
def get_model(model):
model_path = '../saved'
if model == 'LeNet-5':
net = LeNet()
model_name = 'lenet.pth'
elif model == 'VGG-16':
net = Vgg16_Net()
model_name = 'vgg16.pth'
elif model == 'ResNet18':
net = ResNet18()
model_name = 'resnet18.pth'
elif model == 'ResNet34':
net = ResNet34()
model_name = 'resnet34.pth'
elif model == 'ResNet50':
net = ResNet50()
model_name = 'resnet50.pth'
else:
net = ResNet101()
model_name = 'resnet101.pth'
return net, os.path.join(model_path, model_name)
def main(epochs, training=True):
results = defaultdict(list)
model = LeNet()
print(model)
if USE_CUDA: # GPU optimization
model.cuda()
model = torch.nn.DataParallel(model,
device_ids=range(
torch.cuda.device_count()))
cudnn.benchmark = True
dataloader = load_data()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters())
for epoch in range(epochs):
steps, losses, acc = train(
model, # the model
dataloader, # the data provider
criterion, # the loss function
optimizer, # the optimization algorithm
epoch + 1, # current epoch
)
# add observations to the dictionary
results['step'] += steps
results['loss_scores'] += losses
results['acc_scores'] += acc
results['epoch'] += [epoch + 1] * len(steps)
if save:
save_checkpoint({
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
})
return results
ds = tfds.load(name='mnist', as_supervised=True)
def preprocess(x, y):
x = tf.cast(x, tf.float32)
x = tf.reshape(x, shape=[-1])
x /= 255
return x, y
ds['train'] = ds['train'].repeat().shuffle(20000).map(preprocess).batch(128)
train_iter = iter(ds['train'])
ds['test'] = ds['test'].map(preprocess).batch(128)
model = LeNet(input_shape=[784], n_classes=10, l2_reg=1e-5, layer_sizes=[400])
model.build(input_shape=[1, 784])
model.summary()
model.step_idx = tf.Variable(0, trainable=False)
loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
idx = np.random.randint(100, 1000)
writer = tf.summary.create_file_writer(logdir=f'logs/test{idx}')
# optimizer = tf.keras.optimizers.SGD(learning_rate=0.01, momentum=0.9)
optimizer = tf.keras.optimizers.Adam()
print(f'logging index is {idx}')
# %%
def test_lenet():
data = Tensor(np.ones([32, 1, 32, 32]).astype(np.float32) * 0.01)
label = Tensor(np.ones([32]).astype(np.int32))
net = LeNet()
train(net, data, label)
import torch from models.lenet import LeNet import torch.nn as nn import torch.optim as optim net = LeNet() print(net) input = torch.randn(1, 1, 32, 32) out = net(input) net.zero_grad() target = torch.randn(10) target = target.view(1, -1) criterion = nn.MSELoss() # create your optimizer optimizer = optim.SGD(net.parameters(), lr=0.01) # in your training loop: optimizer.zero_grad() # zero the gradient buffers output = net(input) loss = criterion(output, target) loss.backward() optimizer.step()
def main():
# Training settings
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
parser.add_argument('--batch-size',
type=int,
default=64,
metavar='N',
help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size',
type=int,
default=1000,
metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument('--world_size',
type=int,
default=1,
help='number of GPUs to use')
parser.add_argument('--epochs',
type=int,
default=10,
metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('--lr',
type=float,
default=0.01,
metavar='LR',
help='learning rate (default: 0.01)')
parser.add_argument('--wd',
type=float,
default=1e-4,
help='weight decay (default: 5e-4)')
parser.add_argument('--lr-decay-every',
type=int,
default=100,
help='learning rate decay by 10 every X epochs')
parser.add_argument('--lr-decay-scalar',
type=float,
default=0.1,
help='--')
parser.add_argument('--momentum',
type=float,
default=0.5,
metavar='M',
help='SGD momentum (default: 0.5)')
parser.add_argument('--no-cuda',
action='store_true',
default=False,
help='disables CUDA training')
parser.add_argument('--seed',
type=int,
default=1,
metavar='S',
help='random seed (default: 1)')
parser.add_argument(
'--log-interval',
type=int,
default=10,
metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument('--run_test',
default=False,
type=str2bool,
nargs='?',
help='run test only')
parser.add_argument(
'--limit_training_batches',
type=int,
default=-1,
help='how many batches to do per training, -1 means as many as possible'
)
parser.add_argument('--no_grad_clip',
default=False,
type=str2bool,
nargs='?',
help='turn off gradient clipping')
parser.add_argument('--get_flops',
default=False,
type=str2bool,
nargs='?',
help='add hooks to compute flops')
parser.add_argument(
'--get_inference_time',
default=False,
type=str2bool,
nargs='?',
help='runs valid multiple times and reports the result')
parser.add_argument('--mgpu',
default=False,
type=str2bool,
nargs='?',
help='use data paralization via multiple GPUs')
parser.add_argument('--dataset',
default="MNIST",
type=str,
help='dataset for experiment, choice: MNIST, CIFAR10',
choices=["MNIST", "CIFAR10", "Imagenet"])
parser.add_argument('--data',
metavar='DIR',
default='/imagenet',
help='path to imagenet dataset')
parser.add_argument(
'--model',
default="lenet3",
type=str,
help='model selection, choices: lenet3, vgg, mobilenetv2, resnet18',
choices=[
"lenet3", "vgg", "mobilenetv2", "resnet18", "resnet152",
"resnet50", "resnet50_noskip", "resnet20", "resnet34", "resnet101",
"resnet101_noskip", "densenet201_imagenet", 'densenet121_imagenet'
])
parser.add_argument('--tensorboard',
type=str2bool,
nargs='?',
help='Log progress to TensorBoard')
parser.add_argument(
'--save_models',
default=True,
type=str2bool,
nargs='?',
help='if True, models will be saved to the local folder')
# ============================PRUNING added
parser.add_argument(
'--pruning_config',
default=None,
type=str,
help=
'path to pruning configuration file, will overwrite all pruning parameters in arguments'
)
parser.add_argument('--group_wd_coeff',
type=float,
default=0.0,
help='group weight decay')
parser.add_argument('--name',
default='test',
type=str,
help='experiment name(folder) to store logs')
parser.add_argument(
'--augment',
default=False,
type=str2bool,
nargs='?',
help=
'enable or not augmentation of training dataset, only for CIFAR, def False'
)
parser.add_argument('--load_model',
default='',
type=str,
help='path to model weights')
parser.add_argument('--pruning',
default=False,
type=str2bool,
nargs='?',
help='enable or not pruning, def False')
parser.add_argument(
'--pruning-threshold',
'--pt',
default=100.0,
type=float,
help=
'Max error perc on validation set while pruning (default: 100.0 means always prune)'
)
parser.add_argument(
'--pruning-momentum',
default=0.0,
type=float,
help=
'Use momentum on criteria between pruning iterations, def 0.0 means no momentum'
)
parser.add_argument('--pruning-step',
default=15,
type=int,
help='How often to check loss and do pruning step')
parser.add_argument('--prune_per_iteration',
default=10,
type=int,
help='How many neurons to remove at each iteration')
parser.add_argument(
'--fixed_layer',
default=-1,
type=int,
help='Prune only a given layer with index, use -1 to prune all')
parser.add_argument('--start_pruning_after_n_iterations',
default=0,
type=int,
help='from which iteration to start pruning')
parser.add_argument('--maximum_pruning_iterations',
default=1e8,
type=int,
help='maximum pruning iterations')
parser.add_argument('--starting_neuron',
default=0,
type=int,
help='starting position for oracle pruning')
parser.add_argument('--prune_neurons_max',
default=-1,
type=int,
help='prune_neurons_max')
parser.add_argument('--pruning-method',
default=0,
type=int,
help='pruning method to be used, see readme.md')
parser.add_argument('--pruning_fixed_criteria',
default=False,
type=str2bool,
nargs='?',
help='enable or not criteria reevaluation, def False')
parser.add_argument('--fixed_network',
default=False,
type=str2bool,
nargs='?',
help='fix network for oracle or criteria computation')
parser.add_argument(
'--zero_lr_for_epochs',
default=-1,
type=int,
help='Learning rate will be set to 0 for given number of updates')
parser.add_argument(
'--dynamic_network',
default=False,
type=str2bool,
nargs='?',
help=
'Creates a new network graph from pruned model, works with ResNet-101 only'
)
parser.add_argument('--use_test_as_train',
default=False,
type=str2bool,
nargs='?',
help='use testing dataset instead of training')
parser.add_argument('--pruning_mask_from',
default='',
type=str,
help='path to mask file precomputed')
parser.add_argument(
'--compute_flops',
default=True,
type=str2bool,
nargs='?',
help=
'if True, will run dummy inference of batch 1 before training to get conv sizes'
)
# ============================END pruning added
best_prec1 = 0
global global_iteration
global group_wd_optimizer
global_iteration = 0
args = parser.parse_args()
use_cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
args.distributed = args.world_size > 1
if args.distributed:
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size,
rank=0)
device = torch.device("cuda" if use_cuda else "cpu")
if args.model == "lenet3":
model = LeNet(dataset=args.dataset)
elif args.model == "vgg":
model = vgg11_bn(pretrained=True)
elif args.model == "resnet18":
model = PreActResNet18()
elif (args.model == "resnet50") or (args.model == "resnet50_noskip"):
if args.dataset == "CIFAR10":
model = PreActResNet50(dataset=args.dataset)
else:
from models.resnet import resnet50
skip_gate = True
if "noskip" in args.model:
skip_gate = False
if args.pruning_method not in [22, 40]:
skip_gate = False
model = resnet50(skip_gate=skip_gate)
elif args.model == "resnet34":
if not (args.dataset == "CIFAR10"):
from models.resnet import resnet34
model = resnet34()
elif "resnet101" in args.model:
if not (args.dataset == "CIFAR10"):
from models.resnet import resnet101
if args.dataset == "Imagenet":
classes = 1000
if "noskip" in args.model:
model = resnet101(num_classes=classes, skip_gate=False)
else:
model = resnet101(num_classes=classes)
elif args.model == "resnet20":
if args.dataset == "CIFAR10":
NotImplementedError(
"resnet20 is not implemented in the current project")
# from models.resnet_cifar import resnet20
# model = resnet20()
elif args.model == "resnet152":
model = PreActResNet152()
elif args.model == "densenet201_imagenet":
from models.densenet_imagenet import DenseNet201
model = DenseNet201(gate_types=['output_bn'], pretrained=True)
elif args.model == "densenet121_imagenet":
from models.densenet_imagenet import DenseNet121
model = DenseNet121(gate_types=['output_bn'], pretrained=True)
else:
print(args.model, "model is not supported")
# dataset loading section
if args.dataset == "MNIST":
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
train_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../data',
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])),
batch_size=args.batch_size,
shuffle=True,
**kwargs)
test_loader = torch.utils.data.DataLoader(
datasets.MNIST('../data',
train=False,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])),
batch_size=args.test_batch_size,
shuffle=True,
**kwargs)
elif args.dataset == "CIFAR10":
# Data loading code
normalize = transforms.Normalize(
mean=[x / 255.0 for x in [125.3, 123.0, 113.9]],
std=[x / 255.0 for x in [63.0, 62.1, 66.7]])
if args.augment:
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])
else:
transform_train = transforms.Compose([
transforms.ToTensor(),
normalize,
])
transform_test = transforms.Compose([transforms.ToTensor(), normalize])
kwargs = {'num_workers': 8, 'pin_memory': True}
train_loader = torch.utils.data.DataLoader(datasets.CIFAR10(
'../data', train=True, download=True, transform=transform_train),
batch_size=args.batch_size,
shuffle=True,
drop_last=True,
**kwargs)
test_loader = torch.utils.data.DataLoader(
datasets.CIFAR10('../data', train=False, transform=transform_test),
batch_size=args.test_batch_size,
shuffle=True,
**kwargs)
elif args.dataset == "Imagenet":
traindir = os.path.join(args.data, 'train')
valdir = os.path.join(args.data, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_dataset = datasets.ImageFolder(
traindir,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
if args.distributed:
train_sampler = torch.utils.data.distributed.DistributedSampler(
train_dataset)
else:
train_sampler = None
kwargs = {'num_workers': 16}
train_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=args.batch_size,
shuffle=(train_sampler is None),
sampler=train_sampler,
pin_memory=True,
**kwargs)
if args.use_test_as_train:
train_loader = torch.utils.data.DataLoader(
datasets.ImageFolder(
valdir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.batch_size,
shuffle=(train_sampler is None),
**kwargs)
test_loader = torch.utils.data.DataLoader(datasets.ImageFolder(
valdir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
**kwargs)
####end dataset preparation
if args.dynamic_network:
# attempts to load pruned model and modify it be removing pruned channels
# works for resnet101 only
if (len(args.load_model) > 0) and (args.dynamic_network):
if os.path.isfile(args.load_model):
load_model_pytorch(model, args.load_model, args.model)
else:
print("=> no checkpoint found at '{}'".format(args.load_model))
exit()
dynamic_network_change_local(model)
# save the model
log_save_folder = "%s" % args.name
if not os.path.exists(log_save_folder):
os.makedirs(log_save_folder)
if not os.path.exists("%s/models" % (log_save_folder)):
os.makedirs("%s/models" % (log_save_folder))
model_save_path = "%s/models/pruned.weights" % (log_save_folder)
model_state_dict = model.state_dict()
if args.save_models:
save_checkpoint({'state_dict': model_state_dict},
False,
filename=model_save_path)
print("model is defined")
# aux function to get size of feature maps
# First it adds hooks for each conv layer
# Then runs inference with 1 image
output_sizes = get_conv_sizes(args, model)
if use_cuda and not args.mgpu:
model = model.to(device)
elif args.distributed:
model.cuda()
print(
"\n\n WARNING: distributed pruning was not verified and might not work correctly"
)
model = torch.nn.parallel.DistributedDataParallel(model)
elif args.mgpu:
model = torch.nn.DataParallel(model).cuda()
else:
model = model.to(device)
print(
"model is set to device: use_cuda {}, args.mgpu {}, agrs.distributed {}"
.format(use_cuda, args.mgpu, args.distributed))
weight_decay = args.wd
if args.fixed_network:
weight_decay = 0.0
# remove updates from gate layers, because we want them to be 0 or 1 constantly
if 1:
parameters_for_update = []
parameters_for_update_named = []
for name, m in model.named_parameters():
if "gate" not in name:
parameters_for_update.append(m)
parameters_for_update_named.append((name, m))
else:
print("skipping parameter", name, "shape:", m.shape)
total_size_params = sum(
[np.prod(par.shape) for par in parameters_for_update])
print("Total number of parameters, w/o usage of bn consts: ",
total_size_params)
optimizer = optim.SGD(parameters_for_update,
lr=args.lr,
momentum=args.momentum,
weight_decay=weight_decay)
if 1:
# helping optimizer to implement group lasso (with very small weight that doesn't affect training)
# will be used to calculate number of remaining flops and parameters in the network
group_wd_optimizer = group_lasso_decay(
parameters_for_update,
group_lasso_weight=args.group_wd_coeff,
named_parameters=parameters_for_update_named,
output_sizes=output_sizes)
cudnn.benchmark = True
# define objective
criterion = nn.CrossEntropyLoss()
###=======================added for pruning
# logging part
log_save_folder = "%s" % args.name
if not os.path.exists(log_save_folder):
os.makedirs(log_save_folder)
if not os.path.exists("%s/models" % (log_save_folder)):
os.makedirs("%s/models" % (log_save_folder))
train_writer = None
if args.tensorboard:
try:
# tensorboardX v1.6
train_writer = SummaryWriter(log_dir="%s" % (log_save_folder))
except:
# tensorboardX v1.7
train_writer = SummaryWriter(logdir="%s" % (log_save_folder))
time_point = time.strftime("%Y_%m_%d_%H_%M_%S", time.localtime())
textfile = "%s/log_%s.txt" % (log_save_folder, time_point)
stdout = Logger(textfile)
sys.stdout = stdout
print(" ".join(sys.argv))
# initializing parameters for pruning
# we can add weights of different layers or we can add gates (multiplies output with 1, useful only for gradient computation)
pruning_engine = None
if args.pruning:
pruning_settings = dict()
if not (args.pruning_config is None):
pruning_settings_reader = PruningConfigReader()
pruning_settings_reader.read_config(args.pruning_config)
pruning_settings = pruning_settings_reader.get_parameters()
# overwrite parameters from config file with those from command line
# needs manual entry here
# user_specified = [key for key in vars(default_args).keys() if not (vars(default_args)[key]==vars(args)[key])]
# argv_of_interest = ['pruning_threshold', 'pruning-momentum', 'pruning_step', 'prune_per_iteration',
# 'fixed_layer', 'start_pruning_after_n_iterations', 'maximum_pruning_iterations',
# 'starting_neuron', 'prune_neurons_max', 'pruning_method']
has_attribute = lambda x: any([x in a for a in sys.argv])
if has_attribute('pruning-momentum'):
pruning_settings['pruning_momentum'] = vars(
args)['pruning_momentum']
if has_attribute('pruning-method'):
pruning_settings['method'] = vars(args)['pruning_method']
pruning_parameters_list = prepare_pruning_list(
pruning_settings,
model,
model_name=args.model,
pruning_mask_from=args.pruning_mask_from,
name=args.name)
print("Total pruning layers:", len(pruning_parameters_list))
folder_to_write = "%s" % log_save_folder + "/"
log_folder = folder_to_write
pruning_engine = pytorch_pruning(pruning_parameters_list,
pruning_settings=pruning_settings,
log_folder=log_folder)
pruning_engine.connect_tensorboard(train_writer)
pruning_engine.dataset = args.dataset
pruning_engine.model = args.model
pruning_engine.pruning_mask_from = args.pruning_mask_from
pruning_engine.load_mask()
gates_to_params = connect_gates_with_parameters_for_flops(
args.model, parameters_for_update_named)
pruning_engine.gates_to_params = gates_to_params
###=======================end for pruning
# loading model file
if (len(args.load_model) > 0) and (not args.dynamic_network):
if os.path.isfile(args.load_model):
load_model_pytorch(model, args.load_model, args.model)
else:
print("=> no checkpoint found at '{}'".format(args.load_model))
exit()
if args.tensorboard and 0:
if args.dataset == "CIFAR10":
dummy_input = torch.rand(1, 3, 32, 32).to(device)
elif args.dataset == "Imagenet":
dummy_input = torch.rand(1, 3, 224, 224).to(device)
train_writer.add_graph(model, dummy_input)
for epoch in range(1, args.epochs + 1):
if args.distributed:
train_sampler.set_epoch(epoch)
adjust_learning_rate(args, optimizer, epoch, args.zero_lr_for_epochs,
train_writer)
if not args.run_test and not args.get_inference_time:
train(args,
model,
device,
train_loader,
optimizer,
epoch,
criterion,
train_writer=train_writer,
pruning_engine=pruning_engine)
if args.pruning:
# skip validation error calculation and model saving
if pruning_engine.method == 50: continue
# evaluate on validation set
prec1, _ = validate(args,
test_loader,
model,
device,
criterion,
epoch,
train_writer=train_writer)
# remember best prec@1 and save checkpoint
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
model_save_path = "%s/models/checkpoint.weights" % (log_save_folder)
model_state_dict = model.state_dict()
if args.save_models:
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model_state_dict,
'best_prec1': best_prec1,
},
is_best,
filename=model_save_path)
train_loader = cfg.dataset_loader(root=cfg.mnist_dir,
train=True,
data_preprocess=train_data_preprocess)
test_loader = cfg.dataset_loader(root=cfg.mnist_dir,
train=False,
data_preprocess=valid_data_preprocess)
# test_loader = cfg.dataset_loader(root=cfg.cat_dog_test, train=False, shuffle=False,
# data_preprocess=valid_data_preprocess)
# ---------------构建网络、定义损失函数、优化器--------------------------
# 构建网络结构
# net = resnet()
# net = AlexNet(num_classes=cfg.num_classes)
# net = resnet50()
# net = resnet18()
net = LeNet(num_classes=cfg.num_classes)
# net = VGG('VGG11', num_classes=10, dataset='cifar-10')
# 重写网络最后一层
# fc_in_features = net.fc.in_features # 网络最后一层的输入通道
# net.fc = nn.Linear(in_features=fc_in_features, out_features=cfg.num_classes)
# 将网络结构、损失函数放置在GPU上；配置优化器
net = net.to(cfg.device)
# net = nn.DataParallel(net, device_ids=[0, 1])
# criterion = nn.BCEWithLogitsLoss().cuda(device=cfg.device)
criterion = nn.CrossEntropyLoss().cuda(device=cfg.device)
# 常规优化器：随机梯度下降和Adam
#optimizer = optim.SGD(params=net.parameters(), lr=cfg.learning_rate,
# weight_decay=cfg.weight_decay, momentum=cfg.momentum)
optimizer = optim.Adam(params=net.parameters(),
lr=cfg.learning_rate,
transform=transforms.Compose(
[transforms.Resize((32, 32)),
transforms.ToTensor()]))
data_test = MNIST('./data/mnist',
train=False,
download=True,
transform=transforms.Compose(
[transforms.Resize((32, 32)),
transforms.ToTensor()]))
data_train_loader = DataLoader(data_train,
batch_size=256,
shuffle=True,
num_workers=8)
data_test_loader = DataLoader(data_test, batch_size=1024, num_workers=8)
net = LeNet()
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(), lr=2e-3)
cur_batch_win = None
cur_batch_win_opts = {
'title': 'Epoch Loss Trace',
'xlabel': 'Batch Number',
'ylabel': 'Loss',
'width': 1200,
'height': 600,
}
def train(epoch):
global cur_batch_win
# =============================================================================
# model = CNN(num_classes,
# compile_model=False)
# model.load_weights('checkpoints/cnn_best_weights/' + \
# 'epoch.152_val_loss.0.600990.h5')
# model.compile(loss="categorical_crossentropy",
# optimizer='adam',
# metrics=["accuracy"])
# loss,acc = model.evaluate_generator(test, steps=test_steps, verbose=2)
# print("Restored model, accuracy: {:5.2f}%".format(100*acc))
# =============================================================================
model = LeNet(num_classes,
lr=3e-4,
optimizer_type="adam",
reg=1e-3)
#model = Simple_NN(num_classes, lr=.01, reg=0.0)
start = time.time()
model.fit(train,
epochs=10,
steps_per_epoch=steps_per_epoch,
validation_data=valid,
validation_steps=validation_steps,
verbose=1)
end = time.time()
print(end - start)
# Experiments
experiment_notes = "Experiment 7: Modified LeNet
def create_net(task_config, projector_config):
model = task_config['model']
output_size = task_config['output_size']
context_size = projector_config.get('context_size', None)
block_in = projector_config.get('block_in', None)
block_out = projector_config.get('block_out', None)
print("Creating", model)
if context_size > 0:
hyper = True
hyperlayers = ['conv2']
else:
hyper = False
hyperlayers = []
if model == 'deepbind':
num_filters = task_config.get('num_filters', 16)
hidden_dim = task_config.get('hidden_dim', 32)
net = DeepBind(context_size,
block_in,
block_out, {'context_size': 100},
hyper,
filters=num_filters,
hidden_units=hidden_dim)
elif model == 'linear_model':
input_size = task_config.get('input_dim', 20)
net = LinearModel(context_size,
block_in,
block_out,
input_dim=input_size,
output_dim=output_size,
hyper=hyper)
elif model == 'lstm_language_model':
layer_size = task_config.get('layer_size', 32)
net = LSTMLanguageModel(context_size,
block_in,
block_out,
ninp=layer_size,
nhid=layer_size,
hyper=hyper)
elif model == 'wide_resnet':
N = task_config.get('N', 6)
k = task_config.get('k', 1)
num_classes = output_size
net = WideResnet(context_size, block_in, block_out, N, k, num_classes,
hyper)
elif model == 'lenet':
if context_size > 0:
hyperlayers = ['conv2', 'fc1', 'fc2']
net = LeNet(context_size, block_in, block_out, hyperlayers)
else:
print("Please select a valid model kind")
sys.exit(0)
return net
cnn = PreActResNet34(channels=num_channels, num_classes=num_classes)
elif args.model == 'resnet50':
cnn = PreActResNet50(channels=num_channels, num_classes=num_classes)
elif args.model == 'resnet101':
cnn = PreActResNet101(channels=num_channels, num_classes=num_classes)
elif args.model == 'resnet152':
cnn = PreActResNet152(channels=num_channels, num_classes=num_classes)
elif args.model == 'vgg':
cnn = VGG(depth=16, num_classes=num_classes, channels=num_channels)
elif args.model == 'wideresnet':
if args.dataset == 'svhn':
cnn = Wide_ResNet(depth=16, num_classes=num_classes, widen_factor=8, dropout_rate=args.dropout_rate)
else:
cnn = Wide_ResNet(depth=28, num_classes=num_classes, widen_factor=10, dropout_rate=args.dropout_rate)
elif args.model == 'lenet':
cnn = LeNet()
elif args.model == 'magnetlenet':
cnn = MagnetLeNet(num_classes)
elif args.model == 'magnetfashion':
cnn = FashionSimpleNet(num_classes)
if args.spladvise or args.magnet:
if args.shallow_model == 'resnet18':
num_classes = 2
shallow_net = PreActResNet18(channels=num_channels, num_classes=num_classes)
elif args.shallow_model == 'resnet34':
num_classes = 2
shallow_net = PreActResNet34(channels=num_channels, num_classes=num_classes)
elif args.shallow_model == 'resnet50':
num_classes = 2
shallow_net = PreActResNet50(channels=num_channels, num_classes=num_classes)
def main():
##Defining the parser
parser = argparse.ArgumentParser(description="Tensorflow Trainer")
parser.add_argument("--resume", type=str, help="resume from checkpoint: ./path/model.ckpt")
parser.add_argument("--start_iteration", default=0, type=int, help="starting iterations")
parser.add_argument("--stop_iteration", default=1000, type=int, help="starting iterations")
parser.add_argument("--epochs", default=100, type=int, help="total epochs")
parser.add_argument("--gpu", default=0, type=int, help="GPU index")
parser.add_argument("--arch", default="yae", type=str, help="architecture to use for training: yae, cae")
parser.add_argument("--implicit_units", default=32, type=int, help="implicit units in the code")
parser.add_argument("--wdecay", default=0.0, type=float, help="Define the weight decay")
parser.add_argument("--lrate", default= 0.0001, type=float, help="Learning rate for Adam")
parser.add_argument("--mini_batch", default=128, type=int, help="mini-batch size")
parser.add_argument("--lambda_e", default=1.0, type=float, help="Explicit loss mixing coefficient")
parser.add_argument("--lambda_i", default=1.0, type=float, help="Implicit loss mixing coefficient")
parser.add_argument("--beta", default=1.0, type=float, help="beta hyperparameter used in beta-VAE")
args = parser.parse_args()
#Set the GPU
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"]=str(args.gpu)
import tensorflow as tf
#Set global hyperparameters
learning_rate = args.lrate
mini_batch_size = args.mini_batch
tot_epochs = args.epochs
tot_labels = 10
dataset_size = 60000
tot_iterations = int((dataset_size / mini_batch_size) * tot_epochs)
save_every_iteration = tot_iterations-1
print_every_iteration = 25
features_path = "./datasets/mnist/train/features.npy"
labels_path = "./datasets/mnist/train/labels.npy"
dataset_train = Dataset()
dataset_train.load(features_path, labels_path, tot_labels, normalizer=255.0, shuffle=True, verbose=True)
##Set local hyperparameters
if(args.arch=="yae"):
simulation_path = "./results/yae" + "_ep" + str(args.epochs) +"_lambdae" + str(args.lambda_e) + "_lambdai" + str(args.lambda_i)
from models.yae import Autoencoder
my_net = Autoencoder(batch_size=mini_batch_size, channels=1, conv_filters=8, style_size=args.implicit_units, content_size=tot_labels, ksize=(3,3), start_iteration=args.start_iteration, dir_header=simulation_path)
elif(args.arch=="cae"):
simulation_path = "./results/cae" + "_ep" + str(args.epochs) + "_wdecay" + str(args.wdecay) + "_units" + str(args.implicit_units)
from models.cae import Autoencoder
my_net = Autoencoder(batch_size=mini_batch_size, channels=1, conv_filters=8, style_size=args.implicit_units, content_size=tot_labels, ksize=(3,3), start_iteration=args.start_iteration, dir_header=simulation_path)
elif(args.arch=="cvae"):
from models.cvae import Autoencoder
simulation_path = "./results/cvae" + "_ep" + str(args.epochs) + "_wdecay" + str(args.wdecay) + "_units" + str(args.implicit_units) + "_beta" + str(args.beta)
my_net = Autoencoder(batch_size=mini_batch_size, channels=1, conv_filters=8, style_size=args.implicit_units, content_size=tot_labels, ksize=(3,3), start_iteration=args.start_iteration, dir_header=simulation_path, beta=args.beta)
elif(args.arch=="aae"):
from models.aae import Autoencoder
simulation_path = "./results/aae" + "_ep" + str(args.epochs) + "_wdecay" + str(args.wdecay) + "_units" + str(args.implicit_units)
my_net = Autoencoder(batch_size=mini_batch_size, channels=1, conv_filters=4, style_size=args.implicit_units, content_size=tot_labels, ksize=(3,3), start_iteration=args.start_iteration, dir_header=simulation_path)
elif(args.arch=="lenet"):
simulation_path = "./results/lenet" + "_ep" + str(args.epochs) + "_wdecay" + str(args.wdecay) + "_lr" + str(args.lrate)
from models.lenet import LeNet
my_net = LeNet(batch_size=mini_batch_size, channels=1, conv_filters=8, tot_labels=10, ksize=(5,5), start_iteration=args.start_iteration, dir_header=simulation_path)
else:
raise ValueError("[ERROR] The architecture '" + args.arch + "' does not exist!")
#Init the session
sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))
my_net.init_summary(sess)
if args.resume:
print("[INFO] Resuming from checkpoint: " + str(args.resume))
my_net.load(sess, args.resume)
else:
sess.run(tf.global_variables_initializer()) #WARNING: do not call it when the load() method is used
print("[INFO] Starting training...")
for iteration in range(args.start_iteration, tot_iterations):
if(args.arch=="yae"):
input_features, input_labels = dataset_train.return_features_labels(mini_batch_size, onehot=False)
local_loss = my_net.train(sess, input_features, input_labels,
learning_rate, args.lambda_e, args.lambda_i, iteration, print_every_iteration)
elif(args.arch=="cae"):
input_features, input_labels = dataset_train.return_features_labels(mini_batch_size, onehot=False)
local_loss = my_net.train(sess, input_features, input_labels,
learning_rate, iteration, print_every_iteration)
elif(args.arch=="cvae" or args.arch=="aae"):
input_features, input_labels = dataset_train.return_features_labels(mini_batch_size, onehot=False)
local_loss = my_net.train(sess, input_features, input_labels,
learning_rate, iteration, print_every_iteration)
elif(args.arch=="lenet"):
input_features, input_labels = dataset_train.return_features_labels(mini_batch_size, onehot=False)
local_loss = my_net.train(sess, input_features, input_labels,
learning_rate, iteration, print_every_iteration)
else:
raise ValueError("[ERROR] The architecture '" + args.arch + "' does not exist!")
if(iteration % print_every_iteration == 0):
print("Iteration: " + str(iteration) + "/" + str(tot_iterations) + " [" + str(round((iteration/float(tot_iterations))*100.0, 1)) + "%]")
print("Loss: " + str(local_loss))
print("")
if(iteration % save_every_iteration == 0 and iteration!=0):
my_net.save(sess, verbose=True)
def main(args):
check_path(args)
# CIFAR-10的全部类别，一共10类
classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse',
'ship', 'truck')
# 数据集
data_builder = DataBuilder(args)
dataSet = DataSet(data_builder.train_builder(),
data_builder.test_builder(), classes)
# 选择模型
if args.lenet:
net = LeNet()
model_name = args.name_le
elif args.vgg:
net = Vgg16_Net()
model_name = args.name_vgg
elif args.resnet18:
net = ResNet18()
model_name = args.name_res18
elif args.resnet34:
net = ResNet34()
model_name = args.name_res34
elif args.resnet50:
net = ResNet50()
model_name = args.name_res50
elif args.resnet101:
net = ResNet101()
model_name = args.name_res101
elif args.resnet152:
net = ResNet152()
model_name = args.name_res152
# 交叉熵损失函数
criterion = nn.CrossEntropyLoss()
# SGD优化器
optimizer = optim.SGD(net.parameters(),
lr=args.learning_rate,
momentum=args.sgd_momentum,
weight_decay=args.weight_decay)
# 余弦退火调整学习率
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=150)
# 模型的参数保存路径
model_path = os.path.join(args.model_path, model_name)
# 启动训练
if args.do_train:
print("Training...")
trainer = Trainer(net, criterion, optimizer, scheduler,
dataSet.train_loader, dataSet.test_loader,
model_path, args)
trainer.train(epochs=args.epoch)
# t.save(net.state_dict(), model_path)
# 启动测试，如果--do_train也出现，则用刚刚训练的模型进行测试
# 否则就使用已保存的模型进行测试
if args.do_eval:
if not args.do_train and not os.path.exists(model_path):
print(
"Sorry, there's no saved model yet, you need to train first.")
return
# --do_eval
if not args.do_train:
checkpoint = t.load(model_path)
net.load_state_dict(checkpoint['net'])
accuracy = checkpoint['acc']
epoch = checkpoint['epoch']
print("Using saved model, accuracy : %f epoch: %d" %
(accuracy, epoch))
tester = Tester(dataSet.test_loader, net, args)
tester.test()
if args.show_model:
if not os.path.exists(model_path):
print(
"Sorry, there's no saved model yet, you need to train first.")
return
show_model(args)
if args.do_predict:
device = t.device("cuda" if t.cuda.is_available() else "cpu")
checkpoint = t.load(model_path, map_location=device)
net.load_state_dict(checkpoint['net'])
predictor = Predictor(net, classes)
img_path = 'test'
img_name = [os.path.join(img_path, x) for x in os.listdir(img_path)]
for img in img_name:
predictor.predict(img)
def run_training():
# A simple transform which we will apply to the MNIST images
simple_transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize((0.5, ), (1.0, ))])
train_set = datasets.MNIST(root=config.DATA_DIR,
train=True,
transform=simple_transform,
download=True)
test_set = datasets.MNIST(root=config.DATA_DIR,
train=False,
transform=simple_transform,
download=True)
# train_dataset = dataset.ClassificationDataset(image_paths=train_imgs, targets=train_targets,
# resize=(config.IMAGE_HEIGHT, config.IMAGE_WIDTH))
train_loader = torch.utils.data.DataLoader(dataset=train_set,
batch_size=config.BATCH_SIZE,
num_workers=config.NUM_WORKERS,
shuffle=True)
# test_dataset = dataset.ClassificationDataset(image_paths=test_imgs, targets=test_targets,
# resize=(config.IMAGE_HEIGHT, config.IMAGE_WIDTH))
test_loader = torch.utils.data.DataLoader(dataset=test_set,
batch_size=config.BATCH_SIZE,
num_workers=config.NUM_WORKERS,
shuffle=False)
model = LeNet()
if torch.cuda.is_available():
print('GPU available... using GPU')
torch.cuda.manual_seed_all(42)
# device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu')
model.to(config.DEVICE)
optimizer = torch.optim.Adam(model.parameters(), lr=3e-4)
# scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
# optimizer, factor=0.8, patience=5, verbose=True
# )
criterion = nn.CrossEntropyLoss()
train_loss_data = []
test_loss_data = []
for epoch in range(config.EPOCHS):
# training
train_loss = engine.train_fn(model,
train_loader,
optimizer,
criterion,
save_model=True)
# validation
eval_preds, eval_loss = engine.eval_fn(model, test_loader, criterion)
print(
f"Epoch {epoch} => Training Loss: {train_loss}, Val Loss: {eval_loss}"
)
train_loss_data.append(train_loss)
test_loss_data.append(eval_loss)
# print(train_dataset[0])
plot_loss(train_loss_data, test_loss_data, plot_path=config.PLOT_PATH)
print("done")
checkpoint_dir='./checkpoints'
# Placeholders for the input and labels
X_source = tf.placeholder(tf.float32, shape=[batch_size, xydim, xydim, cdim])
X_target = tf.placeholder(tf.float32, shape=[batch_size, xydim, xydim, cdim])
Y_source = tf.placeholder(tf.float32, shape=[batch_size, n_classes])
Y_target = tf.placeholder(tf.float32, shape=[batch_size, n_classes])
# Placeholders for lambdas
lambda_r = tf.placeholder(tf.float32)
lambda_s = tf.placeholder(tf.float32)
lambda_p = tf.placeholder(tf.float32)
opt_lr = tf.placeholder(tf.float32)
# Intialize source/residual streams
lenet_model = LeNet(only_features=True, feature_dim=feature_dim).build(X_source)
resid_model = ResidualNet().build(lenet_model)
''' Weights for the classifier and discriminator '''
''' Weights for the classifier and discriminator '''
wC = {
"classifier": tf.get_variable('classifier', shape=[500, n_classes], initializer=tf.contrib.layers.xavier_initializer())
}
wD = {
"fc1": tf.get_variable('fc1_disc', shape=[n_classes, 500], initializer=tf.contrib.layers.xavier_initializer()),
"fc2": tf.get_variable('fc2_disc', shape=[500, 500], initializer=tf.contrib.layers.xavier_initializer()),
"logits": tf.get_variable('logits_disc', shape=[500, 2], initializer=tf.contrib.layers.xavier_initializer())
}
import os
from config import args
import tensorflow as tf
if args.model == 'lenet':
if args.run_name == 'none':
from models.lenet import LeNet as Model
elif args.run_name == 'dropout':
from models.lenet_dropout import LeNet as Model
elif args.run_name == 'dropconnect':
from models.lenet_dropconnect import LeNet as Model
elif args.model == 'fcnet':
if args.run_name == "dropout":
from models.FCNet_dropout import FCNet as Model
elif args.run_name == "dropconnect":
from models.FCNet_dropconnect import FCNet as Model
elif args.run_name == "none":
from models.FCNet import FCNet as Model
if __name__ == '__main__':
os.environ['CUDA_VISIBLE_DEVICES'] = '1'
model = Model(tf.Session(), args)
if not os.path.exists(args.modeldir + args.run_name):
os.makedirs(args.modeldir + args.run_name)
if not os.path.exists(args.logdir + args.run_name):
os.makedirs(args.logdir + args.run_name)
if args.mode == 'train':
model.train()
elif args.mode == 'test':
model.test(step_num=args.reload_step)
def main_attack(args):
# Use CUDA
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
use_cuda = torch.cuda.is_available()
device = 'cuda' if use_cuda else 'cpu'
# Random seed
random.seed(args.seed)
torch.manual_seed(args.seed)
if use_cuda:
torch.cuda.manual_seed_all(args.seed)
# Data
print(f'==> Preparing dataset {args.dataset}')
if args.dataset in ['cifar10', 'cifar100']:
detph = 28
widen_factor = 10
dropout = 0.3
transform_test = transforms.Compose([transforms.ToTensor()])
elif args.dataset == 'tiny-imagenet':
transform_test = transforms.Compose([transforms.ToTensor()])
elif args.dataset == 'imagenet':
transform_test = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor()
])
elif args.dataset == 'mnist':
transform_test = transforms.Compose([transforms.ToTensor()])
elif args.dataset == 'SVHN':
detph = 16
widen_factor = 4
dropout = 0.4
transform_train = transforms.Compose(
[transforms.RandomCrop(32, padding=4),
transforms.ToTensor()])
transform_test = transforms.Compose([transforms.ToTensor()])
print(f'Running on dataset {args.dataset}')
if args.dataset in ['cifar10', 'cifar100', 'mnist']:
if args.dataset == 'cifar10':
dataloader = datasets.CIFAR10
num_classes = 10
elif args.dataset == 'cifar100':
dataloader = datasets.CIFAR100
num_classes = 100
elif args.dataset == 'mnist':
dataloader = datasets.MNIST
num_classes = 10
testset = dataloader(root='.data',
train=False,
download=False,
transform=transform_test)
elif args.dataset == 'tiny-imagenet':
testset = datasets.ImageFolder('tiny-imagenet-200/val',
transform=transform_test)
num_classes = 200
elif args.dataset == 'imagenet':
testset = datasets.ImageFolder('imagenet/val',
transform=transform_test)
num_classes = 1000
elif args.dataset == 'SVHN':
testset = datasets.SVHN('data',
split='test',
transform=transform_test,
download=True)
num_classes = 10
testloader = data.DataLoader(testset,
batch_size=args.test_batch,
shuffle=False,
num_workers=args.workers)
# Model
if args.arch == 'vgg16':
model = VGG16(dataset=args.dataset,
num_classes=num_classes,
kernels1=args.kernels1,
kernels2=args.kernels2,
kernels3=args.kernels3,
orientations=args.orientations,
learn_theta=args.learn_theta)
elif args.arch == 'resnet18':
model = ResNet18(dataset=args.dataset,
num_classes=num_classes,
kernels1=args.kernels1,
kernels2=args.kernels2,
kernels3=args.kernels3,
orientations=args.orientations,
learn_theta=args.learn_theta)
elif args.arch == 'madry':
model = MadryNet(kernels1=args.kernels1,
kernels2=args.kernels2,
kernels3=args.kernels3,
orientations=args.orientations,
learn_theta=args.learn_theta)
elif args.arch == 'lenet':
model = LeNet(kernels1=args.kernels1,
kernels2=args.kernels2,
kernels3=args.kernels3,
orientations=args.orientations,
learn_theta=args.learn_theta)
elif args.arch == 'alexnet':
model = AlexNet(dataset=args.dataset,
num_classes=num_classes,
kernels1=args.kernels1,
kernels2=args.kernels2,
kernels3=args.kernels3,
orientations=args.orientations,
learn_theta=args.learn_theta)
elif args.arch == 'wide-resnet':
model = Wide_ResNet(dataset=args.dataset,
num_classes=num_classes,
kernels1=args.kernels1,
kernels2=args.kernels2,
kernels3=args.kernels3,
orientations=args.orientations,
learn_theta=args.learn_theta,
finetune=False,
depth=detph,
widen_factor=widen_factor,
dropout_rate=dropout,
use_7x7=args.use_7x7)
print('Model:')
print(model)
if use_cuda:
model = torch.nn.DataParallel(model).cuda()
# Compute number of parameters and print them
param_num = sum(p.numel() for p in model.parameters() if p.requires_grad)
param_txt = f' Total trainable params: {param_num:d}'
print(param_txt)
criterion = nn.CrossEntropyLoss()
# Resume
# Load checkpoint.
print('==> Resuming from checkpoint...')
checkpoint_filename = osp.join(args.checkpoint, 'model_best.pth.tar')
assert osp.isfile(
checkpoint_filename), 'Error: no checkpoint directory found!'
checkpoint = torch.load(checkpoint_filename)
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch,
use_cuda)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
print(f'Running {args.attack} attack!')
if args.attack == 'cw':
c_vals = torch.logspace(start=-2, end=2, steps=9)
for c in c_vals:
print(f'Running attack with c = {c:5.3f}')
attack_cw(model, testloader, device=device, c=c)
print('\n')
else:
if args.dataset == 'mnist':
epsilons = [.1, .2, .3, .4]
else:
epsilons = [2 / 255, 8 / 255, 16 / 255, .1]
print(f'Epsilons are: {epsilons}')
minimum = 0.
maximum = 1.
print(f'Images maxima: {maximum} -- minima: {minimum}')
df = {
'epsilons': [
0.,
],
'test_set_accs': [
test_acc,
],
'flip_rates': [
0.,
],
}
for eps in epsilons:
print(f'Running attack with epsilon = {eps:5.3f}')
acc_test_set, flip_rate = attack_pgd(model,
testloader,
device=device,
minimum=minimum,
maximum=maximum,
eps=eps)
df['epsilons'].append(eps)
df['test_set_accs'].append(acc_test_set)
df['flip_rates'].append(flip_rate)
print('\n')
df = pd.DataFrame.from_dict(df)
print('Overall results: \n', df)
filename = osp.join(args.checkpoint, 'attack_results.csv')
df.to_csv(filename, index=False)
test_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../Embedding_data',
train=False,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])),
batch_size=args.batch_size,
shuffle=True,
**kwargs)
# Model
print('==> Building model..')
net = LeNet(5)
net = net.to(device)
if device == 'cuda':
net = torch.nn.DataParallel(net)
cudnn.benchmark = True
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isdir(
'checkpoint'), 'Error: no checkpoint directory found!'
checkpoint = torch.load('./checkpoint/classification_ckpt.t7')
net.load_state_dict(checkpoint['net'])
best_acc = checkpoint['acc']
start_epoch = checkpoint['epoch']
print('the current best acc is %.3f on epoch %d' %
import init_paths
from utils.helpers import read_json_config, get_args
from data_loaders.mnist_loader import MNISTDataLoader
from models.lenet import LeNet
from trainers.mnist_trainer import MNISTTrainer
try:
args = get_args()
config = read_json_config(args.config)
except:
print('Error on reading config file.')
print(config.experiment_name)
data_loader = MNISTDataLoader(config)
model = LeNet()
trainer = MNISTTrainer(model, config, data_loader)
trainer.train()
# 配置实例化
# cfg = Cifar10Config()
cfg = MnistConfig()
# 数据预处理
test_data_preprocess = transforms.Compose([transforms.ToTensor()])
# 获取测试集的加载器
test_loader = cfg.dataset_loader(root=cfg.mnist_dir,
train=False,
shuffle=False,
data_preprocess=test_data_preprocess)
# ---------------构建网络、定义损失函数、优化器--------------------------
# net = resnet18()
# net = resnet_v2.resnet18(num_classes=cfg.num_classes, type_dataset='cifar-10')
# net = vggnet.VGG(vgg_name='VGG11', num_classes=10, dataset='cifar-10')
net = LeNet(num_classes=10)
# 重写网络最后一层
# fc_in_features = net.fc.in_features # 网络最后一层的输入通道
# net.fc = nn.Linear(in_features=fc_in_features, out_features=cfg.num_classes)
# 加载模型权重、将网络放入GPU
if os.path.exists(cfg.checkpoints):
net.load_state_dict(torch.load(cfg.checkpoints))
print('load model argument...')
# -------------进行测试-----------------
print('进行测试.....')
# 测试函数
print('test stage...\n')
# 将网络结构调成验证模式、定义准确率、标签列表和预测列表
net.eval()
def main():
global best_acc
start_epoch = args.start_epoch # start from epoch 0 or last checkpoint epoch
if not os.path.isdir(args.checkpoint):
mkdir_p(args.checkpoint)
print_training_params(args=args, txt_file_path=txt_file_path)
# Data
print(f'==> Preparing dataset {args.dataset}')
if args.dataset in ['cifar10', 'cifar100']:
detph = 28
widen_factor = 10
dropout = 0.3
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor()
])
transform_test = transforms.Compose([
transforms.ToTensor()
])
elif args.dataset == 'tiny-imagenet':
transform_train = transforms.Compose([
transforms.ToTensor()
])
transform_test = transforms.Compose([
transforms.ToTensor()
])
elif args.dataset == 'imagenet':
transform_train = transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor()
])
transform_test = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor()
])
elif args.dataset == 'mnist':
transform_train = transforms.Compose([
transforms.ToTensor()
])
transform_test = transforms.Compose([
transforms.ToTensor()
])
elif args.dataset == 'SVHN':
detph = 16
widen_factor = 4
dropout = 0.4
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.ToTensor()
])
transform_test = transforms.Compose([
transforms.ToTensor()
])
print(f'Running on dataset {args.dataset}')
if args.dataset in ['cifar10', 'cifar100', 'mnist']:
if args.dataset == 'cifar10':
dataloader = datasets.CIFAR10
num_classes = 10
elif args.dataset == 'cifar100':
dataloader = datasets.CIFAR100
num_classes = 100
elif args.dataset == 'mnist':
dataloader = datasets.MNIST
num_classes = 10
trainset = dataloader(root='.data', train=True, download=True, transform=transform_train)
testset = dataloader(root='.data', train=False, download=False, transform=transform_test)
elif args.dataset == 'imagenet':
trainset = datasets.ImageFolder('imagenet/train', transform=transform_train)
testset = datasets.ImageFolder('imagenet/val', transform=transform_test)
num_classes = 1000
elif args.dataset == 'SVHN':
trainset = datasets.SVHN('data', split='train', transform=transform_train, download=True)
testset = datasets.SVHN('data', split='test', transform=transform_test, download=True)
num_classes = 10
trainloader = data.DataLoader(trainset, batch_size=args.train_batch, shuffle=True, num_workers=args.workers)
testloader = data.DataLoader(testset, batch_size=args.test_batch, shuffle=False, num_workers=args.workers)
# Model
print("==> creating model '{}'".format(args.arch))
if args.arch == 'vgg16':
model = VGG16(
dataset=args.dataset,
num_classes=num_classes,
kernels1=args.kernels1,
kernels2=args.kernels2,
kernels3=args.kernels3,
orientations=args.orientations,
learn_theta=args.learn_theta,
finetune=args.finetune
)
elif args.arch == 'resnet18':
model = ResNet18(
dataset=args.dataset,
num_classes=num_classes,
kernels1=args.kernels1,
kernels2=args.kernels2,
kernels3=args.kernels3,
orientations=args.orientations,
learn_theta=args.learn_theta,
finetune=args.finetune
)
elif args.arch == 'madry':
model = MadryNet(
kernels1=args.kernels1,
kernels2=args.kernels2,
kernels3=args.kernels3,
orientations=args.orientations,
learn_theta=args.learn_theta
)
elif args.arch == 'lenet':
model = LeNet(
kernels1=args.kernels1,
kernels2=args.kernels2,
kernels3=args.kernels3,
orientations=args.orientations,
learn_theta=args.learn_theta
)
elif args.arch == 'alexnet':
model = AlexNet(
dataset=args.dataset,
num_classes=num_classes,
kernels1=args.kernels1,
kernels2=args.kernels2,
kernels3=args.kernels3,
orientations=args.orientations,
learn_theta=args.learn_theta
)
elif args.arch == 'wide-resnet':
model = Wide_ResNet(
dataset=args.dataset,
num_classes=num_classes,
kernels1=args.kernels1,
kernels2=args.kernels2,
kernels3=args.kernels3,
orientations=args.orientations,
learn_theta=args.learn_theta,
finetune=args.finetune,
depth=detph,
widen_factor=widen_factor,
dropout_rate=dropout,
use_7x7=args.use_7x7
)
print('Model:')
print(model)
print_to_log(text=repr(model), txt_file_path=txt_file_path)
if device == 'cuda':
model = torch.nn.DataParallel(model).to(device)
# Compute number of parameters and print them
param_num = sum(p.numel() for p in model.parameters() if p.requires_grad)
param_txt = f' Total trainable params: {param_num:d}'
print_to_log(text=param_txt, txt_file_path=txt_file_path)
print(param_txt)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay)
# Resume
title = f'{args.dataset}-' + args.arch
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint...')
assert osp.isfile(args.resume), 'Error: no checkpoint directory found!'
args.checkpoint = osp.dirname(args.resume)
checkpoint = torch.load(args.resume)
best_acc = checkpoint['best_acc']
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title, resume=True)
else:
logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch, device)
print(' Test Loss: %.8f, Test Acc: %.2f' % (test_loss, test_acc))
return
# Train and val
for epoch in range(start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, args.epochs, state['lr']))
train_loss, train_acc = train(
trainloader, model, criterion, optimizer, epoch, device, train_adv=args.train_adv, args=args)
test_loss, test_acc = test(
testloader, model, criterion, epoch, device)
# append logger file
logger.append([state['lr'], train_loss, test_loss, train_acc, test_acc])
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'acc': test_acc,
'best_acc': best_acc,
'optimizer' : optimizer.state_dict(),
}, is_best, checkpoint=args.checkpoint)
if args.kernels1 is not None:
plot_kernels(model, args.checkpoint, epoch, device)
logger.close()
logger.plot()
savefig(os.path.join(args.checkpoint, 'log.eps'))
print('Best acc:')
print(best_acc)
print('Training finished. Running attack')
main_attack(args)
print('Running SVD computation')
main_svs_computation(args)
test_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../Embedding_data',
train=False,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])),
batch_size=bs,
shuffle=True,
**kwargs)
# Model
print('==> Building model..')
device = 'cuda' if torch.cuda.is_available() else 'cpu'
net = LeNet(5)
net = net.to(device)
if device == 'cuda':
net = torch.nn.DataParallel(net)
cudnn.benchmark = True
# fileList=file_name('checkpoint','.t7')
fileList = ['checkpoint/Fisher2_ckpt.t7']
for ckpt_file in fileList:
print('==> Resuming from checkpoint ' + ckpt_file)
assert os.path.isdir(
'checkpoint'), 'Error: no checkpoint directory found!'
checkpoint = torch.load(ckpt_file)
net.load_state_dict(checkpoint['net'])
best_acc = checkpoint['acc']
net.eval()
def run_magnet_loss():
'''
Test function for the magnet loss
'''
m = 8
d = 8
k = 8
alpha = 1.0
batch_size = m * d
global plotter
plotter = VisdomLinePlotter(env_name=args.name)
trainloader, testloader, trainset, testset, n_train = load_dataset(args)
emb_dim = 2
n_epochs = 15
epoch_steps = len(trainloader)
n_steps = epoch_steps * 15
cluster_refresh_interval = epoch_steps
if args.mnist:
model = torch.nn.DataParallel(LeNet(emb_dim)).cuda()
if args.cifar10:
model = torch.nn.DataParallel(VGG(depth=16, num_classes=emb_dim))
print(model)
optimizer = optim.Adam(model.parameters(), lr=args.lr)
minibatch_magnet_loss = MagnetLoss()
images = getattr(trainset, 'train_data')
labels = getattr(trainset, 'train_labels')
# Get initial embedding
initial_reps = compute_reps(model, trainset, 400)
if args.cifar10:
labels = np.array(labels, dtype=np.float32)
# Create batcher
batch_builder = ClusterBatchBuilder(labels, k, m, d)
batch_builder.update_clusters(initial_reps)
batch_losses = []
batch_example_inds, batch_class_inds = batch_builder.gen_batch()
trainloader.sampler.batch_indices = batch_example_inds
_ = model.train()
losses = AverageMeter()
for i in tqdm(range(n_steps)):
for batch_idx, (img, target) in enumerate(trainloader):
img = Variable(img).cuda()
target = Variable(target).cuda()
optimizer.zero_grad()
output, features = model(img)
batch_loss, batch_example_losses = minibatch_magnet_loss(
output, batch_class_inds, m, d, alpha)
batch_loss.backward()
optimizer.step()
# Update loss index
batch_builder.update_losses(batch_example_inds, batch_example_losses)
batch_losses.append(batch_loss.data[0])
if not i % 1000:
print(i, batch_loss)
if not i % cluster_refresh_interval:
print("Refreshing clusters")
reps = compute_reps(model, trainset, 400)
batch_builder.update_clusters(reps)
if not i % 2000:
n_plot = 10000
plot_embedding(compute_reps(model, trainset, 400)[:n_plot],
labels[:n_plot],
name=i)
batch_example_inds, batch_class_inds = batch_builder.gen_batch()
trainloader.sampler.batch_indices = batch_example_inds
losses.update(batch_loss, 1)
# Log the training loss
if args.visdom:
plotter.plot('loss', 'train', i, losses.avg.data[0])
# Plot loss curve
plot_smooth(batch_losses, "batch-losses")
def main():
##Defining the parser
parser = argparse.ArgumentParser(description="Tensorflow Trainer")
parser.add_argument("--resume",
type=str,
help="the path to the saved network")
parser.add_argument(
"--path",
type=str,
help=
"the root folder of the experiment (it contains the logs and model)")
parser.add_argument("--gendata_path",
type=str,
help="the folder containing the artificial data")
parser.add_argument("--load", type=str, help="load data")
parser.add_argument(
"--type",
default="gendata",
type=str,
help="type of test: gendata (generate a dataset), loss (return losses)"
)
parser.add_argument("--arch",
default="yae",
type=str,
help="architecture to use for training: yae, cae")
parser.add_argument("--gpu", default=0, type=int, help="GPU index")
parser.add_argument("--lambda_e",
default=1.0,
type=float,
help="Explicit loss mixing coefficient")
parser.add_argument("--lambda_i",
default=1.0,
type=float,
help="Implicit loss mixing coefficient")
parser.add_argument("--implicit_units",
default=32,
type=int,
help="implicit units in the code")
parser.add_argument("--tot_samples",
default=16,
type=int,
help="the total sample generted by measure")
parser.add_argument("--batch",
default=20000,
type=int,
help="the batch feed to the LeNet classifier")
args = parser.parse_args()
#Set the GPU
GPU = args.gpu
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = str(GPU)
import tensorflow as tf
tot_labels = 10
mini_batch_size = 10000
dataset_test = Dataset()
features_path = "./datasets/mnist/test/features.npy"
labels_path = "./datasets/mnist/test/labels.npy"
dataset_test.load(features_path,
labels_path,
tot_labels,
normalizer=255.0,
shuffle=False,
verbose=True)
##Set the hyper-parameters based on the chosen dataset
if (args.arch == "yae"):
simulation_path = args.path
from models.yae import Autoencoder
my_net = Autoencoder(batch_size=mini_batch_size,
channels=1,
conv_filters=8,
style_size=args.implicit_units,
content_size=10,
ksize=(3, 3),
start_iteration=0,
dir_header=simulation_path)
elif (args.arch == "cae"):
simulation_path = args.path
from models.cae import Autoencoder
my_net = Autoencoder(batch_size=mini_batch_size,
channels=1,
conv_filters=8,
style_size=args.implicit_units,
content_size=10,
ksize=(3, 3),
start_iteration=0,
dir_header=simulation_path)
elif (args.arch == "cvae"):
simulation_path = args.path
from models.cvae import Autoencoder
my_net = Autoencoder(batch_size=mini_batch_size,
channels=1,
conv_filters=8,
style_size=args.implicit_units,
content_size=10,
ksize=(3, 3),
start_iteration=0,
dir_header=simulation_path)
elif (args.arch == "aae"):
simulation_path = args.path
from models.aae import Autoencoder
my_net = Autoencoder(batch_size=mini_batch_size,
channels=1,
conv_filters=4,
style_size=args.implicit_units,
content_size=10,
ksize=(3, 3),
start_iteration=0,
dir_header=simulation_path)
elif (args.arch == "lenet"):
simulation_path = args.path
from models.lenet import LeNet
my_net = LeNet(batch_size=args.batch,
channels=1,
conv_filters=8,
tot_labels=10,
ksize=(5, 5),
start_iteration=0,
dir_header=simulation_path)
else:
raise ValueError("[ERROR] The dataset does not exist!")
#Init the session
sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))
if args.resume:
print("[INFO] Resuming from checkpoint: " + str(args.resume))
my_net.load(sess, args.resume)
else:
raise ValueError(
"[ERROR] To test a model it is necessary to resume from checkpoint..."
)
if (args.arch == "yae" and args.type == "loss"):
if not os.path.exists(simulation_path + "/test_loss"):
os.makedirs(simulation_path + "/test_loss")
time_id = strftime("%H%M%S_%d%m%Y", gmtime())
test_loss_path = simulation_path + "/test_loss/" + time_id
os.makedirs(test_loss_path)
input_features, input_labels = dataset_test.return_features_labels(
mini_batch_size, onehot=False)
loss, loss_r, loss_c, acc_c, loss_e, loss_i = my_net.test(
sess, input_features, input_labels, args.lambda_e, args.lambda_i)
with open(test_loss_path + "/test_loss.csv", "w") as text_file:
header = "loss, loss_r, loss_c, acc_c, loss_e, loss_i"
body = str(loss) + "," + str(loss_r) + "," + str(
loss_c) + "," + str(acc_c) + "," + str(loss_e) + "," + str(
loss_i)
text_file.write(header + '\n' + body)
print("====================================")
print(header)
print(body)
print("====================================")
elif (args.arch == "cae" and args.type == "loss"):
if not os.path.exists(simulation_path + "/test_loss"):
os.makedirs(simulation_path + "/test_loss")
time_id = strftime("%H%M%S_%d%m%Y", gmtime())
test_loss_path = simulation_path + "/test_loss/" + time_id
os.makedirs(test_loss_path)
input_features, input_labels = dataset_test.return_features_labels(
mini_batch_size, onehot=False)
loss = my_net.test(sess, input_features, input_labels)
with open(test_loss_path + "/test_loss.csv", "w") as text_file:
header = "loss"
body = str(loss)
text_file.write(header + '\n' + body)
print("====================================")
print(header)
print(body)
print("====================================")
elif (args.arch == "cvae" and args.type == "loss"):
if not os.path.exists(simulation_path + "/test_loss"):
os.makedirs(simulation_path + "/test_loss")
time_id = strftime("%H%M%S_%d%m%Y", gmtime())
test_loss_path = simulation_path + "/test_loss/" + time_id
os.makedirs(test_loss_path)
input_features, input_labels = dataset_test.return_features_labels(
mini_batch_size, onehot=False)
loss = my_net.test(sess, input_features, input_labels)
with open(test_loss_path + "/test_loss.csv", "w") as text_file:
header = "loss_r"
body = str(loss)
text_file.write(header + '\n' + body)
print("====================================")
print(header)
print(body)
print("====================================")
elif ((args.arch == "cae" or args.arch == "cvae" or args.arch == "yae"
or args.arch == "aae") and args.type == "gendata"):
if not os.path.exists(simulation_path + "/gendata"):
os.makedirs(simulation_path + "/gendata")
time_id = strftime("%H%M%S_%d%m%Y", gmtime())
gendata_path = simulation_path + "/gendata/" + time_id
os.makedirs(gendata_path)
print("Starting gendata...")
input_features, _ = dataset_test.return_features_labels(10000,
onehot=False,
shuffle=False)
features_list = list()
labels_list = list()
for i in range(0, 10):
print("Input: " + str(i))
print("Input (shape): " + str(input_features.shape))
input_labels = np.ones(10000) * i
if (args.arch == "cae" or args.arch == "cvae"
or args.arch == "aae"):
output = my_net.forward_conditional(
sess, input_features,
input_labels) #size [10000, 32, 32, 1]
elif (args.arch == "yae"):
output = my_net.forward_conditional(sess, input_features,
input_labels,
args.lambda_e,
args.lambda_i)
print("Output (shape): " + str(output.shape))
output = (output * 255).astype(np.uint8)
features_list.append(output)
labels_list.append(input_labels)
print("Saving data...")
features_matrix = np.concatenate(features_list, axis=0)
print("Features (shape): " + str(features_matrix.shape))
np.save(gendata_path + "/features", features_matrix)
labels_matrix = np.concatenate(labels_list, axis=0)
print("Labels (shape): " + str(labels_matrix.shape))
np.save(gendata_path + "/labels", labels_matrix)
print("Done!")
elif (args.type == "metrics"):
print("Iterating the test set...")
original_data_matrix = np.load(features_path)
data_matrix = np.load(args.gendata_path + "/features.npy")
if not os.path.exists(simulation_path + "/sample"):
os.makedirs(simulation_path + "/sample")
if not os.path.exists(simulation_path + "/metrics"):
os.makedirs(simulation_path + "/metrics")
time_id = strftime("%H%M%S_%d%m%Y", gmtime())
sample_path = simulation_path + "/sample/" + time_id
metrics_path = simulation_path + "/metrics/" + time_id
os.makedirs(sample_path)
os.makedirs(metrics_path)
img_ssim_list = list()
img_mse_list = list()
for i in range(10000):
img_original = original_data_matrix[i, :, :, 0]
if (i < args.tot_samples):
cv2.imwrite(sample_path + "/" + str(i) + ".png", img_original)
for j in range(0, 10):
location = (j * 10000) + i
img_generated = data_matrix[location, :, :, 0]
(score, diff) = compare_ssim(img_original,
img_generated,
full=True)
mse = (img_original - img_generated)**2
img_ssim_list.append(score)
img_mse_list.append(mse)
if (i < args.tot_samples):
cv2.imwrite(
sample_path + "/" + str(i) + "_" + str(j) + ".png",
img_generated)
if (i % 1000 == 0):
print(str(i) + "/" + str(10000))
print("====================================")
print("INTERNAL SSIM")
print("Mean: \t" + str(np.mean(img_ssim_list)))
print("Std: \t" + str(np.std(img_ssim_list)))
print("------------------------------------")
print("INTERNAL MSE")
print("Mean: \t" + str(np.mean(img_mse_list)))
print("Std: \t" + str(np.std(img_mse_list)))
print("====================================")
with open(metrics_path + "/test_metrics.csv", "w") as text_file:
header = "SSIM, MSE"
body = str(np.mean(img_ssim_list)) + ',' + str(
np.mean(img_mse_list))
text_file.write(header + '\n' + body)
elif (args.arch == "lenet" and args.type == "accuracy"):
if not os.path.exists(simulation_path + "/test_gendata_accuracy"):
os.makedirs(simulation_path + "/test_gendata_accuracy")
time_id = strftime("%H%M%S_%d%m%Y", gmtime())
accuracy_path = simulation_path + "/test_gendata_accuracy/" + time_id
os.makedirs(accuracy_path)
dataset_test = Dataset()
features_path = args.gendata_path + "/features.npy"
labels_path = args.gendata_path + "/labels.npy"
dataset_test.load(features_path,
labels_path,
tot_labels,
normalizer=255.0,
shuffle=True,
verbose=True)
input_features, input_labels = dataset_test.return_features_labels(
args.batch, onehot=False)
print(input_features.shape)
output = my_net.test(sess, input_features, input_labels)
print("==============================")
print("Loss:\t\t" + str(output[0]))
print("Accuracy:\t" + str(output[1] * 100.0))
print("==============================")
with open(accuracy_path + "/test_gendata_accuracy.csv",
"w") as text_file:
header = "Loss, Accuracy"
body = str(output[0]) + ',' + str(output[1] * 100.0)
text_file.write(header + '\n' + body)
else:
raise ValueError(
"[ERROR] This test does not exists for the model and dataset.")
test_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../Embedding_data',
train=False,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])),
batch_size=args.batch_size,
shuffle=True,
**kwargs)
# Model
print('==> Building model..')
net = LeNet(args.partition_proportion)
net = net.to(device)
if device == 'cuda':
net = torch.nn.DataParallel(net)
cudnn.benchmark = True
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isdir(
'checkpoint'), 'Error: no checkpoint directory found!'
if os.path.isfile('./checkpoint/lifted_ckpt.t7'):
checkpoint = torch.load('./checkpoint/lifted_ckpt.t7')
best_acc = checkpoint['acc']
else:
checkpoint = torch.load('./checkpoint/classification_ckpt.t7')
def get_model(model_name):
if model_name == "lenet":
return LeNet()
