def get_net(name):
if name == 'densenet121':
net = densenet121()
elif name == 'densenet161':
net = densenet161()
elif name == 'densenet169':
net = densenet169()
elif name == 'googlenet':
net = googlenet()
elif name == 'inception_v3':
net = inception_v3()
elif name == 'mobilenet_v2':
net = mobilenet_v2()
elif name == 'resnet18':
net = resnet18()
elif name == 'resnet34':
net = resnet34()
elif name == 'resnet50':
net = resnet50()
elif name == 'resnet_orig':
net = resnet_orig()
elif name == 'vgg11_bn':
net = vgg11_bn()
elif name == 'vgg13_bn':
net = vgg13_bn()
elif name == 'vgg16_bn':
net = vgg16_bn()
elif name == 'vgg19_bn':
net = vgg19_bn()
else:
print(f'{name} not a valid model name')
sys.exit(0)
return net.to(device)
def init_net(self):
net_args = {
"pretrained": True,
"n_input_channels": len(self.kwargs["static"]["imagery_bands"])
}
# https://pytorch.org/docs/stable/torchvision/models.html
if self.kwargs["net"] == "resnet18":
self.model = resnet.resnet18(**net_args)
elif self.kwargs["net"] == "resnet34":
self.model = resnet.resnet34(**net_args)
elif self.kwargs["net"] == "resnet50":
self.model = resnet.resnet50(**net_args)
elif self.kwargs["net"] == "resnet101":
self.model = resnet.resnet101(**net_args)
elif self.kwargs["net"] == "resnet152":
self.model = resnet.resnet152(**net_args)
elif self.kwargs["net"] == "vgg11":
self.model = vgg.vgg11(**net_args)
elif self.kwargs["net"] == "vgg11_bn":
self.model = vgg.vgg11_bn(**net_args)
elif self.kwargs["net"] == "vgg13":
self.model = vgg.vgg13(**net_args)
elif self.kwargs["net"] == "vgg13_bn":
self.model = vgg.vgg13_bn(**net_args)
elif self.kwargs["net"] == "vgg16":
self.model = vgg.vgg16(**net_args)
elif self.kwargs["net"] == "vgg16_bn":
self.model = vgg.vgg16_bn(**net_args)
elif self.kwargs["net"] == "vgg19":
self.model = vgg.vgg19(**net_args)
elif self.kwargs["net"] == "vgg19_bn":
self.model = vgg.vgg19_bn(**net_args)
else:
raise ValueError("Invalid network specified: {}".format(
self.kwargs["net"]))
# run type: 1 = fine tune, 2 = fixed feature extractor
# - replace run type option with "# of layers to fine tune"
if self.kwargs["run_type"] == 2:
layer_count = len(list(self.model.parameters()))
for layer, param in enumerate(self.model.parameters()):
if layer <= layer_count - 5:
param.requires_grad = False
# Parameters of newly constructed modules have requires_grad=True by default
# get existing number for input features
# set new number for output features to number of categories being classified
# see: https://pytorch.org/tutorials/beginner/finetuning_torchvision_models_tutorial.html
if "resnet" in self.kwargs["net"]:
num_ftrs = self.model.fc.in_features
self.model.fc = nn.Linear(num_ftrs, self.ncats)
elif "vgg" in self.kwargs["net"]:
num_ftrs = self.model.classifier[6].in_features
self.model.classifier[6] = nn.Linear(num_ftrs, self.ncats)
from torch.autograd import Variable
import torch.nn.functional as F
import keras #This dependency is only for loading the CIFAR-10 data set
from keras.datasets import cifar10
from copy import deepcopy
import vgg
(X_train, y_train), (X_test, y_test) = cifar10.load_data()
X_train = X_train.astype('float32')
X_train = np.transpose(X_train, axes=(0, 3, 1, 2))
X_test = X_test.astype('float32')
X_test = np.transpose(X_train, axes=(0, 3, 1, 2))
X_train /= 255
X_test /= 255
model = vgg.vgg11_bn()
model.to(cuda)
# Forward pass
opfun = lambda X: model.forward(Variable(torch.from_numpy(X)))
# Forward pass through the network given the input
predsfun = lambda op: np.argmax(op.data.numpy(), 1)
# Do the forward pass, then compute the accuracy
accfun = lambda op, y: np.mean(np.equal(predsfun(op), y.squeeze())) * 100
# Initial point
x0 = deepcopy(model.state_dict())
shuffle=True)
def NeptuneLog():
neptune.log_metric('batch_size', batch_sizes)
neptune.log_metric('learning_rate', learning_rate)
neptune.log_text('pre-trained', str(pretrain_check))
neptune.log_text('model', model_name)
neptune.log_text('date_time', date_time)
neptune.create_experiment(model_name)
NeptuneLog()
if model_name == 'vgg11':
model = vgg.vgg11(pretrained=pretrain_check)
elif model_name == 'vgg11_bn':
model = vgg.vgg11_bn(pretrained=pretrain_check)
elif model_name == 'vgg13':
model = vgg.vgg13(pretrained=pretrain_check)
elif model_name == 'vgg13_bn':
model = vgg.vgg13_bn(pretrained=pretrain_check)
elif model_name == 'vgg16':
model = vgg.vgg16(pretrained=pretrain_check)
elif model_name == 'vgg16_bn':
model = vgg.vgg16_bn(pretrained=pretrain_check)
elif model_name == 'vgg19':
model = vgg.vgg19(pretrained=pretrain_check)
elif model_name == 'vgg19_bn':
model = vgg.vgg19_bn(pretrained=pretrain_check)
model.eval()
model = torch.nn.DataParallel(model).cuda()
(np.eye(10)[labeled_train_set['label']]).astype(np.float32))
Y_valid = torch.from_numpy((np.eye(10)[valid_set['label']]).astype(np.float32))
X_loader = DataLoader(TensorDataset(X_train, Y_train),
batch_size=args.batch_size,
shuffle=False)
X_ul_loader = DataLoader(TensorDataset(Xul_train),
batch_size=args.batch_size_ul,
shuffle=False)
Valid_loader = DataLoader(TensorDataset(X_valid, Y_valid),
batch_size=args.batch_size,
shuffle=False)
vae = VAE(args.latent_dim)
vae.load_state_dict(torch.load(args.resume + '/model-120.pkl'))
net = vgg11_bn()
device = torch.device("cuda" if args.cuda else "cpu")
vae = vae.to(device)
net = net.to(device)
optimizer = optim.Adam(lr=args.lr, params=net.parameters())
def crossentropy(label, logits):
return -(label * (logits + 1e-8).log()).sum(dim=1).mean()
def kldivergence(label, logits):
return (label * ((label + 1e-8).log() -
def get_model(args):
network = args.network
if network == 'vgg11':
model = vgg.vgg11(num_classes=args.class_num)
elif network == 'vgg13':
model = vgg.vgg13(num_classes=args.class_num)
elif network == 'vgg16':
model = vgg.vgg16(num_classes=args.class_num)
elif network == 'vgg19':
model = vgg.vgg19(num_classes=args.class_num)
elif network == 'vgg11_bn':
model = vgg.vgg11_bn(num_classes=args.class_num)
elif network == 'vgg13_bn':
model = vgg.vgg13_bn(num_classes=args.class_num)
elif network == 'vgg16_bn':
model = vgg.vgg16_bn(num_classes=args.class_num)
elif network == 'vgg19_bn':
model = vgg.vgg19_bn(num_classes=args.class_num)
elif network == 'resnet18':
model = models.resnet18(num_classes=args.class_num)
model.conv1 = torch.nn.Conv2d(in_channels=1,
out_channels=model.conv1.out_channels,
kernel_size=model.conv1.kernel_size,
stride=model.conv1.stride,
padding=model.conv1.padding,
bias=model.conv1.bias)
elif network == 'resnet34':
model = models.resnet34(num_classes=args.class_num)
model.conv1 = torch.nn.Conv2d(in_channels=1,
out_channels=model.conv1.out_channels,
kernel_size=model.conv1.kernel_size,
stride=model.conv1.stride,
padding=model.conv1.padding,
bias=model.conv1.bias)
elif network == 'resnet50':
model = models.resnet50(num_classes=args.class_num)
model.conv1 = torch.nn.Conv2d(in_channels=1,
out_channels=model.conv1.out_channels,
kernel_size=model.conv1.kernel_size,
stride=model.conv1.stride,
padding=model.conv1.padding,
bias=model.conv1.bias)
elif network == 'resnet101':
model = models.resnet101(num_classes=args.class_num)
model.conv1 = torch.nn.Conv2d(in_channels=1,
out_channels=model.conv1.out_channels,
kernel_size=model.conv1.kernel_size,
stride=model.conv1.stride,
padding=model.conv1.padding,
bias=model.conv1.bias)
elif network == 'resnet152':
model = models.resnet152(num_classes=args.class_num)
model.conv1 = torch.nn.Conv2d(in_channels=1,
out_channels=model.conv1.out_channels,
kernel_size=model.conv1.kernel_size,
stride=model.conv1.stride,
padding=model.conv1.padding,
bias=model.conv1.bias)
elif network == 'densenet121':
model = densenet.densenet121(num_classes=args.class_num)
elif network == 'densenet169':
model = densenet.densenet169(num_classes=args.class_num)
elif network == 'densenet161':
model = densenet.densenet161(num_classes=args.class_num)
elif network == 'densenet201':
model = densenet.densenet201(num_classes=args.class_num)
return model
def VGG11(VGG11_model_dir):
model = vgg11_bn()
model.load_state_dict(torch.load(VGG11_model_dir))
return model
row, width_lower_bound:width_upper_bound]
vector = np.reshape(vector, [-1])
if len(vector):
cnt += 1
for element in vector:
if element != 0:
cnt -= 1
break
return cnt, total_vector_num
if args.arch == 'vgg11':
cfg = [38, 'M', 127, 'M', 232, 255, 'M', 491, 511, 'M', 512, 501, 'M']
model = vgg.vgg11_bn(pretrained=False, config=cfg)
elif args.arch == 'thinet50':
model = thinet50()
elif args.arch == 'thinet70':
model = thinet70()
elif args.arch == 'resnet50':
model = resnet.resnet50()
elif args.arch == 'vgg19':
checkpoint_cfg = torch.load(
'/home/yw68/network-slimming/pruned_vgg19_75/pruned.pth.tar')
model = vgg_cifar.vgg19(pretrained=False, cfg_vgg19=checkpoint_cfg['cfg'])
def model(model_dir):
model = vgg11_bn()
model.load_state_dict(torch.load(model_dir))
return model
def main():
#torch.manual_seed(123)
global args, best_prec1
best_prec1 = 0
args = parser.parse_args()
if args.evaluate:
args.results_dir = '/tmp'
if args.save is '':
args.save = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
save_path = os.path.join(args.results_dir, args.save)
if not os.path.exists(save_path):
os.makedirs(save_path)
else:
raise OSError('Directory {%s} exists. Use a new one.' % save_path)
setup_logging(os.path.join(save_path, 'log.txt'))
results_file = os.path.join(save_path, 'results.%s')
results = ResultsLog(results_file % 'csv', results_file % 'html')
logging.info("saving to %s", save_path)
logging.debug("run arguments: %s", args)
if 'cuda' in args.type:
#torch.cuda.manual_seed_all(123)
args.gpus = [int(i) for i in args.gpus.split(',')]
# torch.cuda.set_device(args.gpus[0])
cudnn.benchmark = True
if len(args.gpus) != 1:
raise NotImplementedError('Please use one gpu.')
else:
args.gpus = None
if args.batch_size != args.mini_batch_size:
args.mini_batch_size = args.batch_size
warnings.warn(
'--mini-batch-size is enforced to be set as --batch-size {}'.
format(args.mini_batch_size), RuntimeWarning)
logging.info("creating model %s", args.model)
# model = models.__dict__[args.model]
# model_config = {'input_size': args.input_size, 'dataset': args.dataset}
# create model
model = vgg.vgg11_bn()
# logging.info("creating model %s", args.model)
# model = models.__dict__[args.model]
# model_config = {'input_size': args.input_size, 'dataset': args.dataset}
mydict = {}
batchmodel = torch.load("BatchSize250.pth")
for key, value in batchmodel.items():
mydict[key] = value
model.load_state_dict(mydict)
if args.model_config is not '':
model_config = dict(model_config, **literal_eval(args.model_config))
# model = model(**model_config)
# logging.info("created model with configuration: %s", model_config)
#optionally resume from a checkpoint
# if args.evaluate:
# if not os.path.isfile(args.evaluate):
# parser.error('invalid checkpoint: {}'.format(args.evaluate))
# checkpoint = torch.load(args.evaluate, map_location=lambda storage, loc: storage)
# model.load_state_dict(checkpoint['state_dict'])
# logging.info("loaded checkpoint '%s' (epoch %s)",
# args.evaluate, checkpoint['epoch'])
# else:
# raise ValueError("Please specify the path of evaluated model")
num_parameters = sum([l.nelement() for l in model.parameters()])
logging.info("number of parameters: %d", num_parameters)
# Data loading code
default_transform = {
'train':
get_transform(args.dataset,
input_size=args.input_size,
augment=args.augment),
'eval':
get_transform(args.dataset, input_size=args.input_size, augment=False)
}
transform = getattr(model, 'input_transform', default_transform)
# define loss function (criterion) and optimizer
criterion = getattr(model, 'criterion', nn.CrossEntropyLoss)()
criterion.type(args.type)
# model.type(args.type)
val_data = get_dataset(args.dataset, 'val', transform['eval'])
val_loader = torch.utils.data.DataLoader(val_data,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
# for k in model.state_dict():
# if re.match('.*weight.*', k):
# plt.figure()
# plt.hist(model.state_dict()[k].cpu().numpy().reshape(-1), bins='auto')
# plt.title(k)
# plt.show()
val_result = validate(val_loader, model, criterion, 0)
val_loss, val_prec1, val_prec5 = [
val_result[r] for r in ['loss', 'prec1', 'prec5']
]
logging.info('\nValidation Loss {val_loss:.4f} \t'
'Validation Prec@1 {val_prec1:.3f} \t'
'Validation Prec@5 {val_prec5:.3f} \n'.format(
val_loss=val_loss,
val_prec1=val_prec1,
val_prec5=val_prec5))
sharpnesses = []
for time in range(args.times):
sharpness = get_sharpness(val_loader,
model,
criterion,
manifolds=args.manifolds)
sharpnesses.append(sharpness)
logging.info('sharpness {} = {}'.format(time, sharpness))
logging.info('sharpnesses = {}'.format(str(sharpnesses)))
_std = np.std(sharpnesses) * np.sqrt(args.times) / np.sqrt(args.times - 1)
_mean = np.mean(sharpnesses)
logging.info(u'mean sharpness = {sharpness:.4f}\u00b1{err:.4f}'.format(
sharpness=_mean, err=_std))
print(sharpnesses)
return
def train_and_save_model(model_name = 'resnet',
coreset_selector = 'k-centers',
coreset_percentage = 0.1,
trainset_size = 'subset',
device = 'cuda'
):
"""Trains every model in <models> input array and saves resulting weights. Uses other parameters to generate verbose console outputs during training.
Keyword arguments:
model_name -- name of model to be trained. Must be either 'resnet', 'mobilenet', 'vgg' or 'densenet'.
coreset_selector -- name of coreset selector. Must be either 'glister', 'k-centers' or 'random'.
coreset_percentage -- size of coreset as percentage of total train set. Must bei either '0.1', '0.3', '0.5' or '1.0'. If '1.0' is chosen, the model is trained on the full dataset, and coreset_selector is ignored.
trainset_size -- choose wether you want to train on full CIFAR10 or a subset of CIFAR10 (20000 datapoints)
"""
# Data
#factors selected from torch docs
mean = (0.4914, 0.4822, 0.4465)
std = (0.2471, 0.2435, 0.2616)
#preprocessing
transform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean, std)
])
if trainset_size == 'fullset':
train_dataset = dataset_manager.CIFAR10_full(
dataset_manager.__file__,
train=True,
transform=transform
)
elif trainset_size == 'subset':
train_dataset = dataset_manager.CIFAR10_subset(
dataset_manager.__file__,
transform=transform
)
else:
print("Dataset not supported, must be either 'fullset' or 'subset'.")
return
test_dataset = dataset_manager.CIFAR10_full(
dataset_manager.__file__,
train=False,
transform=transform
)
# Models
if model_name=='resnet':
model = resnet18()
elif model_name=='mobilenet':
model = mobilenet_v2()
elif model_name=='vgg':
model = vgg11_bn()
elif model_name=='densenet':
model = densenet121()
else:
print("Model not supported. Must be either 'resnet', 'mobilenet', 'vgg' and 'densenet'.")
return
# Coreset selectors
if coreset_percentage != 0.1 and coreset_percentage != 0.3 and coreset_percentage != 0.5 and coreset_percentage != 1.0:
print("Coreset size not supported. Must be either 0.1, 0.3, 0.5 or 1.0")
return
if coreset_percentage != 1.0:
if coreset_selector=='glister':
coreset_str = str(int(coreset_percentage*100))
train_indices = np.loadtxt('../../Glister/GlisterImage/indices/glister_indices_{}_{}.csv'.format(trainset_size, coreset_str), delimiter=',').astype(int)
elif coreset_selector=='k-centers':
train_indices = np.loadtxt('../../greedy_k_centers/k_centers_indices_{}.csv'.format(trainset_size), delimiter=',')[:int(len(train_dataset)*coreset_percentage)].astype(int)
elif coreset_selector=='random':
train_indices = np.arange(0, len(train_dataset))
train_indices = np.random.choice(train_indices, size=int(len(train_dataset)*coreset_percentage), replace=False)
else:
print("Coreset selector not supported. Must be either 'glister', 'k-centers' or 'random'.")
return
train_dataset = [train_dataset[i] for i in train_indices]
print("Size of selected dataset: ", len(train_dataset))
# Data loader
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=128,
shuffle=True,
drop_last=True
)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=128,
shuffle=False,
drop_last=True
)
# Device configuration
device = torch.device(device if torch.cuda.is_available() else 'cpu')
print("Using device: ", device)
# Training parameters
num_epochs = 100
learning_rate = 1e-2
weight_decay = 1e-2
total_steps = num_epochs * len(train_loader)
model = model.to(device)
# Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(
model.parameters(),
lr=learning_rate,
weight_decay=weight_decay,
momentum=0.9,
nesterov=True
)
# Scheduler
scheduler = WarmupCosineLR(
optimizer,
warmup_epochs=total_steps * 0.3,
max_epochs = total_steps)
# Train the model
accuracies = __train_model_epochs(num_epochs, model, model_name, criterion, optimizer, train_loader, test_loader, str(int(coreset_percentage * 100)), coreset_selector, scheduler=scheduler, verbose=True, device=device)
# take model as parameter, not cacluclate it
if coreset_selector == None:
weights_filename = 'model_weights/{:03}_{}_{}.pt'.format(int(coreset_percentage*100), model_name, trainset_size)
results_filename = 'accuracy_results/{:03}_{}_{}.csv'.format(int(coreset_percentage*100), model_name, trainset_size)
else:
weights_filename = 'model_weights/{:03}_{}_{}_{}.pt'.format(int(coreset_percentage*100), model_name, coreset_selector, trainset_size)
results_filename = 'accuracy_results/{:03}_{}_{}_{}.csv'.format(int(coreset_percentage*100), model_name, coreset_selector, trainset_size)
torch.save(model.state_dict(), weights_filename)
np.savetxt(results_filename, accuracies, delimiter=',')