# model = model.eval()
mean, std = get_mean_std(args.dataset)
pad = int((row.padded_im_size - row.im_size) / 2)
transform = transforms.Compose([
transforms.Pad(pad),
transforms.ToTensor(),
transforms.Normalize(mean, std)
])
if args.dataset in ['MNIST', 'FashionMNIST', 'CIFAR10', 'CIFAR100']:
full_dataset = getattr(datasets, args.dataset)
subset_dataset = get_subset_dataset(
full_dataset=full_dataset,
examples_per_class=args.examples_per_class,
epc_seed=row.epc_seed,
root=osp.join(args.dataset_root, args.dataset),
train=True,
transform=transform,
download=True)
elif args.dataset == 'STL10':
full_dataset = datasets.STL10
subset_dataset = get_subset_dataset(
full_dataset=full_dataset,
examples_per_class=args.examples_per_class,
epc_seed=row.epc_seed,
root=osp.join(args.dataset_root, args.dataset),
split='train',
transform=transform,
download=False)
else:
raise Exception('Unknown dataset: {}'.format(args.dataset))
# Transform
mean, std = get_mean_std(dataset)
pad = int((row.padded_im_size - row.im_size) / 2)
transform = transforms.Compose([
transforms.Pad(pad),
transforms.ToTensor(),
transforms.Normalize(mean, std)
])
# Subsampled dataset
full_dataset = getattr(datasets, dataset)
subset_dataset = get_subset_dataset(full_dataset=full_dataset,
examples_per_class=examples_per_class,
epc_seed=row.epc_seed,
root='data/',
train=True,
transform=transform,
download=True)
# Loader
loader = DataLoader(dataset=subset_dataset,
drop_last=False,
batch_size=batch_size)
# In[6]:
#%% G decomposition
C = row.num_classes
else:
train_transform = transforms.Compose([
transforms.Pad(int((config.padded_im_size - config.im_size) / 2)),
transforms.ToTensor(),
transforms.Normalize(mean, std)
])
test_transform = transforms.Compose([
transforms.Pad((config.padded_im_size - config.im_size) // 2),
transforms.ToTensor(),
transforms.Normalize(mean, std)
])
full_dataset = getattr(datasets, args.dataset)
subset_dataset = get_subset_dataset(full_dataset=full_dataset,
examples_per_class=40,
epc_seed=config.epc_seed,
root=osp.join(args.dataset_root,
args.dataset),
train=True,
transform=test_transform,
download=True)
loader = DataLoader(dataset=subset_dataset,
drop_last=False,
batch_size=args.batch_size)
if args.model in [
'VGG11_bn', 'ResNet18', 'DenseNet3_40', 'LeNet', 'MobileNet'
]:
model = Network().construct(args.model, config)
else:
raise Exception('Unknown model: {}'.format())
gridLabels = np.concatenate((trainingLabels, validationLabels))
indexValid = list(
range(len(trainingLabels),
len(trainingLabels) + len(validationLabels)))
# Train the classifiers on training sets and save the data in a specific folder
accuracy = []
indexName = []
# ========== Baseline Classifier ==========
print("baseline training...")
nameClassifier = "BaselineClassifier"
baseClassifier = baselineClassifier.BaselineClassifier()
trainedClassifier = baseClassifier.get_overlaps(
utils.get_subset_dataset(dataset, trainingClaimSet + validationClaimSet))
thresholds = baseClassifier.calculate_classifier_thresholds(trainedClassifier)
# get the predicted labels
testingData = baseClassifier.get_overlaps(
utils.get_subset_dataset(dataset, testingClaimSet))
#print(testingData)
#print(thresholds)
predLabels = baseClassifier.predict(thresholds, testingData)
# Get the metrics and save them
acc = utils.getAllMetricsAndSave(nameClassifier, predLabels, testingLabels)
# Accuracy
accuracy.append(acc)
indexName.append(nameClassifier)
logger_file_handler = logging.FileHandler(logging_file)
logger.addHandler(logger_file_handler)
logger.info('Arguments: {}'.format(args))
mean, std = get_mean_std(args.dataset)
transform = transforms.Compose([
transforms.Pad(0),
transforms.ToTensor(),
transforms.Normalize(mean, std)
])
full_dataset = getattr(datasets, args.dataset)
train_data = get_subset_dataset(full_dataset=full_dataset,
examples_per_class=args.examples_per_class,
epc_seed=args.epc_seed,
root=osp.join(args.dataset_root,
args.dataset),
train=True,
transform=transform,
download=True)
test_data = get_subset_dataset(full_dataset=full_dataset,
examples_per_class=args.examples_per_class,
epc_seed=args.epc_seed,
root=osp.join(args.dataset_root,
args.dataset),
train=False,
transform=transform,
download=True)
dataloaders = dict()
dataloaders['train'] = data.DataLoader(train_data,
batch_size=args.batch_size,
def train(model,
optimizer,
scheduler,
dataloaders,
criterion,
device,
num_epochs=100,
args=None,
dataset_sizes={
'train': 5e4,
'test': 1e4
},
images_dir=None,
ckpt_dir=None):
logger = logging.getLogger('train')
loss_list = {'train': list(), 'test': list()}
acc_list = {'train': list(), 'test': list()}
assert images_dir is not None
assert ckpt_dir is not None
loss_image_path = osp.join(images_dir, 'loss.png')
acc_image_path = osp.join(images_dir, 'acc.png')
model.train()
full_eigenspectrums = list()
epoch_eigenspectrums = list()
full_eigenspectrums_path = osp.join(ckpt_dir,
'training_eigenspectrum_full.npy')
C = config.num_classes
valid_layers = get_valid_layers(model)
for epoch in range(num_epochs):
logger.info('epoch: %d' % epoch)
with torch.enable_grad():
for batch, truth in dataloaders['train']:
batch = batch.to(device)
truth = truth.to(device)
optimizer.zero_grad()
output = model(batch)
loss = criterion(output, truth)
loss.backward()
optimizer.step()
scheduler.step()
# updates finished for epochs
mean, std = get_mean_std(args.dataset)
pad = int((config.padded_im_size - config.im_size) / 2)
transform = transforms.Compose([
transforms.Pad(pad),
transforms.ToTensor(),
transforms.Normalize(mean, std)
])
if args.dataset in ['MNIST', 'FashionMNIST', 'CIFAR10', 'CIFAR100']:
full_dataset = getattr(datasets, args.dataset)
subset_dataset = get_subset_dataset(
full_dataset=full_dataset,
examples_per_class=args.examples_per_class,
epc_seed=config.epc_seed,
root=osp.join(args.dataset_root, args.dataset),
train=True,
transform=transform,
download=True)
elif args.dataset in ['STL10', 'SVHN']:
full_dataset = getattr(datasets, args.dataset)
subset_dataset = get_subset_dataset(
full_dataset=full_dataset,
examples_per_class=args.examples_per_class,
epc_seed=config.epc_seed,
root=osp.join(args.dataset_root, args.dataset),
split='train',
transform=transform,
download=True)
else:
raise Exception('Unknown dataset: {}'.format(args.dataset))
loader = data.DataLoader(dataset=subset_dataset,
drop_last=False,
batch_size=args.batch_size)
Hess = FullHessian(crit='CrossEntropyLoss',
loader=loader,
device=device,
model=model,
num_classes=C,
hessian_type='Hessian',
init_poly_deg=64,
poly_deg=128,
spectrum_margin=0.05,
poly_points=1024,
SSI_iters=128)
Hess_eigval, \
Hess_eigval_density = Hess.LanczosLoop(denormalize=True)
full_eigenspectrums.append(Hess_eigval)
full_eigenspectrums.append(Hess_eigval_density)
for layer_name, _ in model.named_parameters():
if layer_name not in valid_layers:
continue
Hess = LayerHessian(crit='CrossEntropyLoss',
loader=loader,
device=device,
model=model,
num_classes=C,
layer_name=layer_name,
hessian_type='Hessian',
init_poly_deg=64,
poly_deg=128,
spectrum_margin=0.05,
poly_points=1024,
SSI_iters=128)
Hess_eigval, \
Hess_eigval_density = Hess.LanczosLoop(denormalize=True)
layerwise_eigenspectrums_path = osp.join(
ckpt_dir,
'training_eigenspectrums_epoch_{}_layer_{}.npz'.format(
epoch, layer_name))
np.savez(layerwise_eigenspectrums_path,
eigval=Hess_eigval,
eigval_density=Hess_eigval_density)
for phase in ['train', 'test']:
stats = evaluate_model(model, criterion, dataloaders[phase],
device, dataset_sizes[phase])
loss_list[phase].append(stats['loss'])
acc_list[phase].append(stats['acc'])
logger.info('{}:'.format(phase))
logger.info('\tloss:{}'.format(stats['loss']))
logger.info('\tacc :{}'.format(stats['acc']))
if phase == 'test':
plt.clf()
plt.plot(loss_list['test'], label='test_loss')
plt.plot(loss_list['train'], label='train_loss')
plt.legend()
plt.savefig(loss_image_path)
plt.clf()
plt.plot(acc_list['test'], label='test_acc')
plt.plot(acc_list['train'], label='train_acc')
plt.legend()
plt.savefig(acc_image_path)
plt.clf()
full_eigenspectrums = np.array(full_eigenspectrums)
assert full_eigenspectrums.shape[0] % 2 == 0
assert full_eigenspectrums.shape[0] // 2 == num_epochs
np.save(full_eigenspectrums_path, full_eigenspectrums)
return full_eigenspectrums
