cifar100Manager.load_metadata(x)[1]['predictionAccuracy'][-1],
max(cifar100Manager.load_metadata(x)[1]['predictionAccuracy']))
print(s)
try:
os.mkdir(cifar100modelsFolder)
except:pass
epochsToTrainCIFAR100 = 200
epochsToTrainCIFAR100_diffquant = 20
batch_size = 110
if batch_size % NUM_GPUS != 0:
raise ValueError('Batch size: {} must be a multiple of the number of gpus:{}'.format(batch_size, NUM_GPUS))
cifar100 = datasets.CIFAR100()
train_loader, test_loader = cifar100.getTrainLoader(batch_size), cifar100.getTestLoader(batch_size)
TRAIN_TEACHER_MODEL = False
TRAIN_DISTILLED_MODEL = True
# Teacher model
teacher_model_name = 'cifar100_teacher_new'
teacherModelPath = os.path.join(cifar100modelsFolder, teacher_model_name)
teacherOptions = {'widen_factor':20, 'depth':28, 'dropout_rate':0.3, 'num_classes':100}
teacherModel = Wide_ResNet(**teacherOptions)
if USE_CUDA: teacherModel = teacherModel.cuda()
if NUM_GPUS > 1:
teacherModel = torch.nn.parallel.DataParallel(teacherModel)
if not teacher_model_name in cifar100Manager.saved_models:
cifar100Manager.add_new_model(teacher_model_name, teacherModelPath,
def fetch_dataset(data_name):
print('fetching data {}...'.format(data_name))
if (data_name == 'MNIST'):
train_dir = './data/{}/train'.format(data_name)
test_dir = './data/{}/test'.format(data_name)
train_dataset = datasets.MNIST(root=train_dir,
train=True,
download=True,
transform=transforms.ToTensor())
if (normalize):
stats = make_stats(train_dataset, batch_size=128)
train_transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(stats)])
test_transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(stats)])
else:
train_transform = transforms.Compose([transforms.ToTensor()])
test_transform = transforms.Compose([transforms.ToTensor()])
train_dataset.transform = train_transform
test_dataset = datasets.MNIST(root=test_dir,
train=False,
download=True,
transform=test_transform)
elif (data_name == 'EMNIST' or data_name == 'EMNIST_byclass'
or data_name == 'EMNIST_bymerge' or data_name == 'EMNIST_balanced'
or data_name == 'EMNIST_letters' or data_name == 'EMNIST_digits'
or data_name == 'EMNIST_mnist'):
train_dir = './data/{}/train'.format(data_name.split('_')[0])
test_dir = './data/{}/test'.format(data_name.split('_')[0])
transform = transforms.Compose([transforms.ToTensor()])
split = 'balanced' if len(
data_name.split('_')) == 1 else data_name.split('_')[1]
train_dataset = datasets.EMNIST(root=train_dir,
split=split,
branch=branch,
train=True,
download=True,
transform=transform)
test_dataset = datasets.EMNIST(root=test_dir,
split=split,
branch=branch,
train=False,
download=True,
transform=transform)
elif (data_name == 'FashionMNIST'):
train_dir = './data/{}/train'.format(data_name)
test_dir = './data/{}/test'.format(data_name)
transform = transforms.Compose([transforms.ToTensor()])
train_dataset = datasets.FashionMNIST(root=train_dir,
train=True,
download=True,
transform=transform)
test_dataset = datasets.FashionMNIST(root=test_dir,
train=False,
download=True,
transform=transform)
elif (data_name == 'CIFAR10'):
train_dir = './data/{}/train'.format(data_name)
test_dir = './data/{}/validation'.format(data_name)
train_dataset = datasets.CIFAR10(train_dir,
train=True,
transform=transforms.ToTensor(),
download=True)
if (normalize):
stats = make_stats(train_dataset, batch_size=128)
train_transform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(stats)
])
test_transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(stats)])
else:
train_transform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor()
])
test_transform = transforms.Compose([transforms.ToTensor()])
train_dataset.transform = train_transform
test_dataset = datasets.CIFAR10(test_dir,
train=False,
transform=test_transform,
download=True)
elif (data_name == 'CIFAR100'):
train_dir = './data/{}/train'.format(data_name)
test_dir = './data/{}/validation'.format(data_name)
train_dataset = datasets.CIFAR100(train_dir,
branch=branch,
train=True,
transform=transforms.ToTensor(),
download=True)
if (normalize):
stats = make_stats(train_dataset, batch_size=128)
train_transform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(stats)
])
test_transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(stats)])
else:
train_transform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor()
])
test_transform = transforms.Compose([transforms.ToTensor()])
train_dataset.transform = train_transform
test_dataset = datasets.CIFAR100(test_dir,
branch=branch,
train=False,
transform=test_transform,
download=True)
elif (data_name == 'SVHN'):
train_dir = './data/{}/train'.format(data_name)
test_dir = './data/{}/validation'.format(data_name)
train_dataset = datasets.SVHN(train_dir,
split='train',
transform=transforms.ToTensor(),
download=True)
if (normalize):
stats = make_stats(train_dataset, batch_size=128)
train_transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(stats)])
test_transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(stats)])
else:
train_transform = transforms.Compose([transforms.ToTensor()])
test_transform = transforms.Compose([transforms.ToTensor()])
train_dataset.transform = train_transform
test_dataset = datasets.SVHN(test_dir,
split='test',
transform=test_transform,
download=True)
elif (data_name == 'ImageNet'):
train_dir = './data/{}/train'.format(data_name)
test_dir = './data/{}/validation'.format(data_name)
train_dataset = datasets.ImageFolder(train_dir,
transform=transforms.ToTensor())
if (normalize):
stats = make_stats(train_dataset, batch_size=128)
train_transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize(stats)
])
test_transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize(stats)
])
else:
train_transform = transforms.Compose(
[transforms.Resize((224, 224)),
transforms.ToTensor()])
test_transform = transforms.Compose(
[transforms.Resize((224, 224)),
transforms.ToTensor()])
train_dataset.transform = train_transform
test_dataset = datasets.ImageFolder(test_dir, transform=test_transform)
elif (data_name == 'CUB2011'):
train_dir = './data/{}/train'.format(data_name.split('_')[0])
test_dir = './data/{}/validation'.format(data_name.split('_')[0])
train_dataset = datasets.CUB2011(train_dir,
transform=transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor()
]),
download=True)
if (normalize):
stats = make_stats(train_dataset, batch_size=128)
train_transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize(stats)
])
test_transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize(stats)
])
else:
train_transform = transforms.Compose(
[transforms.Resize((224, 224)),
transforms.ToTensor()])
test_transform = transforms.Compose(
[transforms.Resize((224, 224)),
transforms.ToTensor()])
train_dataset.transform = train_transform
test_dataset = datasets.CUB2011(test_dir,
transform=test_transform,
download=True)
elif (data_name == 'WheatImage' or data_name == 'WheatImage_binary'
or data_name == 'WheatImage_six'):
train_dir = './data/{}/train'.format(data_name.split('_')[0])
test_dir = './data/{}/validation'.format(data_name.split('_')[0])
label_mode = 'six' if len(
data_name.split('_')) == 1 else data_name.split('_')[1]
train_dataset = datasets.WheatImage(train_dir,
label_mode=label_mode,
transform=transforms.Compose([
transforms.Resize((224, 288)),
transforms.ToTensor()
]))
if (normalize):
stats = make_stats(train_dataset, batch_size=128)
train_transform = transforms.Compose([
transforms.Resize((224, 288)),
transforms.RandomHorizontalFlip(),
transforms.RandomVerticalFlip(),
transforms.ToTensor(),
transforms.Normalize(stats)
])
test_transform = transforms.Compose([
transforms.Resize((224, 288)),
transforms.ToTensor(),
transforms.Normalize(stats)
])
else:
train_transform = transforms.Compose([
transforms.Resize((224, 288)),
transforms.RandomHorizontalFlip(),
transforms.RandomVerticalFlip(),
transforms.ToTensor()
])
test_transform = transforms.Compose(
[transforms.Resize((224, 288)),
transforms.ToTensor()])
train_dataset.transform = train_transform
test_dataset = datasets.WheatImage(test_dir,
label_mode=label_mode,
transform=test_transform)
elif (data_name == 'CocoDetection'):
train_dir = './data/Coco/train2017'
train_ann = './data/Coco/annotations/instances_train2017.json'
test_dir = './data/Coco/val2017'
test_ann = './data/Coco/annotations/instances_val2017.json'
transform = transforms.Compose(
[transforms.Resize((224, 224)),
transforms.ToTensor()])
train_dataset = datasets.CocoDetection(train_dir,
train_ann,
transform=transform)
test_dataset = datasets.CocoDetection(test_dir,
test_ann,
transform=transform)
elif (data_name == 'CocoCaptions'):
train_dir = './data/Coco/train2017'
train_ann = './data/Coco/annotations/captions_train2017.json'
test_dir = './data/Coco/val2017'
test_ann = './data/Coco/annotations/captions_val2017.json'
transform = transforms.Compose(
[transforms.Resize((224, 224)),
transforms.ToTensor()])
train_dataset = datasets.CocoCaptions(train_dir,
train_ann,
transform=transform)
test_dataset = datasets.CocoCaptions(test_dir,
test_ann,
transform=transform)
elif (data_name == 'VOCDetection'):
train_dir = './data/VOC/VOCdevkit'
test_dir = './data/VOC/VOCdevkit'
transform = transforms.Compose(
[transforms.Resize((224, 224)),
transforms.ToTensor()])
train_dataset = datasets.VOCDetection(train_dir,
'trainval',
transform=transform)
test_dataset = datasets.VOCDetection(test_dir,
'test',
transform=transform)
elif (data_name == 'VOCSegmentation'):
train_dir = './data/VOC/VOCdevkit'
test_dir = './data/VOC/VOCdevkit'
transform = transforms.Compose(
[transforms.Resize((224, 224)),
transforms.ToTensor()])
train_dataset = datasets.VOCSegmentation(train_dir,
'trainval',
transform=transform)
test_dataset = datasets.VOCSegmentation(test_dir,
'test',
transform=transform)
elif (data_name == 'MOSI' or data_name == 'MOSI_binary'
or data_name == 'MOSI_five' or data_name == 'MOSI_seven'
or data_name == 'MOSI_regression'):
train_dir = './data/{}'.format(data_name.split('_')[0])
test_dir = './data/{}'.format(data_name.split('_')[0])
label_mode = 'five' if len(
data_name.split('_')) == 1 else data_name.split('_')[1]
train_dataset = datasets.MOSI(train_dir,
split='trainval',
label_mode=label_mode,
download=True)
stats = make_stats(train_dataset, batch_size=1)
train_transform = transforms.Compose([transforms.Normalize(stats)])
test_transform = transforms.Compose([transforms.Normalize(stats)])
train_dataset.transform = train_transform
test_dataset = datasets.MOSI(test_dir,
split='test',
label_mode=label_mode,
download=True,
transform=test_transform)
elif (data_name == 'Kodak'):
train_dataset = None
transform = transforms.Compose([transforms.ToTensor()])
test_dir = './data/{}'.format(data_name)
train_dataset = datasets.ImageFolder(test_dir, transform)
test_dataset = datasets.ImageFolder(test_dir, transform)
elif (data_name == 'UCID'):
train_dataset = None
transform = transforms.Compose([transforms.ToTensor()])
test_dir = './data/{}'.format(data_name)
train_dataset = datasets.ImageFolder(test_dir, transform)
test_dataset = datasets.ImageFolder(test_dir, transform)
else:
raise ValueError('Not valid dataset name')
print('data ready')
return train_dataset, test_dataset
create_new_model_manager=create_new)
modelsFolder = os.path.join(SAVED_MODELS_FOLDER, args.data)
try:
os.mkdir(modelsFolder)
except:
pass
epochsToTrainCIFAR = args.epochs
USE_BATCH_NORM = True
AFFINE_BATCH_NORM = True
if args.data == 'cifar10':
data = datasets.CIFAR10()
elif args.data == 'cifar100':
data = datasets.CIFAR100()
if args.test_memory:
if not args.train_teacher:
model = convForwModel.ConvolForwardNet(
**smallerModelSpecs[args.stModel],
activation=args.stud_act,
numBins=args.num_bins,
useBatchNorm=USE_BATCH_NORM,
useAffineTransformInBatchNorm=AFFINE_BATCH_NORM)
else:
model = convForwModel.ConvolForwardNet(
**convForwModel.teacherModelSpec,
useBatchNorm=USE_BATCH_NORM,
useAffineTransformInBatchNorm=AFFINE_BATCH_NORM)
if USE_CUDA: model = model.cuda()
test_loader = data.getTestLoader(1)
def load_datasets(name, root, batch_size):
if name == "mnist":
train_dataset = datasets.MNIST(root=root,
download=True,
train=True,
transform=transforms.Compose([
transforms.Resize(28),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.5], [0.5]),
]))
train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size,
shuffle=True, num_workers=8)
test_dataset = datasets.MNIST(root=root,
download=True,
train=False,
transform=transforms.Compose([
transforms.Resize(28),
transforms.ToTensor(),
transforms.Normalize([0.5], [0.5]),
]))
test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size,
shuffle=False, num_workers=8)
return train_dataloader, test_dataloader
elif name == "fmnist":
train_dataset = datasets.FashionMNIST(root=root,
download=True,
train=True,
transform=transforms.Compose([
transforms.Resize(28),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.5], [0.5]),
]))
train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size,
shuffle=True, num_workers=8)
test_dataset = datasets.FashionMNIST(root=root,
download=True,
train=False,
transform=transforms.Compose([
transforms.Resize(28),
transforms.ToTensor(),
transforms.Normalize([0.5], [0.5]),
]))
test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size,
shuffle=False, num_workers=8)
return train_dataloader, test_dataloader
elif name == "kmnist":
train_dataset = datasets.KMNIST(root=root,
download=True,
train=True,
transform=transforms.Compose([
transforms.Resize(28),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.5], [0.5]),
]))
train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size,
shuffle=True, num_workers=8)
test_dataset = datasets.KMNIST(root=root,
download=True,
train=False,
transform=transforms.Compose([
transforms.Resize(28),
transforms.ToTensor(),
transforms.Normalize([0.5], [0.5]),
]))
test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size,
shuffle=False, num_workers=8)
return train_dataloader, test_dataloader
elif name == "qmnist":
train_dataset = datasets.QMNIST(root=root,
download=True,
train=True,
transform=transforms.Compose([
transforms.Resize(28),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.5], [0.5]),
]))
train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size,
shuffle=True, num_workers=8)
test_dataset = datasets.QMNIST(root=root,
download=True,
what="test50k",
train=False,
transform=transforms.Compose([
transforms.Resize(28),
transforms.ToTensor(),
transforms.Normalize([0.5], [0.5]),
]))
test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size,
shuffle=False, num_workers=8)
return train_dataloader, test_dataloader
elif name == "cifar10":
train_dataset = datasets.CIFAR10(root=root,
download=True,
train=True,
transform=transforms.Compose([
transforms.Resize(32),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
]))
train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size,
shuffle=True, num_workers=8)
test_dataset = datasets.CIFAR10(root=root,
download=True,
train=False,
transform=transforms.Compose([
transforms.Resize(32),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
]))
test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size,
shuffle=False, num_workers=8)
return train_dataloader, test_dataloader
elif name == "cifar100":
train_dataset = datasets.CIFAR100(root=root,
download=True,
train=True,
transform=transforms.Compose([
transforms.Resize(32),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
]))
train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size,
shuffle=True, num_workers=8)
test_dataset = datasets.CIFAR100(root=root,
download=True,
train=False,
transform=transforms.Compose([
transforms.Resize(32),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
]))
test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size,
shuffle=False, num_workers=8)
return train_dataloader, test_dataloader