cifar100Manager.load_metadata(x)[1]['predictionAccuracy'][-1], max(cifar100Manager.load_metadata(x)[1]['predictionAccuracy'])) print(s) try: os.mkdir(cifar100modelsFolder) except:pass epochsToTrainCIFAR100 = 200 epochsToTrainCIFAR100_diffquant = 20 batch_size = 110 if batch_size % NUM_GPUS != 0: raise ValueError('Batch size: {} must be a multiple of the number of gpus:{}'.format(batch_size, NUM_GPUS)) cifar100 = datasets.CIFAR100() train_loader, test_loader = cifar100.getTrainLoader(batch_size), cifar100.getTestLoader(batch_size) TRAIN_TEACHER_MODEL = False TRAIN_DISTILLED_MODEL = True # Teacher model teacher_model_name = 'cifar100_teacher_new' teacherModelPath = os.path.join(cifar100modelsFolder, teacher_model_name) teacherOptions = {'widen_factor':20, 'depth':28, 'dropout_rate':0.3, 'num_classes':100} teacherModel = Wide_ResNet(**teacherOptions) if USE_CUDA: teacherModel = teacherModel.cuda() if NUM_GPUS > 1: teacherModel = torch.nn.parallel.DataParallel(teacherModel) if not teacher_model_name in cifar100Manager.saved_models: cifar100Manager.add_new_model(teacher_model_name, teacherModelPath,
def fetch_dataset(data_name): print('fetching data {}...'.format(data_name)) if (data_name == 'MNIST'): train_dir = './data/{}/train'.format(data_name) test_dir = './data/{}/test'.format(data_name) train_dataset = datasets.MNIST(root=train_dir, train=True, download=True, transform=transforms.ToTensor()) if (normalize): stats = make_stats(train_dataset, batch_size=128) train_transform = transforms.Compose( [transforms.ToTensor(), transforms.Normalize(stats)]) test_transform = transforms.Compose( [transforms.ToTensor(), transforms.Normalize(stats)]) else: train_transform = transforms.Compose([transforms.ToTensor()]) test_transform = transforms.Compose([transforms.ToTensor()]) train_dataset.transform = train_transform test_dataset = datasets.MNIST(root=test_dir, train=False, download=True, transform=test_transform) elif (data_name == 'EMNIST' or data_name == 'EMNIST_byclass' or data_name == 'EMNIST_bymerge' or data_name == 'EMNIST_balanced' or data_name == 'EMNIST_letters' or data_name == 'EMNIST_digits' or data_name == 'EMNIST_mnist'): train_dir = './data/{}/train'.format(data_name.split('_')[0]) test_dir = './data/{}/test'.format(data_name.split('_')[0]) transform = transforms.Compose([transforms.ToTensor()]) split = 'balanced' if len( data_name.split('_')) == 1 else data_name.split('_')[1] train_dataset = datasets.EMNIST(root=train_dir, split=split, branch=branch, train=True, download=True, transform=transform) test_dataset = datasets.EMNIST(root=test_dir, split=split, branch=branch, train=False, download=True, transform=transform) elif (data_name == 'FashionMNIST'): train_dir = './data/{}/train'.format(data_name) test_dir = './data/{}/test'.format(data_name) transform = transforms.Compose([transforms.ToTensor()]) train_dataset = datasets.FashionMNIST(root=train_dir, train=True, download=True, transform=transform) test_dataset = datasets.FashionMNIST(root=test_dir, train=False, download=True, transform=transform) elif (data_name == 'CIFAR10'): train_dir = './data/{}/train'.format(data_name) test_dir = './data/{}/validation'.format(data_name) train_dataset = datasets.CIFAR10(train_dir, train=True, transform=transforms.ToTensor(), download=True) if (normalize): stats = make_stats(train_dataset, batch_size=128) train_transform = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(stats) ]) test_transform = transforms.Compose( [transforms.ToTensor(), transforms.Normalize(stats)]) else: train_transform = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor() ]) test_transform = transforms.Compose([transforms.ToTensor()]) train_dataset.transform = train_transform test_dataset = datasets.CIFAR10(test_dir, train=False, transform=test_transform, download=True) elif (data_name == 'CIFAR100'): train_dir = './data/{}/train'.format(data_name) test_dir = './data/{}/validation'.format(data_name) train_dataset = datasets.CIFAR100(train_dir, branch=branch, train=True, transform=transforms.ToTensor(), download=True) if (normalize): stats = make_stats(train_dataset, batch_size=128) train_transform = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize(stats) ]) test_transform = transforms.Compose( [transforms.ToTensor(), transforms.Normalize(stats)]) else: train_transform = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor() ]) test_transform = transforms.Compose([transforms.ToTensor()]) train_dataset.transform = train_transform test_dataset = datasets.CIFAR100(test_dir, branch=branch, train=False, transform=test_transform, download=True) elif (data_name == 'SVHN'): train_dir = './data/{}/train'.format(data_name) test_dir = './data/{}/validation'.format(data_name) train_dataset = datasets.SVHN(train_dir, split='train', transform=transforms.ToTensor(), download=True) if (normalize): stats = make_stats(train_dataset, batch_size=128) train_transform = transforms.Compose( [transforms.ToTensor(), transforms.Normalize(stats)]) test_transform = transforms.Compose( [transforms.ToTensor(), transforms.Normalize(stats)]) else: train_transform = transforms.Compose([transforms.ToTensor()]) test_transform = transforms.Compose([transforms.ToTensor()]) train_dataset.transform = train_transform test_dataset = datasets.SVHN(test_dir, split='test', transform=test_transform, download=True) elif (data_name == 'ImageNet'): train_dir = './data/{}/train'.format(data_name) test_dir = './data/{}/validation'.format(data_name) train_dataset = datasets.ImageFolder(train_dir, transform=transforms.ToTensor()) if (normalize): stats = make_stats(train_dataset, batch_size=128) train_transform = transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize(stats) ]) test_transform = transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize(stats) ]) else: train_transform = transforms.Compose( [transforms.Resize((224, 224)), transforms.ToTensor()]) test_transform = transforms.Compose( [transforms.Resize((224, 224)), transforms.ToTensor()]) train_dataset.transform = train_transform test_dataset = datasets.ImageFolder(test_dir, transform=test_transform) elif (data_name == 'CUB2011'): train_dir = './data/{}/train'.format(data_name.split('_')[0]) test_dir = './data/{}/validation'.format(data_name.split('_')[0]) train_dataset = datasets.CUB2011(train_dir, transform=transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor() ]), download=True) if (normalize): stats = make_stats(train_dataset, batch_size=128) train_transform = transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize(stats) ]) test_transform = transforms.Compose([ transforms.Resize((224, 224)), transforms.ToTensor(), transforms.Normalize(stats) ]) else: train_transform = transforms.Compose( [transforms.Resize((224, 224)), transforms.ToTensor()]) test_transform = transforms.Compose( [transforms.Resize((224, 224)), transforms.ToTensor()]) train_dataset.transform = train_transform test_dataset = datasets.CUB2011(test_dir, transform=test_transform, download=True) elif (data_name == 'WheatImage' or data_name == 'WheatImage_binary' or data_name == 'WheatImage_six'): train_dir = './data/{}/train'.format(data_name.split('_')[0]) test_dir = './data/{}/validation'.format(data_name.split('_')[0]) label_mode = 'six' if len( data_name.split('_')) == 1 else data_name.split('_')[1] train_dataset = datasets.WheatImage(train_dir, label_mode=label_mode, transform=transforms.Compose([ transforms.Resize((224, 288)), transforms.ToTensor() ])) if (normalize): stats = make_stats(train_dataset, batch_size=128) train_transform = transforms.Compose([ transforms.Resize((224, 288)), transforms.RandomHorizontalFlip(), transforms.RandomVerticalFlip(), transforms.ToTensor(), transforms.Normalize(stats) ]) test_transform = transforms.Compose([ transforms.Resize((224, 288)), transforms.ToTensor(), transforms.Normalize(stats) ]) else: train_transform = transforms.Compose([ transforms.Resize((224, 288)), transforms.RandomHorizontalFlip(), transforms.RandomVerticalFlip(), transforms.ToTensor() ]) test_transform = transforms.Compose( [transforms.Resize((224, 288)), transforms.ToTensor()]) train_dataset.transform = train_transform test_dataset = datasets.WheatImage(test_dir, label_mode=label_mode, transform=test_transform) elif (data_name == 'CocoDetection'): train_dir = './data/Coco/train2017' train_ann = './data/Coco/annotations/instances_train2017.json' test_dir = './data/Coco/val2017' test_ann = './data/Coco/annotations/instances_val2017.json' transform = transforms.Compose( [transforms.Resize((224, 224)), transforms.ToTensor()]) train_dataset = datasets.CocoDetection(train_dir, train_ann, transform=transform) test_dataset = datasets.CocoDetection(test_dir, test_ann, transform=transform) elif (data_name == 'CocoCaptions'): train_dir = './data/Coco/train2017' train_ann = './data/Coco/annotations/captions_train2017.json' test_dir = './data/Coco/val2017' test_ann = './data/Coco/annotations/captions_val2017.json' transform = transforms.Compose( [transforms.Resize((224, 224)), transforms.ToTensor()]) train_dataset = datasets.CocoCaptions(train_dir, train_ann, transform=transform) test_dataset = datasets.CocoCaptions(test_dir, test_ann, transform=transform) elif (data_name == 'VOCDetection'): train_dir = './data/VOC/VOCdevkit' test_dir = './data/VOC/VOCdevkit' transform = transforms.Compose( [transforms.Resize((224, 224)), transforms.ToTensor()]) train_dataset = datasets.VOCDetection(train_dir, 'trainval', transform=transform) test_dataset = datasets.VOCDetection(test_dir, 'test', transform=transform) elif (data_name == 'VOCSegmentation'): train_dir = './data/VOC/VOCdevkit' test_dir = './data/VOC/VOCdevkit' transform = transforms.Compose( [transforms.Resize((224, 224)), transforms.ToTensor()]) train_dataset = datasets.VOCSegmentation(train_dir, 'trainval', transform=transform) test_dataset = datasets.VOCSegmentation(test_dir, 'test', transform=transform) elif (data_name == 'MOSI' or data_name == 'MOSI_binary' or data_name == 'MOSI_five' or data_name == 'MOSI_seven' or data_name == 'MOSI_regression'): train_dir = './data/{}'.format(data_name.split('_')[0]) test_dir = './data/{}'.format(data_name.split('_')[0]) label_mode = 'five' if len( data_name.split('_')) == 1 else data_name.split('_')[1] train_dataset = datasets.MOSI(train_dir, split='trainval', label_mode=label_mode, download=True) stats = make_stats(train_dataset, batch_size=1) train_transform = transforms.Compose([transforms.Normalize(stats)]) test_transform = transforms.Compose([transforms.Normalize(stats)]) train_dataset.transform = train_transform test_dataset = datasets.MOSI(test_dir, split='test', label_mode=label_mode, download=True, transform=test_transform) elif (data_name == 'Kodak'): train_dataset = None transform = transforms.Compose([transforms.ToTensor()]) test_dir = './data/{}'.format(data_name) train_dataset = datasets.ImageFolder(test_dir, transform) test_dataset = datasets.ImageFolder(test_dir, transform) elif (data_name == 'UCID'): train_dataset = None transform = transforms.Compose([transforms.ToTensor()]) test_dir = './data/{}'.format(data_name) train_dataset = datasets.ImageFolder(test_dir, transform) test_dataset = datasets.ImageFolder(test_dir, transform) else: raise ValueError('Not valid dataset name') print('data ready') return train_dataset, test_dataset
create_new_model_manager=create_new) modelsFolder = os.path.join(SAVED_MODELS_FOLDER, args.data) try: os.mkdir(modelsFolder) except: pass epochsToTrainCIFAR = args.epochs USE_BATCH_NORM = True AFFINE_BATCH_NORM = True if args.data == 'cifar10': data = datasets.CIFAR10() elif args.data == 'cifar100': data = datasets.CIFAR100() if args.test_memory: if not args.train_teacher: model = convForwModel.ConvolForwardNet( **smallerModelSpecs[args.stModel], activation=args.stud_act, numBins=args.num_bins, useBatchNorm=USE_BATCH_NORM, useAffineTransformInBatchNorm=AFFINE_BATCH_NORM) else: model = convForwModel.ConvolForwardNet( **convForwModel.teacherModelSpec, useBatchNorm=USE_BATCH_NORM, useAffineTransformInBatchNorm=AFFINE_BATCH_NORM) if USE_CUDA: model = model.cuda() test_loader = data.getTestLoader(1)
def load_datasets(name, root, batch_size): if name == "mnist": train_dataset = datasets.MNIST(root=root, download=True, train=True, transform=transforms.Compose([ transforms.Resize(28), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.5], [0.5]), ])) train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=8) test_dataset = datasets.MNIST(root=root, download=True, train=False, transform=transforms.Compose([ transforms.Resize(28), transforms.ToTensor(), transforms.Normalize([0.5], [0.5]), ])) test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=8) return train_dataloader, test_dataloader elif name == "fmnist": train_dataset = datasets.FashionMNIST(root=root, download=True, train=True, transform=transforms.Compose([ transforms.Resize(28), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.5], [0.5]), ])) train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=8) test_dataset = datasets.FashionMNIST(root=root, download=True, train=False, transform=transforms.Compose([ transforms.Resize(28), transforms.ToTensor(), transforms.Normalize([0.5], [0.5]), ])) test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=8) return train_dataloader, test_dataloader elif name == "kmnist": train_dataset = datasets.KMNIST(root=root, download=True, train=True, transform=transforms.Compose([ transforms.Resize(28), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.5], [0.5]), ])) train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=8) test_dataset = datasets.KMNIST(root=root, download=True, train=False, transform=transforms.Compose([ transforms.Resize(28), transforms.ToTensor(), transforms.Normalize([0.5], [0.5]), ])) test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=8) return train_dataloader, test_dataloader elif name == "qmnist": train_dataset = datasets.QMNIST(root=root, download=True, train=True, transform=transforms.Compose([ transforms.Resize(28), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize([0.5], [0.5]), ])) train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=8) test_dataset = datasets.QMNIST(root=root, download=True, what="test50k", train=False, transform=transforms.Compose([ transforms.Resize(28), transforms.ToTensor(), transforms.Normalize([0.5], [0.5]), ])) test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=8) return train_dataloader, test_dataloader elif name == "cifar10": train_dataset = datasets.CIFAR10(root=root, download=True, train=True, transform=transforms.Compose([ transforms.Resize(32), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), ])) train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=8) test_dataset = datasets.CIFAR10(root=root, download=True, train=False, transform=transforms.Compose([ transforms.Resize(32), transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), ])) test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=8) return train_dataloader, test_dataloader elif name == "cifar100": train_dataset = datasets.CIFAR100(root=root, download=True, train=True, transform=transforms.Compose([ transforms.Resize(32), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), ])) train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=8) test_dataset = datasets.CIFAR100(root=root, download=True, train=False, transform=transforms.Compose([ transforms.Resize(32), transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), ])) test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=8) return train_dataloader, test_dataloader