def get_datasets(name, root, cutout):
# Mean + Std
if name == 'cifar10':
mean = [x / 255 for x in [125.3, 123.0, 113.9]]
std = [x / 255 for x in [63.0, 62.1, 66.7]]
elif name == 'cifar100':
mean = [x / 255 for x in [129.3, 124.1, 112.4]]
std = [x / 255 for x in [68.2, 65.4, 70.4]]
elif name == 'tiered':
mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
elif name == 'imagenet-1k' or name == 'imagenet-100':
mean, std = [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]
else: raise TypeError("Unknow dataset : {:}".format(name))
# Data Argumentation
if name == 'cifar10' or name == 'cifar100':
lists = [transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, padding=4), transforms.ToTensor(),
transforms.Normalize(mean, std)]
if cutout > 0 : lists += [Cutout(cutout)]
train_transform = transforms.Compose(lists)
test_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean, std)])
elif name == 'tiered':
lists = [transforms.RandomHorizontalFlip(), transforms.RandomCrop(80, padding=4), transforms.ToTensor(), transforms.Normalize(mean, std)]
if cutout > 0 : lists += [Cutout(cutout)]
train_transform = transforms.Compose(lists)
test_transform = transforms.Compose([transforms.CenterCrop(80), transforms.ToTensor(), transforms.Normalize(mean, std)])
elif name == 'imagenet-1k' or name == 'imagenet-100':
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
train_transform = transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ColorJitter(
brightness=0.4,
contrast=0.4,
saturation=0.4,
hue=0.2),
transforms.ToTensor(),
normalize,
])
test_transform = transforms.Compose([transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize])
else: raise TypeError("Unknow dataset : {:}".format(name))
if name == 'cifar10':
train_data = dset.CIFAR10 (root, train=True , transform=train_transform, download=True)
test_data = dset.CIFAR10 (root, train=False, transform=test_transform , download=True)
elif name == 'cifar100':
train_data = dset.CIFAR100(root, train=True , transform=train_transform, download=True)
test_data = dset.CIFAR100(root, train=False, transform=test_transform , download=True)
elif name == 'imagenet-1k' or name == 'imagenet-100':
train_data = dset.ImageFolder(osp.join(root, 'train'), train_transform)
test_data = dset.ImageFolder(osp.join(root, 'val'), test_transform)
else: raise TypeError("Unknow dataset : {:}".format(name))
class_num = Dataset2Class[name]
return train_data, test_data, class_num
def LoadDataset(split,
data_path,
n_jobs=2,
train_set='train',
dev_set='val',
batch_size=8,
cuda=True):
if split == 'train':
shuffle = True
dataset_file = train_set + '.csv'
transform = transforms.Compose([
RandomErasing(),
RandomHorizontalFlip(),
Cutout(),
Normaliztion()
])
# transform = transforms.Compose([RandomErasing(), RandomHorizontalFlip(), Cutout(), transforms.Resize(400), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
else:
shuffle = False
dataset_file = dev_set + '.csv'
transform = transforms.Compose([Normaliztion()])
# transform = transforms.Compose([transforms.Resize(400), transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
ds = TrainingDataset(os.path.join(data_path, dataset_file), transform)
return DataLoader(ds,
batch_size=batch_size,
shuffle=shuffle,
drop_last=False,
num_workers=n_jobs,
pin_memory=cuda)
def __init__(self, ROOTDIR, bCutOut=True, cutout_length=16, numberworks=1, dataset_name='cifar10'):
self.classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')
self.datasetNumberClass = 10
self.numberwork = numberworks
self.bCutOut = bCutOut
print('==> Preparing data..')
CIFAR_MEAN = [0.49139968, 0.48215827, 0.44653124]
CIFAR_STD = [0.24703233, 0.24348505, 0.26158768]
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(CIFAR_MEAN, CIFAR_STD),
#
])
if self.bCutOut:
transform_train.transforms.append(Cutout(cutout_length))
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(CIFAR_MEAN, CIFAR_STD),
])
#划分数据
if dataset_name=='cifar100':
self.trainset = torchvision.datasets.CIFAR100(root=ROOTDIR, train=True, download=True, transform=transform_train)
self.testset = torchvision.datasets.CIFAR100(root=ROOTDIR, train=False, download=True, transform=transform_test)
else:
self.trainset = torchvision.datasets.CIFAR10(root=ROOTDIR, train=True, download=True, transform=transform_train)
self.testset = torchvision.datasets.CIFAR10(root=ROOTDIR, train=False, download=True, transform=transform_test)
def load_dataset():
if args.dataset == 'cifar10':
mean, std = [x / 255 for x in [125.3, 123.0, 113.9]], [x / 255 for x in [63.0, 62.1, 66.7]]
dataset = dset.CIFAR10
num_classes = 10
elif args.dataset == 'cifar100':
mean, std = [x / 255 for x in [129.3, 124.1, 112.4]], [x / 255 for x in [68.2, 65.4, 70.4]]
dataset = dset.CIFAR100
num_classes = 100
elif args.dataset != 'imagenet': assert False, "Unknown dataset : {}".format(args.dataset)
if args.dataset == 'cifar10' or args.dataset == 'cifar100':
train_transform = transforms.Compose([transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, padding=4), transforms.ToTensor(), transforms.Normalize(mean, std)])
if args.cutout: train_transform.transforms.append(Cutout(n_holes=1, length=16))
test_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean, std)])
train_data = dataset(args.data_path, train=True, transform=train_transform, download=True)
train_loader = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True)
test_data = dataset(args.data_path, train=False, transform=test_transform, download=True)
test_loader = torch.utils.data.DataLoader(test_data, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True)
elif args.dataset == 'imagenet':
import imagenet_seq
train_loader = imagenet_seq.data.Loader('train', batch_size=args.batch_size, num_workers=args.workers)
test_loader = imagenet_seq.data.Loader('val', batch_size=args.batch_size, num_workers=args.workers)
num_classes = 1000
else: assert False, 'Do not support dataset : {}'.format(args.dataset)
return num_classes, train_loader, test_loader
def build_training_data_loader(self) -> DataLoader:
bucket_name = self.data_config["bucket_name"]
normalize = transforms.Normalize(
mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
)
train_transforms = transforms.Compose(
[
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ColorJitter(
brightness=0.4, contrast=0.4, saturation=0.4, hue=0.2
),
transforms.ToTensor(),
normalize,
]
)
if self.context.get_hparam("cutout"):
train_transforms.transforms.append(
Cutout(self.context.get_hparam("cutout_length"))
)
if self.context.get_hparam("autoaugment"):
train_transforms.transforms.insert(0, AutoAugment)
train_data = ImageNetDataset(
"train",
bucket_name,
streaming=self.data_config["streaming"],
data_download_dir=self.data_config["data_download_dir"],
transform=train_transforms,
)
train_queue = DataLoader(
train_data,
batch_size=self.context.get_per_slot_batch_size(),
shuffle=True,
pin_memory=True,
num_workers=self.data_config["num_workers_train"],
)
return train_queue
help='name of dataset: CIFAR10 or CUB or VOCPart')
parser.add_argument('--np_save', type=str, default='F', metavar='N',
help='name of T or F')
parser.add_argument('--model_type', type=str, default='ex_gradcam5', metavar='N',
help='norm, atten, ex or ex_atten or ex_gradcam or ex_gradcam5')
parser.add_argument('--model', type=str, default='vgg', metavar='N',
help='vgg or resnet or mobilenet or alexnet')
parser.add_argument('--model_init', action='store_true', default=False, help='use Xavier Initialization')
parser.add_argument('--cutout', action='store_true', default=False, help='use cutout')
parser.add_argument('--cutout_length', type=int, default=64, help='cutout length')
args = parser.parse_args()
if args.cutout:
transform_train.transforms.append(Cutout(args.cutout_length))
'''下载训练集 '''
if args.dataset == 'CIFAR10':
train_dataset = datasets.CIFAR10('./data/CIFAR10', train=True, transform=transform_train, download=True)
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
test_dataset = datasets.CIFAR10('./data/CIFAR10', train=False, transform=transform_test, download=True)
test_loader = DataLoader(test_dataset, batch_size=batch_size_t, shuffle=False)
num_classe = 10
weight_folder = './model/CIFAR10/'
title = 'CIFAR-10-' + 'vgg16'
elif args.dataset == 'CUB':
train_dataset = CUBDataset('./data/CUB', train=True, transform=transform_train, download=True)
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
test_dataset = CUBDataset('./data/CUB', train=False, transform=transform_test, download=True)
# Set the random seed manually for reproducibility.
use_cuda = torch.cuda.is_available()
torch.manual_seed(args.seed)
if use_cuda:
torch.cuda.manual_seed(args.seed)
###############################################################################
# Load data
###############################################################################
print('==> Preparing data..')
if args.dataset=='mnist':
trans = ([transforms.ToTensor()])
if args.cutout:
trans += [Cutout(n_holes=1, length=8)]
trans = transforms.Compose(trans)
fulltrainset = torchvision.datasets.MNIST(root=args.data, train=True, transform=trans, download=True)
train_set, valset = _split_train_val(fulltrainset, val_fraction=0.1)
trainloader = torch.utils.data.DataLoader(train_set, batch_size=args.mbs, shuffle=True,
num_workers=0, pin_memory=True)
validloader = torch.utils.data.DataLoader(valset, batch_size=args.mbs, shuffle=False,
num_workers=0, pin_memory=True)
test_set = torchvision.datasets.MNIST(root=args.data, train=False, transform=trans)
testloader = torch.utils.data.DataLoader(test_set, batch_size=args.mbs, shuffle=False, num_workers=0)
def cutout_img(self, img):
cut_out = Cutout(n_holes=1, length=40)
img = cut_out(img)
return img
def __init__(self, dataset_name, validation=True, verbose=True,
img_size=32, batchsize=128, data_num=500, mode="full",
config=None):
self.verbose = verbose
self.img_size = img_size
self.validation = validation
self.batchsize = batchsize
self.dataset_name = dataset_name
self.data_num = data_num
self.mode = mode
self.config = config
# load dataset
if dataset_name == 'cifar10' or dataset_name == 'tinyimagenet':
if dataset_name == 'cifar10':
self.n_class = 10
self.channel = 3
if self.validation:
# TODO: are we actually testing on the validation set here!?
self.dataloader, self.test_dataloader = get_train_valid_loader(
data_dir='./', batch_size=self.batchsize, augment=True,
random_seed=2018, num_workers=1, pin_memory=True,
data_num=self.data_num)
else:
train_dataset = dset.CIFAR10(root='./', train=True, download=True,
transform=transforms.Compose([
transforms.RandomCrop(
32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(
(0.49139968, 0.48215827, 0.44653124), (0.24703233, 0.24348505, 0.26158768)),
Cutout(16),
]))
test_dataset = dset.CIFAR10(root='./', train=False, download=True,
transform=transforms.Compose([
# transforms.Scale(self.img_size),
transforms.ToTensor(),
transforms.Normalize(
(0.49139968, 0.48215827, 0.44653124), (0.24703233, 0.24348505, 0.26158768)),
]))
self.dataloader = torch.utils.data.DataLoader(
train_dataset, batch_size=self.batchsize, shuffle=True,
num_workers=int(4), drop_last=True)
self.test_dataloader = torch.utils.data.DataLoader(
test_dataset, batch_size=self.batchsize, shuffle=True,
num_workers=int(4), drop_last=True)
elif dataset_name == 'tinyimagenet':
self.n_class = 200
self.channel = 3
if self.validation:
self.dataloader, self.test_dataloader = get_train_valid_loader_tinyimagenet(
data_dir='/home/suganuma/dataset/tiny-imagenet-200/train',
batch_size=self.batchsize, augment=True, random_seed=2018,
num_workers=4, pin_memory=False, data_num=self.data_num)
else:
if self.mode == "full":
transform_train = transforms.Compose([
transforms.RandomCrop(64, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(
(0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
Cutout(16), ])
trainset = torchvision.datasets.ImageFolder(
root='/home/suganuma/dataset/tiny-imagenet-200/train',
transform=transform_train)
self.dataloader = torch.utils.data.DataLoader(
trainset, batch_size=self.batchsize, shuffle=True,
num_workers=8, drop_last=True)
else:
self.dataloader, _ = get_train_valid_loader_tinyimagenet(
data_dir='/home/suganuma/dataset/tiny-imagenet-200/train',
batch_size=self.batchsize, augment=True,
random_seed=2018, num_workers=4, pin_memory=False,
data_num=self.data_num)
print("train num", self.data_num)
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), ])
testset = torchvision.datasets.ImageFolder(
root='/home/suganuma/dataset/tiny-imagenet-200/val',
transform=transform_test)
self.test_dataloader = torch.utils.data.DataLoader(
testset, batch_size=self.batchsize, shuffle=False,
num_workers=4, drop_last=True)
else:
print('\tInvalid input dataset name at CNN_train()')
exit(1)
def main():
global best_acc
global learn
start_epoch = 0
# Data
print('Getting the Dataset %s' % args.dataset)
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
if args.cutout:
transform_train.transforms.append(
Cutout(n_holes=args.n_holes, length=args.length))
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
dataloader = datasets.CIFAR10
trainset = dataloader(root='D:/resden_final/CIFAR/',
train=True,
download=True,
transform=transform_train)
trainloader = data.DataLoader(trainset,
batch_size=64,
shuffle=True,
num_workers=4)
testset = dataloader(root='D:/resden_final/CIFAR/',
train=False,
download=False,
transform=transform_test)
testloader = data.DataLoader(testset,
batch_size=100,
shuffle=False,
num_workers=4)
print("Model '{}'".format(args.arch))
model = models.__dict__[args.arch](
num_classes=10,
depth=args.depth,
k1=12,
k2=12,
dropRate=args.dropo,
)
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
summary(model, (3, 32, 32))
model_size = (sum(p.numel() for p in model.parameters()) / 1000000.0)
print('Total Number of Parameters: %.2f Million' % model_size)
with open('D:/resden_final/modelsummary.txt', 'w') as f:
with redirect_stdout(f):
summary(model, (3, 32, 32))
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=learn,
momentum=0.9,
weight_decay=1e-4)
if args.evaluate:
print('\nEvaluation only')
test_loss, test_acc = test(testloader, model, criterion, start_epoch,
use_cuda)
print(' Test Loss: %.8f, Test Accuracy: %.2f' %
(test_loss, test_acc))
return
tra = []
tea = []
trl = []
tel = []
# Train and val
for epoch in range(start_epoch, 300):
change_lr(optimizer, epoch)
print('\nEpoch: [%d | %d] LR: %f' % (epoch + 1, 300, learn))
train_loss, train_acc = train(trainloader, model, criterion, optimizer,
epoch, use_cuda)
test_loss, test_acc = test(testloader, model, criterion, epoch,
use_cuda)
tra.append(train_acc)
tea.append(test_acc)
trl.append(train_loss)
tel.append(test_loss)
# save model
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
print('Best acc:')
print(best_acc)
plt.figure(1)
plt.plot(tra)
plt.title('Training Accuracy vs Epochs')
plt.ylabel('Accuracy')
plt.xlabel('Epochs')
plt.savefig('D:/resden_final/train_acc.png')
plt.figure(2)
plt.plot(tea)
plt.title('Testing Accuracy vs Epochs')
plt.ylabel('Accuracy')
plt.xlabel('Epochs')
plt.savefig('D:/resden_final/test_acc.png')
plt.figure(3)
plt.plot(trl)
plt.title('Training Loss vs Epochs')
plt.ylabel('Loss')
plt.xlabel('Epochs')
plt.savefig('D:/resden_final/train_loss.png')
plt.figure(4)
plt.plot(tel)
plt.title('Testing Loss vs Epochs')
plt.ylabel('Loss')
plt.xlabel('Epochs')
plt.savefig('D:/resden_final/test_loss.png')
def main():
os.makedirs(SAVEPATH, exist_ok=True)
print('save path:', SAVEPATH)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print('device:', device)
print('weight_decay:', WEIGHTDECAY)
print('momentum:', MOMENTUM)
print('batch_size:', BATCHSIZE)
print('lr:', LR)
print('epoch:', EPOCHS)
print('Label smoothing:', LABELSMOOTH)
print('Stochastic Weight Averaging:', SWA)
if SWA:
print('Swa lr:', SWA_LR)
print('Swa start epoch:', SWA_START)
print('Cutout augmentation:', CUTOUT)
if CUTOUT:
print('Cutout size:', CUTOUTSIZE)
print('Activation:', ACTIVATION)
# get model
model = get_seresnet_cifar(activation=ACTIVATION)
# get loss function
if LABELSMOOTH:
criterion = LabelSmoothingLoss(classes=10, smoothing=0.1)
else:
criterion = torch.nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(),
lr=LR,
momentum=MOMENTUM,
weight_decay=WEIGHTDECAY,
nesterov=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer=optimizer,
T_max=EPOCHS,
eta_min=0)
model = model.to(device)
criterion = criterion.to(device)
# Check number of parameters your model
pytorch_total_params = sum(p.numel() for p in model.parameters())
print(f"Number of parameters: {pytorch_total_params}")
if int(pytorch_total_params) > 2000000:
print('Your model has the number of parameters more than 2 millions..')
return
if SWA:
# apply swa
swa_model = AveragedModel(model)
swa_scheduler = SWALR(optimizer, swa_lr=SWA_LR)
swa_total_params = sum(p.numel() for p in swa_model.parameters())
print(f"Swa parameters: {swa_total_params}")
# cinic mean, std
normalize = transforms.Normalize(mean=[0.47889522, 0.47227842, 0.43047404],
std=[0.24205776, 0.23828046, 0.25874835])
if CUTOUT:
train_transform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(), normalize,
Cutout(size=CUTOUTSIZE)
])
else:
train_transform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(), normalize
])
train_dataset = torchvision.datasets.ImageFolder('/content/train',
transform=train_transform)
train_loader = DataLoader(train_dataset,
batch_size=BATCHSIZE,
shuffle=True,
num_workers=4,
pin_memory=True)
# colab reload
start_epoch = 0
if os.path.isfile(os.path.join(SAVEPATH, 'latest_checkpoint.pth')):
checkpoint = torch.load(os.path.join(SAVEPATH,
'latest_checkpoint.pth'))
start_epoch = checkpoint['epoch']
scheduler.load_state_dict(checkpoint['scheduler'])
model.load_state_dict(checkpoint['model'])
optimizer.load_state_dict(checkpoint['optimizer'])
if SWA:
swa_scheduler.load_state_dict(checkpoint['swa_scheduler'])
swa_model.load_state_dict(checkpoint['swa_model'])
print(start_epoch, 'load parameter')
for epoch in range(start_epoch, EPOCHS):
print("\n----- epoch: {}, lr: {} -----".format(
epoch, optimizer.param_groups[0]["lr"]))
# train for one epoch
start_time = time.time()
train(train_loader, epoch, model, optimizer, criterion, device)
elapsed_time = time.time() - start_time
print('==> {:.2f} seconds to train this epoch\n'.format(elapsed_time))
# learning rate scheduling
if SWA and epoch > SWA_START:
swa_model.update_parameters(model)
swa_scheduler.step()
else:
scheduler.step()
if SWA:
checkpoint = {
'epoch': epoch + 1,
'model': model.state_dict(),
'optimizer': optimizer.state_dict(),
'scheduler': scheduler.state_dict(),
'swa_model': swa_model.state_dict(),
'swa_scheduler': swa_scheduler.state_dict()
}
else:
checkpoint = {
'epoch': epoch + 1,
'model': model.state_dict(),
'optimizer': optimizer.state_dict(),
'scheduler': scheduler.state_dict()
}
torch.save(checkpoint, os.path.join(SAVEPATH, 'latest_checkpoint.pth'))
if epoch % 10 == 0:
torch.save(checkpoint,
os.path.join(SAVEPATH, '%d_checkpoint.pth' % epoch))
def getDataloaders(data, config_of_data, splits=['train', 'val', 'test'],
aug=True, use_validset=True, data_root='data', batch_size=64, normalized=True,
data_aug=False, cutout=False, n_holes=1, length=16,
num_workers=3, **kwargs):
train_loader, val_loader, test_loader = None, None, None
if data.find('cifar10') >= 0:
print('loading ' + data)
print(config_of_data)
if data.find('cifar100') >= 0:
d_func = dset.CIFAR100
normalize = transforms.Normalize(mean=[0.5071, 0.4867, 0.4408],
std=[0.2675, 0.2565, 0.2761])
else:
d_func = dset.CIFAR10
normalize = transforms.Normalize(mean=[0.4914, 0.4824, 0.4467],
std=[0.2471, 0.2435, 0.2616])
if data_aug:
print('with data augmentation')
aug_trans = [
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
]
else:
aug_trans = []
common_trans = [transforms.ToTensor()]
if normalized:
print('dataset is normalized')
common_trans.append(normalize)
train_compose = aug_trans + common_trans
if cutout:
train_compose.append(Cutout(n_holes=n_holes, length=length))
train_compose = transforms.Compose(train_compose)
test_compose = transforms.Compose(common_trans)
if use_validset:
# uses last 5000 images of the original training split as the
# validation set
if 'train' in splits:
train_set = d_func(data_root, train=True, transform=train_compose,
download=True)
train_loader = torch.utils.data.DataLoader(
train_set, batch_size=batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
range(45000)),
num_workers=num_workers, pin_memory=True)
if 'val' in splits:
val_set = d_func(data_root, train=True, transform=test_compose)
val_loader = torch.utils.data.DataLoader(
val_set, batch_size=batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
range(45000, 50000)),
num_workers=num_workers, pin_memory=True)
if 'test' in splits:
test_set = d_func(data_root, train=False, transform=test_compose)
test_loader = torch.utils.data.DataLoader(
test_set, batch_size=batch_size, shuffle=True,
num_workers=num_workers, pin_memory=True)
else:
if 'train' in splits:
train_set = d_func(data_root, train=True, transform=train_compose,
download=True)
train_loader = torch.utils.data.DataLoader(
train_set, batch_size=batch_size, shuffle=True,
num_workers=num_workers, pin_memory=True)
if 'val' in splits or 'test' in splits:
test_set = d_func(data_root, train=False, transform=test_compose)
test_loader = torch.utils.data.DataLoader(
test_set, batch_size=batch_size, shuffle=True,
num_workers=num_workers, pin_memory=True)
val_loader = test_loader
else:
raise NotImplemented
return train_loader, val_loader, test_loader
def main():
print('save path:', SAVEPATH)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print('device:', device)
print('weight_decay:', WEIGHTDECAY)
print('momentum:', MOMENTUM)
print('batch_size:', BATCHSIZE)
print('lr:', LR)
print('epoch:', EPOCHS)
print('Label smoothing:', LABELSMOOTH)
print('Stochastic Weight Averaging:', SWA)
if SWA:
print('Swa lr:', SWA_LR)
print('Swa start epoch:', SWA_START)
print('Cutout augmentation:', CUTOUT)
if CUTOUT:
print('Cutout size:', CUTOUTSIZE)
print('Activation:', ACTIVATION)
model = get_seresnet_cifar(activation=ACTIVATION)
if SWA:
swa_model = AveragedModel(model)
normalize = transforms.Normalize(mean=[0.47889522, 0.47227842, 0.43047404],
std=[0.24205776, 0.23828046, 0.25874835])
if SWA:
if CUTOUT:
train_transform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(), normalize,
Cutout(size=CUTOUTSIZE)
])
else:
train_transform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(), normalize
])
train_dataset = torchvision.datasets.ImageFolder(
'/content/train', transform=train_transform)
train_loader = DataLoader(train_dataset,
batch_size=BATCHSIZE,
shuffle=True,
num_workers=4,
pin_memory=True)
print('swa data ready')
if os.path.isfile(os.path.join(SAVEPATH, 'latest_checkpoint.pth')):
checkpoint = torch.load(os.path.join(SAVEPATH,
'latest_checkpoint.pth'))
model.load_state_dict(checkpoint['model'])
model = model.to(device)
model.eval()
if SWA:
print('swa update batch norm')
swa_model.load_state_dict(checkpoint['swa_model'])
swa_model = swa_model.to(device)
torch.optim.swa_utils.update_bn(train_loader, swa_model, device)
swa_model.eval()
print('model ready')
test_transform = transforms.Compose([transforms.ToTensor(), normalize])
test_dataset = TestImageFolder('/content/test', transform=test_transform)
test_loader = DataLoader(test_dataset,
batch_size=BATCHSIZE,
num_workers=4,
shuffle=False)
print('test data ready')
print('Make an evaluation csv file for kaggle submission...')
Category = []
Id = []
with torch.no_grad():
for data in test_loader:
(input, _), name = data
input = input.to(device)
output = swa_model(input)
output = torch.argmax(output, dim=1)
Id = Id + list(name)
Category = Category + output.tolist()
#Id = list(range(0, 90000))
samples = {'Id': Id, 'Target': Category}
df = pd.DataFrame(samples, columns=['Id', 'Target'])
df.to_csv(os.path.join(SAVEPATH, 'submission.csv'), index=False)
print('Done!!')
base_learning_rate = args.base_lr * args.batchsize / 128.
if use_cuda:
# data parallel
n_gpu = torch.cuda.device_count()
# Data
print('==> Preparing data..')
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
if args.cutout:
transform_train.transforms.append(Cutout(n_holes=1, length=16))
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
])
if args.dataset.lower() == 'cifar10':
dset_class = torchvision.datasets.CIFAR10
num_class = 10
elif args.dataset.lower() == 'cifar100':
dset_class = torchvision.datasets.CIFAR100
num_class = 100
trainset = dset_class(root='./data', train=True, download=True, transform=transform_train)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, shuffle=True)
def load_dataset():
if args.dataset == 'cifar10':
mean, std = [x / 255 for x in [125.3, 123.0, 113.9]
], [x / 255 for x in [63.0, 62.1, 66.7]]
dataset = dset.CIFAR10
num_classes = 10
elif args.dataset == 'cifar100':
mean, std = [x / 255 for x in [129.3, 124.1, 112.4]
], [x / 255 for x in [68.2, 65.4, 70.4]]
dataset = dset.CIFAR100
num_classes = 100
elif args.dataset != 'imagenet':
assert False, "Unknown dataset : {}".format(args.dataset)
if args.dataset == 'cifar10' or args.dataset == 'cifar100':
train_transform = transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.RandomCrop(32, padding=4),
transforms.ToTensor(),
transforms.Normalize(mean, std)
])
if args.cutout:
train_transform.transforms.append(Cutout(n_holes=1, length=16))
test_transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(mean, std)])
if args.evaluate:
train_data = dataset(args.data_path,
train=True,
transform=train_transform,
download=True)
test_data = dataset(args.data_path,
train=False,
transform=test_transform,
download=True)
train_loader = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
test_loader = torch.utils.data.DataLoader(
test_data,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
else:
# partition training set into two instead. note that test_data is defined using train=True
train_data = dataset(args.data_path,
train=True,
transform=train_transform,
download=True)
test_data = dataset(args.data_path,
train=True,
transform=test_transform,
download=True)
indices = list(range(len(train_data)))
np.random.shuffle(indices)
split = int(0.9 * len(train_data))
train_indices, test_indices = indices[:split], indices[split:]
train_sampler = SubsetRandomSampler(train_indices)
test_sampler = SubsetRandomSampler(test_indices)
train_loader = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
num_workers=args.workers,
pin_memory=True,
sampler=train_sampler)
test_loader = torch.utils.data.DataLoader(
test_data,
batch_size=args.batch_size,
num_workers=args.workers,
pin_memory=True,
sampler=test_sampler)
elif args.dataset == 'imagenet':
import imagenet_seq
train_loader = imagenet_seq.data.Loader('train',
batch_size=args.batch_size,
num_workers=args.workers)
test_loader = imagenet_seq.data.Loader('val',
batch_size=args.batch_size,
num_workers=args.workers)
num_classes = 1000
else:
assert False, 'Do not support dataset : {}'.format(args.dataset)
return num_classes, train_loader, test_loader
if opt.erasing_p > 0:
print('use random eraseing, p={}'.format(opt.erasing_p))
transform_train_list = transform_train_list + [
RandomErasing(probability=opt.erasing_p, mean=[0.0, 0.0, 0.0])
]
if opt.color_jitter:
transform_train_list = [
transforms.ColorJitter(
brightness=0.1, contrast=0.1, saturation=0.1, hue=0)
] + transform_train_list
if opt.cutout:
print('use cutout')
transform_train_list = transform_train_list + [
Cutout(n_holes=1, length=16)
]
data_transforms = {
'train': transforms.Compose(transform_train_list),
'val': transforms.Compose(transform_val_list),
}
train_all = ''
if opt.train_all:
train_all = '_all'
print('train all.')
image_datasets = {}
image_datasets['train'] = datasets.ImageFolder(
os.path.join(data_dir, 'train' + train_all), data_transforms['train'])
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
logging_file = 'ShakeShake_SSI_%s_2x%s.log' % (opt.depth, opt.width)
filehandler = logging.FileHandler(logging_file)
streamhandler = logging.StreamHandler()
logger = logging.getLogger('')
logger.setLevel(logging.INFO)
logger.addHandler(filehandler)
logger.addHandler(streamhandler)
logger.info(opt)
use_cutout = opt.cutout
cutout = Cutout() if use_cutout else None
normalizer = transforms.Normalize(mean=[0.4914, 0.4824, 0.4467],
std=[0.2471, 0.2435, 0.2616])
def transformer(im):
im = np.array(im)
im = np.pad(im, pad_width=((4, 4), (4, 4), (0, 0)), mode='constant')
if use_cutout:
im = cutout(im)
im = Image.fromarray(im)
auglist = transforms.Compose([
transforms.RandomCrop(32),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalizer