def main():
torch.set_num_threads(3)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
if args.dataset == 'cifar10':
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
elif args.dataset == 'cifar100':
train_transform, valid_transform = utils._data_transforms_cifar100(args)
train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR100(root=args.data, train=False, download=True, transform=valid_transform)
elif args.dataset == 'svhn':
train_transform, valid_transform = utils._data_transforms_svhn(args)
train_data = dset.SVHN(root=args.data, split='train', download=True, transform=train_transform)
valid_data = dset.SVHN(root=args.data, split='test', download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True)
if args.save == 'darts_0':
save = 'search-baseline-20200223-215626-0-6930'
elif args.save == 'darts_1':
save = 'search-baseline-20200223-215626-1-859'
elif args.save == 'random_0':
save = 'search-baseline-20200223-215626-0-alpha-random-0.3-2195'
elif args.save == 'random_1':
save = 'search-baseline-20200223-215626-1-alpha-random-0.3-529'
elif args.save == 'pgd_0':
save = 'search-baseline-20200223-215626-0-alpha-pgd_linf-0.3-6660'
elif args.save == 'pgd_1':
save = 'search-baseline-20200223-215626-1-alpha-pgd_linf-0.3-5746'
save = os.path.join(
'/home/xiangning/SmoothDARTS/experiments/nasbench201/cifar10', save)
api = API('/nfs/data/xiangning/data/NAS-Bench-201-v1_0-e61699.pth')
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(C=16, N=5, max_nodes=4, num_classes=10, criterion=criterion).cuda()
model = load_checkpoint(model, save)
acc_oneshots, acc_trues= get_accuracies(model, api, valid_queue, criterion, 100)
tau = get_kendalltau(acc_oneshots, acc_trues)
print(acc_oneshots[:100])
print(acc_trues[:100])
print(tau)
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
utils.load(model, args.model_path)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
_, test_transform = utils._data_transforms_cifar10(args)
test_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=test_transform)
test_queue = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
model.drop_path_prob = args.drop_path_prob
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
def __init__(self, save_path, seed, batch_size, grad_clip, epochs, num_intermediate_nodes, search_space, cutout,
resume_iter=None, init_channels=16):
args = {}
args['data'] = '../data'
args['epochs'] = epochs
args['learning_rate'] = 0.025
args['batch_size'] = batch_size
args['learning_rate_min'] = 0.001
args['momentum'] = 0.9
args['weight_decay'] = 3e-4
args['init_channels'] = init_channels
# Adapted to nasbench
args['layers'] = 9
args['drop_path_prob'] = 0.3
args['grad_clip'] = grad_clip
args['train_portion'] = 0.5
args['seed'] = seed
args['log_interval'] = 50
args['save'] = save_path
args['gpu'] = 0
args['cuda'] = True
args['cutout'] = cutout
args['cutout_length'] = 16
args['report_freq'] = 50
args['output_weights'] = True
args['steps'] = num_intermediate_nodes
args['search_space'] = search_space.search_space_number
self.search_space = search_space
args = AttrDict(args)
self.args = args
# Dump the config of the run, but if only if it doesn't yet exist
config_path = os.path.join(args.save, 'config.json')
if not os.path.exists(config_path):
with open(config_path, 'w') as fp:
json.dump(args.__dict__, fp)
self.seed = seed
np.random.seed(args.seed)
random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = False
cudnn.enabled = True
cudnn.deterministic = True
torch.cuda.manual_seed_all(args.seed)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
self.train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True, num_workers=0, worker_init_fn=np.random.seed(args.seed))
self.valid_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]),
pin_memory=True, num_workers=0, worker_init_fn=np.random.seed(args.seed))
_, test_transform = utils._data_transforms_cifar10(args)
test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform)
self.test_queue = torch.utils.data.DataLoader(
test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
self.train_iter = iter(self.train_queue)
self.valid_iter = iter(self.valid_queue)
self.steps = 0
self.epochs = 0
self.total_loss = 0
self.start_time = time.time()
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
self.criterion = criterion
model = Network(args.init_channels, 10, args.layers, self.criterion, output_weights=args.output_weights,
search_space=search_space, steps=args.steps)
model = model.cuda()
self.model = model
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(
self.model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
self.optimizer = optimizer
self.scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
if resume_iter is not None:
self.steps = resume_iter
self.epochs = int(resume_iter / len(self.train_queue))
logging.info("Resuming from epoch %d" % self.epochs)
self.objs = utils.AvgrageMeter()
self.top1 = utils.AvgrageMeter()
self.top5 = utils.AvgrageMeter()
for i in range(self.epochs):
self.scheduler.step()
size = 0
for p in model.parameters():
size += p.nelement()
logging.info('param size: {}'.format(size))
total_params = sum(x.data.nelement() for x in model.parameters())
logging.info('Args: {}'.format(args))
logging.info('Model total parameters: {}'.format(total_params))
def main():
torch.set_num_threads(3)
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels, n_classes, args.layers, criterion,
spaces_dict[args.search_space])
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
model2 = ResNet18()
model2 = model2.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
optimizer2 = torch.optim.SGD(model2.parameters(),
args.learning_rate2,
momentum=args.momentum,
weight_decay=args.weight_decay2)
if args.dataset == 'cifar10':
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
elif args.dataset == 'cifar100':
train_transform, valid_transform = utils._data_transforms_cifar100(
args)
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
elif args.dataset == 'svhn':
train_transform, valid_transform = utils._data_transforms_svhn(args)
train_data = dset.SVHN(root=args.data,
split='train',
download=True,
transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
if 'debug' in args.save:
split = args.batch_size
num_train = 2 * args.batch_size
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
scheduler2 = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer2, float(args.epochs), eta_min=args.learning_rate_min)
# architect = Architect(model, args)
architect = Architect(model2, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
lr2 = scheduler2.get_lr()[0]
if args.cutout:
# increase the cutout probability linearly throughout search
train_transform.transforms[
-1].cutout_prob = args.cutout_prob * epoch / (args.epochs - 1)
logging.info('epoch %d lr %e cutout_prob %e', epoch, lr,
train_transform.transforms[-1].cutout_prob)
else:
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
print(F.softmax(model.alphas_normal, dim=-1))
print(F.softmax(model.alphas_reduce, dim=-1))
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer,
optimizer2, lr, lr2, model2, epoch)
logging.info('train_acc %f', train_acc)
writer.add_scalar('Acc/train', train_acc, epoch)
writer.add_scalar('Obj/train', train_obj, epoch)
# validation
valid_acc, valid_obj = infer(valid_queue, model2, criterion)
logging.info('valid_acc %f', valid_acc)
writer.add_scalar('Acc/valid', valid_acc, epoch)
writer.add_scalar('Obj/valid', valid_obj, epoch)
utils.save(model, os.path.join(args.save, 'weights.pt'))
writer.close()
def main():
torch.set_num_threads(3)
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
if args.perturb_alpha == 'none':
perturb_alpha = None
elif args.perturb_alpha == 'pgd_linf':
perturb_alpha = Linf_PGD_alpha
elif args.perturb_alpha == 'random':
perturb_alpha = Random_alpha
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
#######################################
resnet18 = models.resnet18(num_classes=10)
# torch.cuda.clear_memory_allocated()
# del Variables
# gc.collect()
# torch.cuda.empty_cache()
resnet18 = resnet18.cuda()
model2 = resnet18
######################################
model = Network(args.init_channels, n_classes, args.layers, criterion,
spaces_dict[args.search_space])
model = model.cuda()
model.load_state_dict(torch.load("weights-9.pt"))
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
if args.dataset == 'cifar10':
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
elif args.dataset == 'cifar100':
train_transform, valid_transform = utils._data_transforms_cifar100(
args)
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
elif args.dataset == 'svhn':
train_transform, valid_transform = utils._data_transforms_svhn(args)
train_data = dset.SVHN(root=args.data,
split='train',
download=True,
transform=train_transform)
# num_train = len(train_data)+24
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
# if 'debug' in args.save:
# split = args.batch_size
# num_train = 2 * args.batch_size
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True)
# train_queue = torch.utils.data.DataLoader(
# train_data, batch_size=args.batch_size,
# sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:5]),
# pin_memory=True)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
if args.cutout:
# increase the cutout probability linearly throughout search
train_transform.transforms[
-1].cutout_prob = args.cutout_prob * epoch / (args.epochs - 1)
logging.info('epoch %d lr %e cutout_prob %e', epoch, lr,
train_transform.transforms[-1].cutout_prob)
else:
logging.info('epoch %d lr %e', epoch, lr)
if args.perturb_alpha:
epsilon_alpha = 0.03 + (args.epsilon_alpha -
0.03) * epoch / args.epochs
logging.info('epoch %d epsilon_alpha %e', epoch, epsilon_alpha)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
print(F.softmax(model.alphas_normal, dim=-1))
print(F.softmax(model.alphas_reduce, dim=-1))
# training
# _, _, delta = train3(train_queue, valid_queue, model, architect, criterion, optimizer, lr,
# perturb_alpha, epsilon_alpha, model2, epoch)
# train_acc, train_obj = train4(train_queue, valid_queue, model, architect, criterion, optimizer, lr,
# perturb_alpha, epsilon_alpha, model2, epoch, delta)
# train_acc, train_obj = train2(train_queue, valid_queue, model, architect, criterion, optimizer, lr,
# perturb_alpha, epsilon_alpha, model2, epoch)
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr,
perturb_alpha, epsilon_alpha, model2,
epoch)
logging.info('train_acc %f', train_acc)
writer.add_scalar('Acc/train', train_acc, epoch)
writer.add_scalar('Obj/train', train_obj, epoch)
# validation
# valid_acc, valid_obj = infer(valid_queue, model, criterion)
############################################################################################################
valid_acc, valid_obj = infer(valid_queue, resnet18, criterion)
############################################################################################################
logging.info('valid_acc %f', valid_acc)
writer.add_scalar('Acc/valid', valid_acc, epoch)
writer.add_scalar('Obj/valid', valid_obj, epoch)
utils.save(model, os.path.join(args.save, 'weights.pt'))
# for epoch in range(args.epochs):
# scheduler.step()
# lr = scheduler.get_lr()[0]
# if args.cutout:
# # increase the cutout probability linearly throughout search
# train_transform.transforms[-1].cutout_prob = args.cutout_prob * epoch / (args.epochs - 1)
# logging.info('epoch %d lr %e cutout_prob %e', epoch, lr,
# train_transform.transforms[-1].cutout_prob)
# else:
# logging.info('epoch %d lr %e', epoch, lr)
# if args.perturb_alpha:
# epsilon_alpha = 0.03 + (args.epsilon_alpha - 0.03) * epoch / args.epochs
# logging.info('epoch %d epsilon_alpha %e', epoch, epsilon_alpha)
# genotype = model.genotype()
# logging.info('genotype = %s', genotype)
# print(F.softmax(model.alphas_normal, dim=-1))
# print(F.softmax(model.alphas_reduce, dim=-1))
# # training
# _, _, delta = train3(train_queue, valid_queue, model, architect, criterion, optimizer, lr,
# perturb_alpha, epsilon_alpha, model2, epoch)
# for epoch in range(args.epochs+150):
# train_acc, train_obj = train4(train_queue, valid_queue, model, architect, criterion, optimizer, lr,
# perturb_alpha, epsilon_alpha, model2, epoch, delta)
# # train_acc, train_obj = train2(train_queue, valid_queue, model, architect, criterion, optimizer, lr,
# # perturb_alpha, epsilon_alpha, model2, epoch)
# # train_acc, train_obj = train(train_queue, valid_queue, model, architect, criterion, optimizer, lr)
# logging.info('train_acc %f', train_acc)
# writer.add_scalar('Acc/train', train_acc, epoch)
# writer.add_scalar('Obj/train', train_obj, epoch)
# # validation
# # valid_acc, valid_obj = infer(valid_queue, model, criterion)
# ############################################################################################################
# valid_acc, valid_obj = infer(valid_queue, resnet18, criterion)
# ############################################################################################################
# logging.info('valid_acc %f', valid_acc)
# writer.add_scalar('Acc/valid', valid_acc, epoch)
# writer.add_scalar('Obj/valid', valid_obj, epoch)
# utils.save(model, os.path.join(args.save, 'weights.pt'))
writer.close()
def main():
torch.set_num_threads(3)
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, n_classes, args.layers, args.auxiliary,
genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
if args.dataset == 'cifar10':
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
elif args.dataset == 'cifar100':
train_transform, valid_transform = utils._data_transforms_cifar100(
args)
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR100(root=args.data,
train=False,
download=True,
transform=valid_transform)
elif args.dataset == 'svhn':
train_transform, valid_transform = utils._data_transforms_svhn(args)
train_data = dset.SVHN(root=args.data,
split='train',
download=True,
transform=train_transform)
valid_data = dset.SVHN(root=args.data,
split='test',
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
writer.add_scalar('Acc/train', train_acc, epoch)
writer.add_scalar('Obj/train', train_obj, epoch)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
writer.add_scalar('Acc/valid', valid_acc, epoch)
writer.add_scalar('Obj/valid', valid_obj, epoch)
utils.save(model, os.path.join(args.save, 'weights.pt'))
writer.close()
def main():
torch.set_num_threads(3)
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, n_classes, args.layers, args.auxiliary,
genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
if args.dataset == 'cifar10':
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
elif args.dataset == 'cifar100':
train_transform, valid_transform = utils._data_transforms_cifar100(
args)
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR100(root=args.data,
train=False,
download=True,
transform=valid_transform)
elif args.dataset == 'svhn':
train_transform, valid_transform = utils._data_transforms_svhn(args)
train_data = dset.SVHN(root=args.data,
split='train',
download=True,
transform=train_transform)
valid_data = dset.SVHN(root=args.data,
split='test',
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
best_acc_top1 = 0
best_acc_top5 = 0
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
if args.cutout:
# increase the cutout probability linearly throughout search
train_transform.transforms[-1].cutout_prob = args.cutout_prob * \
epoch / (args.epochs - 1)
logging.info('epoch %d lr %e cutout_prob %e', epoch, lr,
train_transform.transforms[-1].cutout_prob)
else:
logging.info('epoch %d lr %e', epoch, lr)
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
writer.add_scalar('Acc/train', train_acc, epoch)
writer.add_scalar('Obj/train', train_obj, epoch)
# valid_acc, valid_obj = infer(valid_queue, model, criterion)
valid_acc_top1, valid_acc_top5, valid_obj = infer(
valid_queue, model, criterion)
# logging.info('Valid_acc_top1: %f', valid_acc_top1)
logging.info('valid_acc %f', valid_acc_top1)
writer.add_scalar('Acc/valid', valid_acc_top1, epoch)
writer.add_scalar('Obj/valid', valid_obj, epoch)
is_best = False
if valid_acc_top5 > best_acc_top5:
best_acc_top5 = valid_acc_top5
if valid_acc_top1 > best_acc_top1:
best_acc_top1 = valid_acc_top1
is_best = True
utils.save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_acc_top1': best_acc_top1,
'optimizer': optimizer.state_dict(),
}, is_best, args.save)
utils.save(model, os.path.join(args.save, 'weights.pt'))
# utils.save(model, os.path.join(args.model_path, 'model.pt'))
# torch.save(model, os.path.join(arges.model_path, 'model.pt'))
writer.close()
def main():
torch.set_num_threads(3)
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
if args.perturb_alpha == 'none':
perturb_alpha = None
elif args.perturb_alpha == 'pgd_linf':
perturb_alpha = Linf_PGD_alpha
elif args.perturb_alpha == 'random':
perturb_alpha = Random_alpha
api = API('/nfs/data/xiangning/data/NAS-Bench-201-v1_0-e61699.pth')
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(C=args.init_channels, N=5, max_nodes=4, num_classes=n_classes, criterion=criterion)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
if args.dataset == 'cifar10':
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
elif args.dataset == 'cifar100':
train_transform, valid_transform = utils._data_transforms_cifar100(args)
train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform)
elif args.dataset == 'svhn':
train_transform, valid_transform = utils._data_transforms_svhn(args)
train_data = dset.SVHN(root=args.data, split='train', download=True, transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
if 'debug' in args.save:
split = args.batch_size
num_train = 2 * args.batch_size
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]),
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
if args.cutout:
# increase the cutout probability linearly throughout search
train_transform.transforms[-1].cutout_prob = args.cutout_prob * epoch / (args.epochs - 1)
logging.info('epoch %d lr %e cutout_prob %e', epoch, lr,
train_transform.transforms[-1].cutout_prob)
else:
logging.info('epoch %d lr %e', epoch, lr)
if args.perturb_alpha:
epsilon_alpha = 0.03 + (args.epsilon_alpha - 0.03) * epoch / args.epochs
logging.info('epoch %d epsilon_alpha %e', epoch, epsilon_alpha)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
model.show_alphas()
# training
train_acc, train_obj = train(train_queue, valid_queue, model, architect, criterion, optimizer, lr,
perturb_alpha, epsilon_alpha)
logging.info('train_acc %f', train_acc)
writer.add_scalar('Acc/train', train_acc, epoch)
writer.add_scalar('Obj/train', train_obj, epoch)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
writer.add_scalar('Acc/valid', valid_acc, epoch)
writer.add_scalar('Obj/valid', valid_obj, epoch)
# nasbench201
result = api.query_by_arch(model.genotype())
logging.info('{:}'.format(result))
cifar10_train, cifar10_test, cifar100_train, cifar100_valid, \
cifar100_test, imagenet16_train, imagenet16_valid, imagenet16_test = distill(result)
writer.add_scalars('nasbench201/cifar10', {'train':cifar10_train,'test':cifar10_test}, epoch)
writer.add_scalars('nasbench201/cifar100', {'train':cifar100_train,'valid':cifar100_valid, 'test':cifar100_test}, epoch)
writer.add_scalars('nasbench201/imagenet16', {'train':imagenet16_train,'valid':imagenet16_valid, 'test':imagenet16_test}, epoch)
utils.save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
'alpha': model.arch_parameters()
}, False, args.save)
writer.close()
