def main():
# Select the search space to search in
if args.search_space == '1':
search_space = SearchSpace1()
elif args.search_space == '2':
search_space = SearchSpace2()
elif args.search_space == '3':
search_space = SearchSpace3()
else:
raise ValueError('Unknown search space')
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels,
CIFAR_CLASSES,
args.layers,
criterion,
output_weights=args.output_weights,
steps=search_space.num_intermediate_nodes,
search_space=search_space)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
# Save the one shot model architecture weights for later analysis
filehandler = open(
os.path.join(args.save,
'one_shot_architecture_{}.obj'.format(epoch)), 'wb')
numpy_tensor_list = []
for tensor in model.arch_parameters():
numpy_tensor_list.append(tensor.detach().cpu().numpy())
pickle.dump(numpy_tensor_list, filehandler)
# Save the entire one-shot-model
filepath = os.path.join(args.save,
'one_shot_model_{}.obj'.format(epoch))
torch.save(model.state_dict(), filepath)
logging.info('architecture', numpy_tensor_list)
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr,
epoch)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
# Select the search space to search in
if args.search_space == '1':
search_space = SearchSpace1()
elif args.search_space == '2':
search_space = SearchSpace2()
elif args.search_space == '3':
search_space = SearchSpace3()
else:
raise ValueError('Unknown search space')
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels,
CIFAR_CLASSES,
args.layers,
criterion,
output_weights=args.output_weights,
steps=search_space.num_intermediate_nodes,
search_space=search_space)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
nasbench = None
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
# Save the one shot model architecture weights for later analysis
arch_filename = os.path.join(
args.save, 'one_shot_architecture_{}.obj'.format(epoch))
with open(arch_filename, 'wb') as filehandler:
numpy_tensor_list = []
for tensor in model.arch_parameters():
numpy_tensor_list.append(tensor.detach().cpu().numpy())
pickle.dump(numpy_tensor_list, filehandler)
# Save the entire one-shot-model
filepath = os.path.join(args.save,
'one_shot_model_{}.obj'.format(epoch))
torch.save(model.state_dict(), filepath)
logging.info('architecture')
logging.info(numpy_tensor_list)
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr,
epoch)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
logging.info('STARTING EVALUATION')
if nasbench is None:
nasbench = NasbenchWrapper(
dataset_file='/nasbench_data/nasbench_only108.tfrecord')
test, valid, runtime, params = naseval.eval_one_shot_model(
config=args.__dict__,
model=arch_filename,
nasbench_results=nasbench)
index = np.random.choice(list(range(3)))
logging.info(
'TEST ERROR: %.3f | VALID ERROR: %.3f | RUNTIME: %f | PARAMS: %d' %
(test[index], valid[index], runtime[index], params[index]))
if args.s3_bucket is not None:
for root, dirs, files in os.walk(args.save):
for f in files:
if 'one_shot_model' not in f:
path = os.path.join(root, f)
upload_to_s3(path, args.s3_bucket, path)
def main():
torch.set_num_threads(3)
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels, n_classes, args.layers, criterion,
spaces_dict[args.search_space])
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
model2 = ResNet18()
model2 = model2.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
optimizer2 = torch.optim.SGD(model2.parameters(),
args.learning_rate2,
momentum=args.momentum,
weight_decay=args.weight_decay2)
if args.dataset == 'cifar10':
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
elif args.dataset == 'cifar100':
train_transform, valid_transform = utils._data_transforms_cifar100(
args)
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
elif args.dataset == 'svhn':
train_transform, valid_transform = utils._data_transforms_svhn(args)
train_data = dset.SVHN(root=args.data,
split='train',
download=True,
transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
if 'debug' in args.save:
split = args.batch_size
num_train = 2 * args.batch_size
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
scheduler2 = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer2, float(args.epochs), eta_min=args.learning_rate_min)
# architect = Architect(model, args)
architect = Architect(model2, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
lr2 = scheduler2.get_lr()[0]
if args.cutout:
# increase the cutout probability linearly throughout search
train_transform.transforms[
-1].cutout_prob = args.cutout_prob * epoch / (args.epochs - 1)
logging.info('epoch %d lr %e cutout_prob %e', epoch, lr,
train_transform.transforms[-1].cutout_prob)
else:
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
print(F.softmax(model.alphas_normal, dim=-1))
print(F.softmax(model.alphas_reduce, dim=-1))
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer,
optimizer2, lr, lr2, model2, epoch)
logging.info('train_acc %f', train_acc)
writer.add_scalar('Acc/train', train_acc, epoch)
writer.add_scalar('Obj/train', train_obj, epoch)
# validation
valid_acc, valid_obj = infer(valid_queue, model2, criterion)
logging.info('valid_acc %f', valid_acc)
writer.add_scalar('Acc/valid', valid_acc, epoch)
writer.add_scalar('Obj/valid', valid_obj, epoch)
utils.save(model, os.path.join(args.save, 'weights.pt'))
writer.close()
def main():
torch.set_num_threads(3)
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
if args.perturb_alpha == 'none':
perturb_alpha = None
elif args.perturb_alpha == 'pgd_linf':
perturb_alpha = Linf_PGD_alpha
elif args.perturb_alpha == 'random':
perturb_alpha = Random_alpha
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
#######################################
resnet18 = models.resnet18(num_classes=10)
# torch.cuda.clear_memory_allocated()
# del Variables
# gc.collect()
# torch.cuda.empty_cache()
resnet18 = resnet18.cuda()
model2 = resnet18
######################################
model = Network(args.init_channels, n_classes, args.layers, criterion,
spaces_dict[args.search_space])
model = model.cuda()
model.load_state_dict(torch.load("weights-9.pt"))
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
if args.dataset == 'cifar10':
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
elif args.dataset == 'cifar100':
train_transform, valid_transform = utils._data_transforms_cifar100(
args)
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
elif args.dataset == 'svhn':
train_transform, valid_transform = utils._data_transforms_svhn(args)
train_data = dset.SVHN(root=args.data,
split='train',
download=True,
transform=train_transform)
# num_train = len(train_data)+24
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
# if 'debug' in args.save:
# split = args.batch_size
# num_train = 2 * args.batch_size
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True)
# train_queue = torch.utils.data.DataLoader(
# train_data, batch_size=args.batch_size,
# sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:5]),
# pin_memory=True)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
if args.cutout:
# increase the cutout probability linearly throughout search
train_transform.transforms[
-1].cutout_prob = args.cutout_prob * epoch / (args.epochs - 1)
logging.info('epoch %d lr %e cutout_prob %e', epoch, lr,
train_transform.transforms[-1].cutout_prob)
else:
logging.info('epoch %d lr %e', epoch, lr)
if args.perturb_alpha:
epsilon_alpha = 0.03 + (args.epsilon_alpha -
0.03) * epoch / args.epochs
logging.info('epoch %d epsilon_alpha %e', epoch, epsilon_alpha)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
print(F.softmax(model.alphas_normal, dim=-1))
print(F.softmax(model.alphas_reduce, dim=-1))
# training
# _, _, delta = train3(train_queue, valid_queue, model, architect, criterion, optimizer, lr,
# perturb_alpha, epsilon_alpha, model2, epoch)
# train_acc, train_obj = train4(train_queue, valid_queue, model, architect, criterion, optimizer, lr,
# perturb_alpha, epsilon_alpha, model2, epoch, delta)
# train_acc, train_obj = train2(train_queue, valid_queue, model, architect, criterion, optimizer, lr,
# perturb_alpha, epsilon_alpha, model2, epoch)
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr,
perturb_alpha, epsilon_alpha, model2,
epoch)
logging.info('train_acc %f', train_acc)
writer.add_scalar('Acc/train', train_acc, epoch)
writer.add_scalar('Obj/train', train_obj, epoch)
# validation
# valid_acc, valid_obj = infer(valid_queue, model, criterion)
############################################################################################################
valid_acc, valid_obj = infer(valid_queue, resnet18, criterion)
############################################################################################################
logging.info('valid_acc %f', valid_acc)
writer.add_scalar('Acc/valid', valid_acc, epoch)
writer.add_scalar('Obj/valid', valid_obj, epoch)
utils.save(model, os.path.join(args.save, 'weights.pt'))
# for epoch in range(args.epochs):
# scheduler.step()
# lr = scheduler.get_lr()[0]
# if args.cutout:
# # increase the cutout probability linearly throughout search
# train_transform.transforms[-1].cutout_prob = args.cutout_prob * epoch / (args.epochs - 1)
# logging.info('epoch %d lr %e cutout_prob %e', epoch, lr,
# train_transform.transforms[-1].cutout_prob)
# else:
# logging.info('epoch %d lr %e', epoch, lr)
# if args.perturb_alpha:
# epsilon_alpha = 0.03 + (args.epsilon_alpha - 0.03) * epoch / args.epochs
# logging.info('epoch %d epsilon_alpha %e', epoch, epsilon_alpha)
# genotype = model.genotype()
# logging.info('genotype = %s', genotype)
# print(F.softmax(model.alphas_normal, dim=-1))
# print(F.softmax(model.alphas_reduce, dim=-1))
# # training
# _, _, delta = train3(train_queue, valid_queue, model, architect, criterion, optimizer, lr,
# perturb_alpha, epsilon_alpha, model2, epoch)
# for epoch in range(args.epochs+150):
# train_acc, train_obj = train4(train_queue, valid_queue, model, architect, criterion, optimizer, lr,
# perturb_alpha, epsilon_alpha, model2, epoch, delta)
# # train_acc, train_obj = train2(train_queue, valid_queue, model, architect, criterion, optimizer, lr,
# # perturb_alpha, epsilon_alpha, model2, epoch)
# # train_acc, train_obj = train(train_queue, valid_queue, model, architect, criterion, optimizer, lr)
# logging.info('train_acc %f', train_acc)
# writer.add_scalar('Acc/train', train_acc, epoch)
# writer.add_scalar('Obj/train', train_obj, epoch)
# # validation
# # valid_acc, valid_obj = infer(valid_queue, model, criterion)
# ############################################################################################################
# valid_acc, valid_obj = infer(valid_queue, resnet18, criterion)
# ############################################################################################################
# logging.info('valid_acc %f', valid_acc)
# writer.add_scalar('Acc/valid', valid_acc, epoch)
# writer.add_scalar('Obj/valid', valid_obj, epoch)
# utils.save(model, os.path.join(args.save, 'weights.pt'))
writer.close()
def main():
# Select the search space to search in
if args.search_space == '1':
search_space = SearchSpace1()
elif args.search_space == '2':
search_space = SearchSpace2()
elif args.search_space == '3':
search_space = SearchSpace3()
else:
raise ValueError('Unknown search space')
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.num_linear_layers,
args.init_channels,
CIFAR_CLASSES,
args.layers,
criterion,
output_weights=args.output_weights,
steps=search_space.num_intermediate_nodes,
search_space=search_space)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
# Read a random sample of architectures
archs = pickle.load(
open(
'/home/siemsj/projects/darts_weight_sharing_analysis/nasbench_analysis/architecture_inductive_bias/sampled_architectures_from_search_space_3.obj',
'rb'))
arch = archs[args.arch_idx]
arch_parameters = get_weights_from_arch(arch, model)
model._arch_parameters = arch_parameters
try:
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
# increase the cutout probability linearly throughout search
train_transform.transforms[
-1].cutout_prob = args.cutout_prob * epoch / (args.epochs - 1)
logging.info('epoch %d lr %e cutout_prob %e', epoch, lr,
train_transform.transforms[-1].cutout_prob)
# Save the one shot model architecture weights for later analysis
arch_filename = os.path.join(
args.save, 'one_shot_architecture_{}.obj'.format(epoch))
with open(arch_filename, 'wb') as filehandler:
numpy_tensor_list = []
for tensor in model.arch_parameters():
numpy_tensor_list.append(tensor.detach().cpu().numpy())
pickle.dump(numpy_tensor_list, filehandler)
# Save the entire one-shot-model
# filepath = os.path.join(args.save, 'one_shot_model_{}.obj'.format(epoch))
# torch.save(model.state_dict(), filepath)
logging.info('architecture', numpy_tensor_list)
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr,
epoch)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
logging.info('STARTING EVALUATION')
test, valid, runtime, params = naseval.eval_one_shot_model(
config=args.__dict__, model=arch_filename)
index = np.random.choice(list(range(3)))
logging.info(
'TEST ERROR: %.3f | VALID ERROR: %.3f | RUNTIME: %f | PARAMS: %d' %
(test[index], valid[index], runtime[index], params[index]))
except Exception as e:
logging.exception('message')
def main():
torch.set_num_threads(3)
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
if args.perturb_alpha == 'none':
perturb_alpha = None
elif args.perturb_alpha == 'pgd_linf':
perturb_alpha = Linf_PGD_alpha
elif args.perturb_alpha == 'random':
perturb_alpha = Random_alpha
api = API('/nfs/data/xiangning/data/NAS-Bench-201-v1_0-e61699.pth')
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(C=args.init_channels, N=5, max_nodes=4, num_classes=n_classes, criterion=criterion)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
if args.dataset == 'cifar10':
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
elif args.dataset == 'cifar100':
train_transform, valid_transform = utils._data_transforms_cifar100(args)
train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform)
elif args.dataset == 'svhn':
train_transform, valid_transform = utils._data_transforms_svhn(args)
train_data = dset.SVHN(root=args.data, split='train', download=True, transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
if 'debug' in args.save:
split = args.batch_size
num_train = 2 * args.batch_size
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]),
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
if args.cutout:
# increase the cutout probability linearly throughout search
train_transform.transforms[-1].cutout_prob = args.cutout_prob * epoch / (args.epochs - 1)
logging.info('epoch %d lr %e cutout_prob %e', epoch, lr,
train_transform.transforms[-1].cutout_prob)
else:
logging.info('epoch %d lr %e', epoch, lr)
if args.perturb_alpha:
epsilon_alpha = 0.03 + (args.epsilon_alpha - 0.03) * epoch / args.epochs
logging.info('epoch %d epsilon_alpha %e', epoch, epsilon_alpha)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
model.show_alphas()
# training
train_acc, train_obj = train(train_queue, valid_queue, model, architect, criterion, optimizer, lr,
perturb_alpha, epsilon_alpha)
logging.info('train_acc %f', train_acc)
writer.add_scalar('Acc/train', train_acc, epoch)
writer.add_scalar('Obj/train', train_obj, epoch)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
writer.add_scalar('Acc/valid', valid_acc, epoch)
writer.add_scalar('Obj/valid', valid_obj, epoch)
# nasbench201
result = api.query_by_arch(model.genotype())
logging.info('{:}'.format(result))
cifar10_train, cifar10_test, cifar100_train, cifar100_valid, \
cifar100_test, imagenet16_train, imagenet16_valid, imagenet16_test = distill(result)
writer.add_scalars('nasbench201/cifar10', {'train':cifar10_train,'test':cifar10_test}, epoch)
writer.add_scalars('nasbench201/cifar100', {'train':cifar100_train,'valid':cifar100_valid, 'test':cifar100_test}, epoch)
writer.add_scalars('nasbench201/imagenet16', {'train':imagenet16_train,'valid':imagenet16_valid, 'test':imagenet16_test}, epoch)
utils.save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
'alpha': model.arch_parameters()
}, False, args.save)
writer.close()
def main():
if not 'debug' in args.save:
from nasbench_analysis import eval_darts_one_shot_model_in_nasbench as naseval
# Select the search space to search in
if args.search_space == '1':
search_space = SearchSpace1()
elif args.search_space == '2':
search_space = SearchSpace2()
elif args.search_space == '3':
search_space = SearchSpace3()
else:
raise ValueError('Unknown search space')
torch.set_num_threads(3)
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels,
CIFAR_CLASSES,
args.layers,
criterion,
output_weights=args.output_weights,
steps=search_space.num_intermediate_nodes,
search_space=search_space)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
if 'debug' in args.save:
split = args.batch_size
num_train = 2 * args.batch_size
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
analyzer = Analyzer(model, args)
architect = Architect(model, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
# Save the one shot model architecture weights for later analysis
arch_filename = os.path.join(
args.save, 'one_shot_architecture_{}.obj'.format(epoch))
with open(arch_filename, 'wb') as filehandler:
numpy_tensor_list = []
for tensor in model.arch_parameters():
numpy_tensor_list.append(tensor.detach().cpu().numpy())
pickle.dump(numpy_tensor_list, filehandler)
# # Save the entire one-shot-model
# filepath = os.path.join(args.save, 'one_shot_model_{}.obj'.format(epoch))
# torch.save(model.state_dict(), filepath)
for i in numpy_tensor_list:
print(i)
# training
train_acc, train_obj, ev = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr,
epoch, analyzer)
logging.info('train_acc %f', train_acc)
logging.info('eigenvalue %f', ev)
writer.add_scalar('Acc/train', train_acc, epoch)
writer.add_scalar('Obj/train', train_obj, epoch)
writer.add_scalar('Analysis/eigenvalue', ev, epoch)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
writer.add_scalar('Acc/valid', valid_acc, epoch)
writer.add_scalar('Obj/valid', valid_obj, epoch)
utils.save(model, os.path.join(args.save, 'weights.pt'))
if not 'debug' in args.save:
# benchmark
logging.info('STARTING EVALUATION')
test, valid, runtime, params = naseval.eval_one_shot_model(
config=args.__dict__, model=arch_filename)
index = np.random.choice(list(range(3)))
test, valid, runtime, params = np.mean(test), np.mean(
valid), np.mean(runtime), np.mean(params)
logging.info(
'TEST ERROR: %.3f | VALID ERROR: %.3f | RUNTIME: %f | PARAMS: %d'
% (test, valid, runtime, params))
writer.add_scalar('Analysis/test', test, epoch)
writer.add_scalar('Analysis/valid', valid, epoch)
writer.add_scalar('Analysis/runtime', runtime, epoch)
writer.add_scalar('Analysis/params', params, epoch)
writer.close()