def main(args):
reset_seed(args.seed)
prepare_logger(args)
logger.info("These are the hyper-parameters you want to tune:\n%s",
pprint.pformat(vars(args)))
device = 'cuda' if torch.cuda.is_available() else 'cpu'
train_loader, test_loader = data_preprocess(args)
# model = models.__dict__[args.model](num_classes=10)
model = CNN(32, 3, args.channels, 10, args.layers)
model.to(device)
criterion = nn.CrossEntropyLoss()
if args.optimizer == 'adam':
optimizer = optim.Adam(model.parameters(),
lr=args.initial_lr,
weight_decay=args.weight_decay)
else:
if args.optimizer == 'sgd':
optimizer_cls = optim.SGD
elif args.optimizer == 'rmsprop':
optimizer_cls = optim.RMSprop
optimizer = optimizer_cls(model.parameters(),
lr=args.initial_lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
if args.lr_scheduler == 'cosin':
scheduler = optim.lr_scheduler.CosineAnnealingLR(
optimizer, args.epochs, eta_min=args.ending_lr)
elif args.lr_scheduler == 'linear':
scheduler = optim.lr_scheduler.StepLR(optimizer,
step_size=15,
gamma=0.1)
trainer = DartsTrainer(
model,
loss=criterion,
metrics=lambda output, target: accuracy(output, target),
optimizer=optimizer,
num_epochs=args.epochs,
dataset_train=train_loader,
dataset_valid=test_loader,
batch_size=args.batch_size,
log_frequency=args.log_frequency,
unrolled=args.unrolled,
callbacks=[
LRSchedulerCallback(scheduler),
ArchitectureCheckpoint("./checkpoints_layer5")
])
if args.visualization:
trainer.enable_visualization()
trainer.train()
def generate_callbacks(self):
'''
Args：
func: a function to generate other callbacks, must return a list
Return:
a list of callbacks.
'''
self.ckpt_callback = CheckpointCallback(
checkpoint_dir=self.cfg.logger.path,
name='best_search.pth',
mode=self.cfg.callback.checkpoint.mode)
self.arch_callback = ArchitectureCheckpoint(self.cfg.logger.path)
self.relevance_callback = RelevanceCallback(
save_path=self.cfg.logger.path,
filename=self.cfg.callback.relevance.filename)
callbacks = [
self.ckpt_callback, self.arch_callback, self.relevance_callback
]
return callbacks
tanh_constant=1.1,
cell_exit_extra_step=True)
else:
raise AssertionError
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(),
0.05,
momentum=0.9,
weight_decay=1.0E-4)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,
T_max=num_epochs,
eta_min=0.001)
trainer = enas.EnasTrainer(model,
loss=criterion,
metrics=accuracy,
reward_function=reward_accuracy,
optimizer=optimizer,
callbacks=[
LRSchedulerCallback(lr_scheduler),
ArchitectureCheckpoint("./checkpoints")
],
batch_size=args.batch_size,
num_epochs=num_epochs,
dataset_train=dataset_train,
dataset_valid=dataset_valid,
log_frequency=args.log_frequency,
mutator=mutator)
trainer.train()
criterion = nn.CrossEntropyLoss()
if args.arch is not None:
logger.info('model retraining...')
with open(args.arch, 'r') as f:
arch = json.load(f)
for trial in query_nb201_trial_stats(arch, 200, 'cifar100'):
pprint.pprint(trial)
apply_fixed_architecture(model, args.arch)
dataloader_train = DataLoader(dataset_train, batch_size=args.batch_size, shuffle=True, num_workers=0)
dataloader_valid = DataLoader(dataset_valid, batch_size=args.batch_size, shuffle=True, num_workers=0)
train(args, model, dataloader_train, dataloader_valid, criterion, optim,
torch.device('cuda' if torch.cuda.is_available() else 'cpu'))
exit(0)
trainer = enas.EnasTrainer(model,
loss=criterion,
metrics=lambda output, target: accuracy(output, target, topk=(1,)),
reward_function=reward_accuracy,
optimizer=optim,
callbacks=[LRSchedulerCallback(lr_scheduler), ArchitectureCheckpoint("./checkpoints")],
batch_size=args.batch_size,
num_epochs=args.epochs,
dataset_train=dataset_train,
dataset_valid=dataset_valid,
log_frequency=args.log_frequency)
if args.visualization:
trainer.enable_visualization()
trainer.train()
def model_creator(layers):
model = CNN(32, 3, args.channels, 10, layers, n_nodes=args.nodes)
criterion = nn.CrossEntropyLoss()
optim = torch.optim.SGD(model.parameters(),
0.025,
momentum=0.9,
weight_decay=3.0E-4)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optim, args.epochs, eta_min=0.001)
return model, criterion, optim, lr_scheduler
logger.info("initializing trainer")
trainer = PdartsTrainer(
model_creator,
init_layers=args.init_layers,
metrics=lambda output, target: accuracy(output, target, topk=(1, )),
pdarts_num_layers=args.add_layers,
pdarts_num_to_drop=args.dropped_ops,
num_epochs=args.epochs,
dataset_train=dataset_train,
dataset_valid=dataset_valid,
batch_size=args.batch_size,
log_frequency=args.log_frequency,
unrolled=args.unrolled,
callbacks=[ArchitectureCheckpoint("./checkpoints")])
logger.info("training")
trainer.train()
def main(args):
reset_seed(args.seed)
prepare_logger(args)
logger.info("These are the hyper-parameters you want to tune:\n%s",
pprint.pformat(vars(args)))
if args.model == 'nas':
logger.info("Using NAS.\n")
if args.fix_arch:
if not os.path.exists(args.arc_checkpoint):
print(args.arc_checkpoint,
'does not exist, don not fix archetect')
args.fix_arch = False
device = 'cuda' if torch.cuda.is_available() else 'cpu'
if args.model == 'nas':
if not args.fix_arch:
model = CNN(32, 3, args.channels, 10, args.layers)
trainset, testset = data_preprocess(args)
else:
model = CNN(32, 3, args.channels, 10, args.layers)
apply_fixed_architecture(model, args.arc_checkpoint)
model.to(device)
train_loader, test_loader = data_preprocess(args)
else:
train_loader, test_loader = data_preprocess(args)
model = models.__dict__[args.model]()
model.to(device)
criterion = nn.CrossEntropyLoss()
if args.optimizer == 'adam':
optimizer = optim.Adam(model.parameters(),
lr=args.initial_lr,
weight_decay=args.weight_decay)
else:
if args.optimizer == 'sgd':
optimizer_cls = optim.SGD
elif args.optimizer == 'rmsprop':
optimizer_cls = optim.RMSprop
optimizer = optimizer_cls(model.parameters(),
lr=args.initial_lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer,
args.epochs,
eta_min=args.ending_lr)
if args.model == 'nas' and not args.fix_arch:
trainer = DartsTrainer(model,
loss=criterion,
metrics=lambda output, target: accuracyTopk(
output, target, topk=(1, )),
optimizer=optimizer,
num_epochs=args.epochs,
dataset_train=trainset,
dataset_valid=testset,
batch_size=args.batch_size,
log_frequency=args.log_frequency,
unrolled=args.unrolled,
callbacks=[
LRSchedulerCallback(scheduler),
ArchitectureCheckpoint("./checkpoints")
])
if args.visualization:
trainer.enable_visualization()
trainer.train()
trainer.export("final_arch.json")
else:
for epoch in range(1, args.epochs + 1):
train(model, train_loader, criterion, optimizer, scheduler, args,
epoch, device)
top1, _ = test(model, test_loader, criterion, args, epoch, device)
nni.report_intermediate_result(top1)
logger.info("Final accuracy is: %.6f", top1)
nni.report_final_result(top1)
model = DartsStackedCells(3, args.channels, 10, args.layers)
model = model.to(device)
trainset, testset = getDatasets()
if __name__ == '__main__':
criterion = nn.CrossEntropyLoss()
# optimizer = optim.SGD(model.parameters(), 0.025, momentum=0.9, weight_decay=3.0E-4)
optimizer = optim.Adamax(model.parameters(), lr=0.025)
lr_scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer,
args.epochs,
eta_min=0.001)
trainer = DartsTrainer(
model,
criterion,
lambda outputs, labels: getAccuracy(outputs, labels),
optimizer,
args.epochs,
trainset,
testset,
batch_size=args.batch_size,
log_frequency=args.log_frequency,
unrolled=args.unrolled,
callbacks=[
LRSchedulerCallback(lr_scheduler),
ArchitectureCheckpoint('./checkpoints')
])
if args.visualization:
trainer.enable_visualization()
trainer.train()
model = CNN(32, 3, args.channels, 6, args.layers)
criterion = nn.CrossEntropyLoss()
optim = torch.optim.SGD(model.parameters(),
0.025,
momentum=0.9,
weight_decay=3.0E-4)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optim,
args.epochs,
eta_min=0.001)
trainer = DartsTrainer(
model,
loss=criterion,
metrics=lambda output, target: accuracy(output, target, topk=(1, )),
optimizer=optim,
num_epochs=args.epochs,
dataset_train=dataset_train,
dataset_valid=dataset_valid,
batch_size=args.batch_size,
log_frequency=args.log_frequency,
unrolled=args.unrolled,
callbacks=[
LRSchedulerCallback(lr_scheduler),
ArchitectureCheckpoint("./checkpoints_aoi")
])
if args.visualization:
trainer.enable_visualization()
trainer.train()
trainer.export(file="final_architecture.json")
if args.v1:
from nni.algorithms.nas.pytorch.darts import DartsTrainer
trainer = DartsTrainer(model,
loss=criterion,
metrics=lambda output, target: accuracy(
output, target, topk=(1, )),
optimizer=optim,
num_epochs=args.epochs,
dataset_train=dataset_train,
dataset_valid=dataset_valid,
batch_size=args.batch_size,
log_frequency=args.log_frequency,
unrolled=args.unrolled,
callbacks=[
LRSchedulerCallback(lr_scheduler),
ArchitectureCheckpoint(args.checkpath)
],
workers=0)
if args.visualization:
trainer.enable_visualization()
trainer.train()
else:
from nni.retiarii.oneshot.pytorch import DartsTrainer
trainer = DartsTrainer(model=model,
loss=criterion,
metrics=lambda output, target: accuracy(
output, target, topk=(1, )),
optimizer=optim,
num_epochs=args.epochs,
dataset=dataset_train,