def __init__(self):
self.config = SearchConfig()
self.writer = None
if self.config.tb_dir != "":
from torch.utils.tensorboard import SummaryWriter
self.writer = SummaryWriter(self.config.tb_dir, flush_secs=20)
init_gpu_params(self.config)
set_seed(self.config)
self.logger = FileLogger('./log', self.config.is_master,
self.config.is_master)
self.load_data()
self.logger.info(self.config)
self.model = SearchCNNController(self.config, self.n_classes,
self.output_mode)
self.load_model()
self.init_kd_component()
if self.config.n_gpu > 0:
self.model.to(device)
if self.config.n_gpu > 1:
self.model = torch.nn.parallel.DistributedDataParallel(
self.model,
device_ids=[self.config.local_rank],
find_unused_parameters=True)
self.model_to_print = self.model if self.config.multi_gpu is False else self.model.module
self.architect = Architect(self.model, self.teacher_model, self.config,
self.emd_tool)
mb_params = param_size(self.model)
self.logger.info("Model size = {:.3f} MB".format(mb_params))
self.eval_result_map = []
self.init_optim()
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
number_of_classes = class_dict[args.dataset]
in_channels = inp_channel_dict[args.dataset]
model = Network(args.init_channels, number_of_classes, args.layers,
criterion, in_channels)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
# Get transforms to apply on data
train_transform, valid_transform = utils.get_data_transforms(args)
# Get the training queue
train_queue, valid_queue = get_training_queues(args, train_transform)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
print(F.softmax(model.alphas_normal, dim=-1))
print(F.softmax(model.alphas_reduce, dim=-1))
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
logging.info("args = %s", args)
torch.backends.cudnn.benchmark = True
torch.backends.cudnn.enabled=True
model = SearchSpace()
model.cuda()
optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
train_samples = Rain800(args.data+'training/', args.steps*args.batch_size, args.patch_size)
train_queue = torch.utils.data.DataLoader(train_samples, batch_size=args.batch_size, pin_memory=True)
val_samples = Rain800(args.data+'test_syn/', 30*args.batch_size, args.patch_size)
valid_queue = torch.utils.data.DataLoader(val_samples, batch_size=args.batch_size, pin_memory=True)
best_psnr = 0
best_psnr_epoch = 0
best_ssim = 0
best_ssim_epoch = 0
best_loss = float("inf")
best_loss_epoch = 0
for epoch in range(args.epochs):
lr = scheduler.get_lr()[0]
logging.info('epoch %d/%d lr %e', epoch+1, args.epochs, lr)
# training
train(epoch, train_queue, valid_queue, model, architect, optimizer, lr)
# validation
psnr, ssim, loss = infer(valid_queue, model)
if psnr > best_psnr and not math.isinf(psnr):
utils.save(model, os.path.join(args.save, 'best_psnr_weights.pt'))
best_psnr_epoch = epoch+1
best_psnr = psnr
if ssim > best_ssim:
utils.save(model, os.path.join(args.save, 'best_ssim_weights.pt'))
best_ssim_epoch = epoch+1
best_ssim = ssim
if loss < best_loss:
utils.save(model, os.path.join(args.save, 'best_loss_weights.pt'))
best_loss_epoch = epoch+1
best_loss = loss
scheduler.step()
logging.info('psnr:%6f ssim:%6f loss:%6f -- best_psnr:%6f best_ssim:%6f best_loss:%6f', psnr, ssim, loss, best_psnr, best_ssim, best_loss)
logging.info('arch:%s', torch.argmax(model.arch_parameters()[0], dim=1))
logging.info('BEST_LOSS(epoch):%6f(%d), BEST_PSNR(epoch):%6f(%d), BEST_SSIM(epoch):%6f(%d)', best_loss, best_loss_epoch, best_psnr, best_psnr_epoch, best_ssim, best_ssim_epoch)
utils.save(model, os.path.join(args.save, 'last_weights.pt'))
def main():
# if not torch.cuda.is_available():
# logging.info('no gpu device available')
# sys.exit(1)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
# logging.info('gpu device = %d' % args.gpu)
# logging.info("args = %s", args)
criterion = nn.MSELoss()
# criterion = criterion.cuda()
model = Network(args.network_inputsize, args.network_outputsize,
args.max_width, args.max_depth, criterion)
# model = model.cuda()
optimizer = torch.optim.Adam(
model.parameters(),
args.learning_rate,
#momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
plt.ion()
for epoch in range(args.epochs):
lr = scheduler.get_lr()[0]
# lr = args.learning_rate
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
print(F.softmax(model.w_alpha, dim=-1))
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr,
epoch)
logging.info('train_acc %f', train_acc)
scheduler.step()
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion, epoch)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
plt.draw()
plt.pause(0.1)
plt.ioff()
plt.show()
def main():
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
data = locate('get_{}'.format(args.dataset))(args)
train_data, val_data, test_data = data
if args.dataset in 'mnist':
model = MLP(args)
elif args.dataset in 'cifar10' or args.dataset in 'cifar100':
model = Resnet18(args)
else:
raise Exception('error')
weight_arch = data_selection(data[0])
architect = Architect(model, weight_arch, args)
train_loader = DataLoader(train_data, batch_size = args.batch_size, shuffle = True, drop_last = True)
val_loader = DataLoader(val_data, batch_size = 64, shuffle = True, drop_last = False)
test_loader = DataLoader(test_data, batch_size = 64, shuffle = True, drop_last = False)
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
print(optimizer.state)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.n_epochs), eta_min=args.learning_rate_min)
for epoch in range(args.n_epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
train_acc, train_obj = Train(train_loader, val_data, model, args, architect, weight_arch, optimizer)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(val_loader, model)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
logger.info("Logger is set - training start")
# set default gpu device id
torch.cuda.set_device(config.gpus[0])
# set seed
np.random.seed(config.seed)
torch.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
torch.backends.cudnn.benchmark = True
# get data with meta info
input_size, input_channels, n_classes, train_data = utils.get_data(
config.dataset, config.data_path, cutout_length=0, validation=False)
net_crit = nn.CrossEntropyLoss().to(device)
model = SearchCNNController(input_channels,
config.init_channels,
n_classes,
config.layers,
net_crit,
device_ids=config.gpus)
model = model.to(device)
# weights optimizer
w_optim = torch.optim.SGD(model.weights(),
config.w_lr,
momentum=config.w_momentum,
weight_decay=config.w_weight_decay)
# alphas optimizer
alpha_optim = torch.optim.Adam(model.alphas(),
config.alpha_lr,
betas=(0.5, 0.999),
weight_decay=config.alpha_weight_decay)
# split data to train/validation
n_train = len(train_data)
split = n_train // 2
indices = list(range(n_train))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(
indices[:split])
valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(
indices[split:])
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=train_sampler,
num_workers=config.workers,
pin_memory=False)
valid_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=valid_sampler,
num_workers=config.workers,
pin_memory=False)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim, config.epochs, eta_min=config.w_lr_min)
architect = Architect(model, config.w_momentum, config.w_weight_decay)
# training loop
best_top1 = -1.0
best_epoch = 0
################################ restore from last time #############################################
epoch_restore = config.epoch_restore
if config.restore:
utils.load_state_dict(model,
config.path,
extra='model',
parallel=(len(config.gpus) > 1))
if not config.model_only:
utils.load_state_dict(w_optim,
config.path,
extra='w_optim',
parallel=False)
utils.load_state_dict(alpha_optim,
config.path,
extra='alpha_optim',
parallel=False)
utils.load_state_dict(lr_scheduler,
config.path,
extra='lr_scheduler',
parallel=False)
utils.load_state_dict(epoch_restore,
config.path,
extra='epoch_restore',
parallel=False)
#####################################################################################################
for epoch in range(epoch_restore, config.epochs):
lr_scheduler.step()
lr = lr_scheduler.get_lr()[0]
model.print_alphas(logger)
# training
train(train_loader, valid_loader, model, architect, w_optim,
alpha_optim, lr, epoch)
# validation
cur_step = (epoch + 1) * len(train_loader)
top1 = validate(valid_loader, model, epoch, cur_step)
# top1 = 0.0
# log
# genotype
genotype = model.genotype()
logger.info("genotype = {}".format(genotype))
# genotype as a image
plot_path = os.path.join(config.plot_path,
"EP{:02d}".format(epoch + 1))
caption = "Epoch {}".format(epoch + 1)
plot(genotype.normal, plot_path + "-normal", caption)
plot(genotype.reduce, plot_path + "-reduce", caption)
# save
if best_top1 < top1:
best_top1 = top1
best_genotype = genotype
is_best = True
best_epoch = epoch + 1
else:
is_best = False
utils.save_checkpoint(model, config.path, is_best)
######################################## save all state ###################################################
utils.save_state_dict(model,
config.path,
extra='model',
is_best=is_best,
parallel=(len(config.gpus) > 1),
epoch=epoch + 1,
acc=top1,
last_state=((epoch + 1) >= config.epochs))
utils.save_state_dict(lr_scheduler,
config.path,
extra='lr_scheduler',
is_best=is_best,
parallel=False,
epoch=epoch + 1,
acc=top1,
last_state=((epoch + 1) >= config.epochs))
utils.save_state_dict(alpha_optim,
config.path,
extra='alpha_optim',
is_best=is_best,
parallel=False,
epoch=epoch + 1,
acc=top1,
last_state=((epoch + 1) >= config.epochs))
utils.save_state_dict(w_optim,
config.path,
extra='w_optim',
is_best=is_best,
parallel=False,
epoch=epoch + 1,
acc=top1,
last_state=((epoch + 1) >= config.epochs))
############################################################################################################
print("")
logger.info("Best Genotype at {} epch.".format(best_epoch))
logger.info("Final best Prec@1 = {:.4%}".format(best_top1))
logger.info("Best Genotype = {}".format(best_genotype))
def main():
logger.info("Logger is set - training start")
# set default gpu device id
torch.cuda.set_device(config.gpus[0])
# set seed
np.random.seed(config.seed)
torch.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
torch.backends.cudnn.benchmark = True
# get data with meta info
dahai_train_dataset = utils.MyDataset(data_dir=TRAIN_DATA_PATH, )
dahai_dev_dataset = utils.MyDataset(data_dir=DEV_DAHAI_DATA_PATH, )
# zhikang_test_dataset = utils.MyDataset(window_size=WINDOW_SIZE,
# window_step=WINDOW_STEP_DEV,
# data_path=TEST_ZHIKANG_DATA_PATH,
# voice_embed_path=TEST_ZHIKANG_VOICE_EMBEDDING_PATH,
# w2i=w2i,
# sent_max_len=SENT_MAX_LEN,
# )
train_data = utils.DataProvider(batch_size=config.batch_size,
dataset=dahai_train_dataset,
is_cuda=config.is_cuda)
dev_data = utils.DataProvider(batch_size=config.batch_size,
dataset=dahai_dev_dataset,
is_cuda=config.is_cuda)
# test_data = utils.DataProvider(batch_size=config.batch_size, dataset=zhikang_test_dataset, is_cuda=config.is_cuda)
print("train data nums:", len(train_data.dataset), "dev data nums:",
len(dev_data.dataset))
net_crit = nn.CrossEntropyLoss(reduction="none").to(device)
model = SearchCNNController(config.embedding_dim,
config.init_channels,
config.n_classes,
config.layers,
net_crit,
config=config,
n_nodes=config.n_nodes,
device_ids=config.gpus)
model = model.to(device).float()
# weights optimizer
w_optim = torch.optim.SGD(model.weights(),
config.w_lr,
momentum=config.w_momentum,
weight_decay=config.w_weight_decay)
# alphas optimizer
alpha_optim = torch.optim.Adam(model.alphas(),
config.alpha_lr,
betas=(0.5, 0.999),
weight_decay=config.alpha_weight_decay)
###### 余弦退火-调整学习率
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim, config.epochs, eta_min=config.w_lr_min)
architect = Architect(model, config.w_momentum, config.w_weight_decay)
# training loop
best_acc = 0.
best_genotype = model.genotype()
while True:
epoch = train_data.epoch
if epoch > config.epochs - 1:
break
lr_scheduler.step()
lr = lr_scheduler.get_lr()[0]
model.print_alphas(logger)
# training
train(train_data, dev_data, epoch, model, architect, w_optim,
alpha_optim, lr)
# validation
cur_step = train_data.iteration
valid_acc = validate(dev_data, model, epoch, cur_step)
# log
# genotype
genotype = model.genotype()
logger.info("genotype = {}".format(genotype))
# genotype as a image
plot_path = os.path.join(config.plot_path,
"EP{:02d}".format(epoch + 1))
caption = "Epoch {}".format(epoch + 1)
plot(genotype.normal, plot_path + "-normal", caption)
plot(genotype.reduce, plot_path + "-reduce", caption)
# save
if best_acc < valid_acc:
best_acc = valid_acc
best_genotype = genotype
is_best = True
else:
is_best = False
utils.save_checkpoint(model, config.path, is_best)
print("")
logger.info("Final best Prec@1 = {:.4%}".format(best_acc))
logger.info("Best Genotype = {}".format(best_genotype))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
dataset = Dataset(args.dataset)
train_examples = torch.from_numpy(dataset.get_train().astype('int64'))
valid_examples = torch.from_numpy(dataset.get_valid().astype('int64'))
CLASSES = dataset.get_shape()[0]
criterion = nn.CrossEntropyLoss(reduction='mean')
#criterion = CrossEntropyLabelSmooth(CLASSES, args.label_smooth)
criterion = criterion.cuda()
regularizer = {
'N2': N2(args.reg),
'N3': N3(args.reg),
}[args.regularizer]
model = Network(args.channels, CLASSES, args.layers,
criterion, regularizer, args.interleaved,
dataset.get_shape(), args.emb_dim, args.init, args.steps)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = {
'Adagrad':
lambda: optim.Adagrad(model.parameters(), lr=args.learning_rate),
#momentum=args.momentum,
#weight_decay=args.weight_decay)
'Adam':
lambda: optim.Adam(model.parameters(),
lr=args.learning_rate,
betas=(args.decay1, args.decay2)),
'SGD':
lambda: optim.SGD(model.parameters(), lr=args.learning_rate)
}[args.optimizer]()
# optimizer = torch.optim.SGD(
# model.parameters(),
# args.learning_rate,
# #TODO can we reintroduce these?
# momentum=args.momentum,
# weight_decay=args.weight_decay)
train_queue = torch.utils.data.DataLoader(
train_examples,
batch_size=args.batch_size,
shuffle=True,
#sampler=torch.utils.data.sampler.RandomSampler(),
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(
valid_examples,
batch_size=args.batch_size,
shuffle=True,
#sampler=torch.utils.data.sampler.RandomSampler(),
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
best_acc = 0
patience = 0
curve = {'valid': [], 'test': []}
architect = Architect(model, args)
for epoch in range(args.epochs):
model.epoch = epoch
print('model temperature param', 1.05**model.epoch)
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
print(F.softmax((1.05**epoch) * model.alphas_normal, dim=-1))
train_epoch(train_examples, train_queue, valid_queue, model, architect,
criterion, optimizer, regularizer, args.batch_size,
args.learning_rate)
if (epoch + 1) % args.report_freq == 0:
valid, test = [
avg_both(*dataset.eval(model, split,
-1 if split != 'train' else 50000))
for split in ['valid', 'test']
]
curve['valid'].append(valid)
curve['test'].append(test)
#curve['train'].append(train)
#print("\t TRAIN: ", train)
print("\t VALID: ", valid)
print("\t TEST: ", test)
is_best = False
if valid['MRR'] > best_acc:
best_acc = valid['MRR']
is_best = True
patience = 0
else:
patience += 1
def main():
config = SearchConfig(section='fine-tune')
device = torch.device("cuda")
# tensorboard
writer = SummaryWriter(log_dir=os.path.join(config.path, "tb"))
writer.add_text('config', config.as_markdown(), 0)
logger = utils.get_logger(
os.path.join(config.path, "{}_tune.log".format(config.name)))
config.print_params(logger.info)
logger.info("Logger is set - training start")
# set default gpu device id
torch.cuda.set_device(config.gpus[0])
# set seed
np.random.seed(config.seed)
torch.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
torch.backends.cudnn.benchmark = True
# get data with meta info
input_size, input_channels, n_classes, train_data, valid_data = utils.get_data(
config.dataset, config.data_path, cutout_length=0, validation=True)
logger.debug('loading checkpoint')
best_path = os.path.join(config.path, 'best.pth.tar')
model = torch.load(best_path)
model.prune()
model = model.to(device)
# weights optimizer
w_optim = torch.optim.SGD(model.weights(),
config.w_lr,
momentum=config.w_momentum,
weight_decay=config.w_weight_decay)
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
shuffle=True,
num_workers=config.workers,
pin_memory=True)
valid_loader = torch.utils.data.DataLoader(valid_data,
batch_size=config.batch_size,
shuffle=False,
num_workers=config.workers,
pin_memory=True)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim, config.epochs, eta_min=config.w_lr_min)
architect = Architect(model, config.w_momentum, config.w_weight_decay)
model.print_alphas(logger)
first_top1 = validate(valid_loader, model, -1, 0, device, config, logger,
writer)
os.system('mkdir -p ' + config.fine_tune_path)
# training loop
best_top1 = 0.
for epoch in range(config.epochs):
lr_scheduler.step()
lr = lr_scheduler.get_lr()[0]
model.print_alphas(logger)
# training
train(train_loader, model, architect, w_optim, lr, epoch, writer,
device, config, logger)
# validation
cur_step = (epoch + 1) * len(train_loader)
top1 = validate(valid_loader, model, epoch, cur_step, device, config,
logger, writer)
# save
if best_top1 < top1:
best_top1 = top1
is_best = True
else:
is_best = False
utils.save_checkpoint(model, config.fine_tune_path, is_best)
print("")
logger.info("Initial best Prec@1 = {:.4%}".format(first_top1))
logger.info("Final best Prec@1 = {:.4%}".format(best_top1))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
if args.loss_func == 'cce':
criterion = nn.CrossEntropyLoss().cuda()
elif args.loss_func == 'rll':
criterion = utils.RobustLogLoss().cuda()
else:
assert False, "Invalid loss function '{}' given. Must be in {'cce', 'rll'}".format(
args.loss_func)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, criterion)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
# Load dataset
if args.gold_fraction == 0:
train_data = CIFAR10(root=args.data,
train=True,
gold=False,
gold_fraction=args.gold_fraction,
corruption_prob=args.corruption_prob,
corruption_type=args.corruption_type,
transform=train_transform,
download=True,
seed=args.seed)
if args.clean_valid:
gold_train_data = CIFAR10(root=args.data,
train=True,
gold=True,
gold_fraction=1.0,
corruption_prob=args.corruption_prob,
corruption_type=args.corruption_type,
transform=train_transform,
download=True,
seed=args.seed)
else:
train_data = CIFAR10(root=args.data,
train=True,
gold=True,
gold_fraction=args.gold_fraction,
corruption_prob=args.corruption_prob,
corruption_type=args.corruption_type,
transform=train_transform,
download=True,
seed=args.seed)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True,
num_workers=2)
if args.clean_valid:
valid_queue = torch.utils.data.DataLoader(
gold_train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:]),
pin_memory=True,
num_workers=2)
else:
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:]),
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
print(F.softmax(model.alphas_normal, dim=-1))
print(F.softmax(model.alphas_reduce, dim=-1))
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main(pretrain=True):
config.save = 'ckpt/{}'.format(config.save)
create_exp_dir(config.save,
scripts_to_save=glob.glob('*.py') + glob.glob('*.sh'))
logger = SummaryWriter(config.save)
log_format = '%(asctime)s %(message)s'
logging.basicConfig(stream=sys.stdout,
level=logging.INFO,
format=log_format,
datefmt='%m/%d %I:%M:%S %p')
fh = logging.FileHandler(os.path.join(config.save, 'log.txt'))
fh.setFormatter(logging.Formatter(log_format))
logging.getLogger().addHandler(fh)
assert type(pretrain) == bool or type(pretrain) == str
update_arch = True
if pretrain == True:
update_arch = False
logging.info("args = %s", str(config))
# preparation ################
torch.backends.cudnn.enabled = True
torch.backends.cudnn.benchmark = True
seed = config.seed
np.random.seed(seed)
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(seed)
# Model #######################################
model = Network(config.layers,
slimmable=config.slimmable,
width_mult_list=config.width_mult_list,
width_mult_list_sh=config.width_mult_list_sh,
loss_weight=config.loss_weight,
prun_modes=config.prun_modes,
quantize=config.quantize)
model = torch.nn.DataParallel(model).cuda()
# print(model)
# teacher_model = Generator(3, 3)
# teacher_model.load_state_dict(torch.load(config.generator_A2B))
# teacher_model = torch.nn.DataParallel(teacher_model).cuda()
# for param in teacher_model.parameters():
# param.require_grads = False
if type(pretrain) == str:
partial = torch.load(pretrain + "/weights.pt")
state = model.state_dict()
pretrained_dict = {
k: v
for k, v in partial.items()
if k in state and state[k].size() == partial[k].size()
}
state.update(pretrained_dict)
model.load_state_dict(state)
# else:
# features = [model.module.stem, model.module.cells, model.module.header]
# init_weight(features, nn.init.kaiming_normal_, nn.InstanceNorm2d, config.bn_eps, config.bn_momentum, mode='fan_in', nonlinearity='relu')
architect = Architect(model, config)
# Optimizer ###################################
base_lr = config.lr
parameters = []
parameters += list(model.module.stem.parameters())
parameters += list(model.module.cells.parameters())
parameters += list(model.module.header.parameters())
if config.opt == 'Adam':
optimizer = torch.optim.Adam(parameters,
lr=base_lr,
betas=config.betas)
elif config.opt == 'Sgd':
optimizer = torch.optim.SGD(parameters,
lr=base_lr,
momentum=config.momentum,
weight_decay=config.weight_decay)
else:
logging.info("Wrong Optimizer Type.")
sys.exit()
# lr policy ##############################
total_iteration = config.nepochs * config.niters_per_epoch
if config.lr_schedule == 'linear':
lr_policy = torch.optim.lr_scheduler.LambdaLR(
optimizer,
lr_lambda=LambdaLR(config.nepochs, 0, config.decay_epoch).step)
elif config.lr_schedule == 'exponential':
lr_policy = torch.optim.lr_scheduler.ExponentialLR(
optimizer, config.lr_decay)
elif config.lr_schedule == 'multistep':
lr_policy = torch.optim.lr_scheduler.MultiStepLR(
optimizer, milestones=config.milestones, gamma=config.gamma)
else:
logging.info("Wrong Learning Rate Schedule Type.")
sys.exit()
# data loader ###########################
transforms_ = [
# transforms.Resize(int(config.image_height*1.12), Image.BICUBIC),
# transforms.RandomCrop(config.image_height),
# transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
]
# train_loader_model = DataLoader(ImageDataset(config.dataset_path, transforms_=transforms_, unaligned=True, portion=config.train_portion),
# batch_size=config.batch_size, shuffle=True, num_workers=config.num_workers)
# train_loader_arch = DataLoader(ImageDataset(config.dataset_path, transforms_=transforms_, unaligned=True, portion=config.train_portion-1),
# batch_size=config.batch_size, shuffle=True, num_workers=config.num_workers)
train_loader_model = DataLoader(PairedImageDataset(
config.dataset_path,
config.target_path,
transforms_=transforms_,
portion=config.train_portion),
batch_size=config.batch_size,
shuffle=True,
num_workers=config.num_workers)
train_loader_arch = DataLoader(PairedImageDataset(
config.dataset_path,
config.target_path,
transforms_=transforms_,
portion=config.train_portion - 1),
batch_size=config.batch_size,
shuffle=True,
num_workers=config.num_workers)
transforms_ = [
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
]
test_loader = DataLoader(ImageDataset(config.dataset_path,
transforms_=transforms_,
mode='test'),
batch_size=1,
shuffle=False,
num_workers=config.num_workers)
tbar = tqdm(range(config.nepochs), ncols=80)
valid_fid_history = []
flops_history = []
flops_supernet_history = []
best_fid = 1000
best_epoch = 0
for epoch in tbar:
logging.info(pretrain)
logging.info(config.save)
logging.info("lr: " + str(optimizer.param_groups[0]['lr']))
logging.info("update arch: " + str(update_arch))
# training
tbar.set_description("[Epoch %d/%d][train...]" %
(epoch + 1, config.nepochs))
train(pretrain,
train_loader_model,
train_loader_arch,
model,
architect,
optimizer,
lr_policy,
logger,
epoch,
update_arch=update_arch)
torch.cuda.empty_cache()
lr_policy.step()
# validation
if epoch and not (epoch + 1) % config.eval_epoch:
tbar.set_description("[Epoch %d/%d][validation...]" %
(epoch + 1, config.nepochs))
save(model, os.path.join(config.save, 'weights_%d.pt' % epoch))
with torch.no_grad():
if pretrain == True:
model.module.prun_mode = "min"
valid_fid = infer(epoch, model, test_loader, logger)
logger.add_scalar('fid/val_min', valid_fid, epoch)
logging.info("Epoch %d: valid_fid_min %.3f" %
(epoch, valid_fid))
if len(model.module._width_mult_list) > 1:
model.module.prun_mode = "max"
valid_fid = infer(epoch, model, test_loader, logger)
logger.add_scalar('fid/val_max', valid_fid, epoch)
logging.info("Epoch %d: valid_fid_max %.3f" %
(epoch, valid_fid))
model.module.prun_mode = "random"
valid_fid = infer(epoch, model, test_loader, logger)
logger.add_scalar('fid/val_random', valid_fid, epoch)
logging.info("Epoch %d: valid_fid_random %.3f" %
(epoch, valid_fid))
else:
model.module.prun_mode = None
valid_fid, flops = infer(epoch,
model,
test_loader,
logger,
finalize=True)
logger.add_scalar('fid/val', valid_fid, epoch)
logging.info("Epoch %d: valid_fid %.3f" %
(epoch, valid_fid))
logger.add_scalar('flops/val', flops, epoch)
logging.info("Epoch %d: flops %.3f" % (epoch, flops))
valid_fid_history.append(valid_fid)
flops_history.append(flops)
if update_arch:
flops_supernet_history.append(architect.flops_supernet)
if valid_fid < best_fid:
best_fid = valid_fid
best_epoch = epoch
logging.info("Best fid:%.3f, Best epoch:%d" %
(best_fid, best_epoch))
if update_arch:
state = {}
state['alpha'] = getattr(model.module, 'alpha')
state['beta'] = getattr(model.module, 'beta')
state['ratio'] = getattr(model.module, 'ratio')
state['beta_sh'] = getattr(model.module, 'beta_sh')
state['ratio_sh'] = getattr(model.module, 'ratio_sh')
state["fid"] = valid_fid
state["flops"] = flops
torch.save(
state,
os.path.join(config.save,
"arch_%d_%f.pt" % (epoch, flops)))
if config.flops_weight > 0:
if flops < config.flops_min:
architect.flops_weight /= 2
elif flops > config.flops_max:
architect.flops_weight *= 2
logger.add_scalar("arch/flops_weight",
architect.flops_weight, epoch + 1)
logging.info("arch_flops_weight = " +
str(architect.flops_weight))
save(model, os.path.join(config.save, 'weights.pt'))
if update_arch:
torch.save(state, os.path.join(config.save, "arch.pt"))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, criterion)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
#用来优化w的优化器
optimizer = torch.optim.SGD(
model.parameters(), #优化器更新的参数，这里更新的是w
args.learning_rate, #初始值是0.025，使用的余弦退火调度更新学习率，每个epoch的学习率都不一样
momentum=args.momentum, #0.9
weight_decay=args.weight_decay) #正则化参数3e-4
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[:split]
), #自定义从样本中取数据的策略，当train_portion=0.5时，就是前一半的数据用于train
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]), #数据集中后一半的数据用于验证
pin_memory=True,
num_workers=2)
# 学习率更新参数，每次迭代调整不同的学习率
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( #使用余弦退火调度设置各组参数组的学习率
optimizer,
float(args.epochs),
eta_min=args.learning_rate_min)
# 创建用于更新α的architect
architect = Architect(model, args)
# 经历50个epoch后搜索完毕
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0] #得到本次迭代的学习率lr
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype() #对应论文2.4 选出来权重值大的两个前驱节点，并把(操作，前驱节点)存下来
logging.info('genotype = %s', genotype)
print(F.softmax(model.alphas_normal, dim=-1))
print(F.softmax(model.alphas_reduce, dim=-1))
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main(args):
global log
log = logging.getLogger("train_search")
CIFAR_CLASSES = 10
if args.set == 'cifar100':
CIFAR_CLASSES = 100
if not torch.cuda.is_available():
log.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
log.info('gpu device = %d' % args.gpu)
log.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, criterion)
model = model.cuda()
log.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, _ = utils._data_transforms_cifar10(args)
if args.set == 'cifar100':
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
else:
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
targets = train_data.targets
train_idx = np.arange(len(targets))
if args.subsample > 0:
train_idx, _ = train_test_split(train_idx,
test_size=1 - args.subsample,
shuffle=True,
stratify=targets)
num_train = len(train_idx)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
train_idx[indices[:split]]),
pin_memory=True,
num_workers=4)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
train_idx[indices[split:num_train]]),
pin_memory=True,
num_workers=4)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, args.epochs, eta_min=args.learning_rate_min)
architect = Architect(model, args)
train_acc = None
valid_acc = None
l1_loss = torch.zeros(1)
l2_loss = torch.zeros(1)
criterion_loss = torch.zeros(1)
genotype = model.genotype()
log.info('initial genotype = %s', genotype)
for epoch in range(args.epochs):
lr = scheduler.get_last_lr()[0]
log.info('epoch %d lr %e', epoch, lr)
# model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
# training
train_acc, train_obj, l1_loss, l2_loss, criterion_loss = train(
train_queue, valid_queue, model, architect, criterion, optimizer,
lr, epoch, args.grad_clip, args.report_lines, args.unrolled,
args.criterion_weight, args.l1_weight, args.l2_weight)
scheduler.step()
log.info('train_acc %f', train_acc)
log.info('%s %f', L1_LOSS, l1_loss)
log.info('%s %f', L2_LOSS, l2_loss)
log.info('criterion_loss %f', criterion_loss)
# validation
if args.epochs - epoch <= 1:
valid_acc, valid_obj = infer(valid_queue, model, criterion,
args.report_lines)
log.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
genotype = model.genotype()
log.info('genotype = %s', genotype)
log.info('last genotype = %s', genotype)
model = TrainNetwork(36, CIFAR_CLASSES, 20, False, genotype)
model_size_mb = utils.count_parameters_in_MB(model)
log.info("Train model param size = %.2fMB", model_size_mb)
return {
L1_LOSS: {
tuple([args.l1_weight, args.criterion_weight]): {
TRAIN_ACC: train_acc,
VALID_ACC: valid_acc,
REG_LOSS: l1_loss.cpu().data.item(),
CRITERION_LOSS: criterion_loss.cpu().data.item(),
SIZE: model_size_mb,
GENOTYPE: genotype
}
},
L2_LOSS: {
tuple([args.l2_weight, args.criterion_weight]): {
TRAIN_ACC: train_acc,
VALID_ACC: valid_acc,
REG_LOSS: l2_loss.cpu().data.item(),
CRITERION_LOSS: criterion_loss.cpu().data.item(),
SIZE: model_size_mb,
GENOTYPE: genotype
}
}
}
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
in_channels, num_classes, dataset_in_torch = utils.dataset_fields(
args) # new
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels, in_channels, num_classes, args.layers,
criterion)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD( # SGD for weights
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_data = utils.dataset_split_and_transform(dataset_in_torch,
args) # new
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
prune = Prune(args.epochs_pre_prune)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
logging.info(F.softmax(model.alphas_normal, dim=-1))
logging.info(F.softmax(model.alphas_reduce, dim=-1))
# Pruning
if epoch > args.epochs_pre_prune:
if epoch == args.epochs - 1:
prune.num_to_zero = 90 - (
len(prune.zeros_indices_alphas_normal)
) #need to prune 90 alphas by the end
if args.sparse == 'sparse':
prune.num_to_zero_sparse(epoch, args)
prune.prune_all_alphas(model)
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
logger.info("Logger is set - training start")
# set default gpu device id
torch.cuda.set_device(config.gpus[0])
# get data with meta info
input_size, input_channels, n_classes, train_data = utils.get_data(
config.dataset, config.data_path, cutout_length=0, validation=False)
net_crit = nn.CrossEntropyLoss().to(device)
# set seed
np.random.seed(config.seed)
torch.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
model_1 = SearchCNNController(input_channels,
config.init_channels,
n_classes,
config.layers,
net_crit,
device_ids=config.gpus)
torch.manual_seed(config.seed + 1)
torch.cuda.manual_seed_all(config.seed + 1)
model_2 = SearchCNNController(input_channels,
config.init_channels,
n_classes,
config.layers,
net_crit,
device_ids=config.gpus)
torch.backends.cudnn.benchmark = True
model_1 = model_1.to(device)
model_2 = model_2.to(device)
# weights optimizer
w_optim_1 = torch.optim.SGD(model_1.weights(),
config.w_lr,
momentum=config.w_momentum,
weight_decay=config.w_weight_decay)
# alphas optimizer
alpha_optim_1 = torch.optim.Adam(model_1.alphas(),
config.alpha_lr,
betas=(0.5, 0.999),
weight_decay=config.alpha_weight_decay)
# weights optimizer
w_optim_2 = torch.optim.SGD(model_2.weights(),
config.w_lr,
momentum=config.w_momentum,
weight_decay=config.w_weight_decay)
# alphas optimizer
alpha_optim_2 = torch.optim.Adam(model_2.alphas(),
config.alpha_lr,
betas=(0.5, 0.999),
weight_decay=config.alpha_weight_decay)
# split data to train/validation
n_train = len(train_data)
split = n_train // 2
indices = list(range(n_train))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(
indices[:split])
valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(
indices[split:])
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=train_sampler,
num_workers=config.workers,
pin_memory=True)
valid_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=valid_sampler,
num_workers=config.workers,
pin_memory=True)
lr_scheduler_1 = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim_1, config.epochs, eta_min=config.w_lr_min)
lr_scheduler_2 = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim_2, config.epochs, eta_min=config.w_lr_min)
architect = Architect(model_1, model_2, config.w_momentum,
config.w_weight_decay)
# training loop
best_top1_1 = 0.
best_top1_2 = 0.
for epoch in range(config.epochs):
lr_scheduler_1.step()
lr_1 = lr_scheduler_1.get_lr()[0]
lr_scheduler_2.step()
lr_2 = lr_scheduler_2.get_lr()[0]
model_1.print_alphas(logger)
model_2.print_alphas(logger)
# training
train(train_loader, valid_loader, model_1, model_2, architect,
w_optim_1, w_optim_2, alpha_optim_1, alpha_optim_2, lr_1, lr_2,
epoch, config.lmbda)
# validation
cur_step = (epoch + 1) * len(train_loader)
top1_1, top1_2 = validate(valid_loader, model_1, model_2, epoch,
cur_step)
# log
# genotype
genotype_1 = model_1.genotype()
genotype_2 = model_2.genotype()
logger.info("genotype_1 = {}".format(genotype_1))
logger.info("genotype_2 = {}".format(genotype_2))
# genotype as a image
# plot_path = os.path.join(config.plot_path, "EP{:02d}".format(epoch+1))
# caption = "Epoch {}".format(epoch+1)
# plot(genotype_1.normal, plot_path + "-normal", caption)
# plot(genotype_1.reduce, plot_path + "-reduce", caption)
# plot(genotype_2.normal, plot_path + "-normal", caption)
# plot(genotype_2.reduce, plot_path + "-reduce", caption)
# save
if best_top1_1 < top1_1:
best_top1_1 = top1_1
best_genotype_1 = genotype_1
is_best_1 = True
else:
is_best_1 = False
if best_top1_2 < top1_2:
best_top1_2 = top1_2
best_genotype_2 = genotype_2
is_best_2 = True
else:
is_best_2 = False
utils.save_checkpoint(model_1, config.path, 1, is_best_1)
utils.save_checkpoint(model_2, config.path, 2, is_best_2)
print("")
logger.info("Final best Prec@1_1 = {:.4%}".format(best_top1_1))
logger.info("Best Genotype_1 = {}".format(best_genotype_1))
logger.info("Final best Prec@1_2 = {:.4%}".format(best_top1_2))
logger.info("Best Genotype_2 = {}".format(best_genotype_2))
def main():
if hvd.rank() == 0:
logger.info("Logger is set - training start")
# set default gpu device id
# torch.cuda.set_device(config.gpus[0])
# set seed
np.random.seed(config.seed)
torch.manual_seed(config.seed)
# torch.cuda.manual_seed_all(config.seed)
torch.backends.cudnn.benchmark = False
# get data with meta info
(train_X, train_y), (valid_X, valid_y) = load_data()
in_dim = np.shape(train_X)[1]
out_dim = np.shape(train_y)[1]
train_X, train_y = (torch.tensor(train_X,
dtype=torch.float), torch.tensor(train_y))
train_data = torch.utils.data.TensorDataset(train_X, train_y)
valid_X, valid_y = (torch.tensor(valid_X,
dtype=torch.float), torch.tensor(valid_y))
valid_data = torch.utils.data.TensorDataset(valid_X, valid_y)
print("in_dim: ", in_dim)
print("out_dim: ", out_dim)
net_crit = nn.MSELoss().to(device)
layers = 1
n_nodes = 4
model = SearchFCNNController(in_dim,
out_dim,
layers,
net_crit,
n_nodes=n_nodes,
device_ids=config.gpus)
model = model.to(device)
# weights optimizer
# By default, Adasum doesn't need scaling up learning rate.
lr_scaler = hvd.size()
# w_optim = torch.optim.SGD(
# model.weights(),
# config.w_lr * lr_scaler,
# momentum=config.w_momentum,
# weight_decay=config.w_weight_decay,
# )
w_optim = torch.optim.Adagrad(model.weights(),
config.w_lr * lr_scaler,
weight_decay=config.w_weight_decay)
# w_optim = torch.optim.RMSprop(model.weights())
# alphas optimizer
alpha_lr = config.alpha_lr
alpha_optim = torch.optim.Adam(
model.alphas(),
alpha_lr,
betas=(0.5, 0.999),
weight_decay=config.alpha_weight_decay,
)
# split data to train/validation
train_sampler = torch.utils.data.distributed.DistributedSampler(
train_data, num_replicas=hvd.size(), rank=hvd.rank())
# valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices_valid)
valid_sampler = torch.utils.data.distributed.DistributedSampler(
valid_data, num_replicas=hvd.size(), rank=hvd.rank())
train_loader = torch.utils.data.DataLoader(
train_data,
batch_size=config.batch_size,
sampler=train_sampler,
num_workers=config.workers,
pin_memory=True,
)
# vis.
# dataiter = iter(train_loader)
# images, labels = dataiter.next()
# writer.add_graph(model, [images[0]])
# writer.close()
valid_loader = torch.utils.data.DataLoader(
valid_data,
batch_size=config.batch_size,
sampler=valid_sampler,
num_workers=config.workers,
pin_memory=True,
)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim, config.epochs, eta_min=config.w_lr_min)
architect = Architect(model,
config.w_momentum,
config.w_weight_decay,
allow_unused=False)
# Horovod: broadcast parameters & optimizer state.
hvd.broadcast_parameters(model.state_dict(), root_rank=0)
hvd.broadcast_optimizer_state(w_optim, root_rank=0)
# Horovod: (optional) compression algorithm.
# compression = hvd.Compression.fp16
# Horovod: wrap optimizer with DistributedOptimizer.
w_optim = hvd.DistributedOptimizer(
w_optim,
named_parameters=model.named_parameters(),
# compression=compression,
# op=hvd.Adasum,
op=hvd.Average,
)
# training loop
best_top1 = None
epochs = config.epochs
for epoch in range(epochs):
lr = lr_scheduler.get_lr()[0]
if hvd.rank() == 0:
model.print_alphas(logger)
# training
train(
train_loader,
valid_loader,
model,
architect,
w_optim,
alpha_optim,
lr,
epoch,
train_sampler,
)
lr_scheduler.step()
# validation
cur_step = (epoch + 1) * len(train_loader)
top1 = validate(valid_loader, model, epoch, cur_step)
top1 = metric_average(top1, name="avg_val_top1")
if hvd.rank() == 0:
# log
# genotype
genotype = model.genotype()
logger.info("genotype = {}".format(genotype))
# genotype as a image
plot_path = "." + os.path.join(config.plot_path,
"EP{:02d}".format(epoch + 1))
caption = "Epoch {}".format(epoch + 1)
plot(genotype.normal, plot_path + "-normal", caption)
# save
if best_top1 is None or best_top1 < top1:
best_top1 = top1
best_genotype = genotype
is_best = True
else:
is_best = False
# utils.save_checkpoint(model, "." + config.path, is_best)
print("")
if hvd.rank() == 0:
best_genotype = model.genotype()
with open("." + config.path + "/best_genotype.txt", "w") as f:
f.write(str(best_genotype))
logger.info("Final best TopR2@1 = {:.3f}".format(best_top1))
logger.info("Best Genotype = {}".format(best_genotype))
def search(self, train_x, train_y, valid_x, valid_y, metadata):
np.random.seed(self.seed)
# torch.cuda.set_device(self.gpu)
cudnn.benchmark = True
torch.manual_seed(self.seed)
cudnn.enabled = True
torch.cuda.manual_seed(self.seed)
helper_function()
n_classes = metadata['n_classes']
# check torch available
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
cudnn.benchmark = True
cudnn.enabled = True
data_channel = np.array(train_x).shape[1]
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(self.init_channels, data_channel, n_classes,
self.layers, criterion)
model = model.cuda()
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
self.learning_rate,
momentum=self.momentum,
weight_decay=self.weight_decay)
architect = Architect(model, self)
bin_op = bin_utils_search.BinOp(model, self)
best_genotypes = []
train_pack = list(zip(train_x, train_y))
valid_pack = list(zip(valid_x, valid_y))
train_loader = torch.utils.data.DataLoader(train_pack,
int(self.batch_size),
pin_memory=True)
valid_loader = torch.utils.data.DataLoader(valid_pack,
int(self.batch_size),
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(self.epochs), eta_min=self.learning_rate_min)
best_accuracy = 0
best_epoch = 0
train_epoch = time.time()
for epoch in range(self.epochs):
print("=== SEARCH STAGE EPOCH {} ===".format(epoch))
scheduler.step()
lr = scheduler.get_last_lr()
genotype = model.genotype()
genotype_img = model.genotype(self.gamma)
model.drop_path_prob = self.drop_path_prob * epoch / self.epochs
train_acc, train_obj = train(train_loader, valid_loader, model,
architect, criterion, optimizer, lr,
bin_op, epoch)
valid_acc, valid_obj = infer(valid_loader, model, criterion,
bin_op)
average_epoch_t = (time.time() - train_epoch) / (epoch + 1)
if best_accuracy < valid_acc:
best_accuracy = valid_acc
best_epoch = epoch
saved_model = model
if len(best_genotypes) > 0:
best_genotypes[0] = genotype
best_genotypes[1] = genotype_img
else:
best_genotypes.append(genotype)
best_genotypes.append(genotype_img)
prog_str = " Train Acc: {:>8.3f}%, Val Acc: {:>8.3f}%, Mem Alloc: {}, T Remaining Est: {}".format(
train_acc, valid_acc, cache_stats(human_readable=True),
show_time(average_epoch_t * (self.epochs - epoch)))
prog_str += "\n Current best score: Val Acc: {:>9.3f}% @ epoch {}".format(
best_accuracy, best_epoch)
prog_str += "\n genotype = {} ".format(best_genotypes)
print(prog_str)
with open('./genotypes.py', 'a') as f:
f.write(self.geno_name + ' = ' + str(best_genotypes[0]) + '\n')
f.write(self.geno_name + '_img' + ' = ' + str(best_genotypes[1]) +
'\n')
if data_channel == 3:
print('==== data_channel is 3 ====')
return_model = NetworkImageNet(self.init_channels, n_classes,
self.layers, best_genotypes[0])
return return_model
return saved_model
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
KD_loss = kd_loss.KDLoss(args.temp)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, criterion,
KD_loss)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
if not args.cls:
print('not cls')
trainloader, infer_pair_loader, infer_random_loader, valloader = dataset.load_dataset(
args.dataset, args.dataroot, batch_size=args.batch_size)
else:
trainloader, infer_pair_loader, infer_random_loader, valloader = dataset.load_dataset(
args.dataset, args.dataroot, 'pair', batch_size=args.batch_size)
print(len(trainloader))
print(len(infer_pair_loader))
print(len(valloader))
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
print(F.softmax(model.alphas_normal, dim=-1))
print(F.softmax(model.alphas_reduce, dim=-1))
# training
train_acc, train_obj = train(trainloader, infer_pair_loader, model,
architect, criterion, KD_loss, optimizer,
lr)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(infer_random_loader, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels, args.n_class, args.layers, criterion)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
_, _, n_classes, train_data, val_dat, test_dat = utils2.get_data(
"custom",
args.train_data_path,
args.val_data_path,
args.test_data_path,
cutout_length=0,
validation=True,
validation2=True,
n_class=args.n_class,
image_size=args.image_size)
#balanced split to train/validation
print(train_data)
# split data to train/validation
num_train = len(train_data)
n_val = len(val_dat)
n_test = len(test_dat)
indices1 = list(range(num_train))
indices2 = list(range(n_val))
indices3 = list(range(n_test))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices1)
valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices2)
test_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices3)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
sampler=train_sampler,
num_workers=2,
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(val_dat,
batch_size=args.batch_size,
sampler=valid_sampler,
num_workers=2,
pin_memory=True)
test_queue = torch.utils.data.DataLoader(test_dat,
batch_size=args.batch_size,
sampler=test_sampler,
num_workers=2,
pin_memory=True)
"""
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.set=='cifar100':
train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform)
else:
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]),
pin_memory=True, num_workers=2)
"""
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
bestMetric = -999
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
#print(F.softmax(model.alphas_normal, dim=-1))
#print(F.softmax(model.alphas_reduce, dim=-1))
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr,
epoch)
logging.info('train_acc %f', train_acc)
# validation
#if args.epochs-epoch<=1:
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
if (valid_acc > bestMetric):
bestMetric = valid_acc
utils.save(model, os.path.join(args.save, 'best_weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, criterion, args.rho, args.ewma)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
datapath = os.path.join(utils.get_dir(), args.data)
train_data = dset.CIFAR10(root=datapath, train=True, download=True, transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]),
pin_memory=True, num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, int(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
model.initialize_Z_and_U()
loggers = {"train": {"loss": [], "acc": [], "step": []},
"val": {"loss": [], "acc": [], "step": []},
"infer": {"loss": [], "acc": [], "step": []},
"ath": {"threshold": [], "step": []},
"zuth": {"threshold": [], "step": []},
"astep": [],
"zustep": []}
if args.constant_alpha_threshold < 0:
alpha_threshold = args.init_alpha_threshold
else:
alpha_threshold = args.constant_alpha_threshold
zu_threshold = args.init_zu_threshold
alpha_counter = 0
ewma = -1
for epoch in range(args.epochs):
valid_iter = iter(valid_queue)
model.clear_U()
scheduler.step()
lr = scheduler.get_last_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
print(torch.clamp(model.alphas_normal, min=0.1, max=1.0))
print(torch.clamp(model.alphas_reduce, min=0.1, max=1.0))
# training
train_acc, train_obj, alpha_threshold, zu_threshold, alpha_counter, ewma = train(train_queue, valid_iter, model,
architect, criterion,
optimizer, lr,
loggers, alpha_threshold,
zu_threshold, alpha_counter,
ewma,
args)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
utils.log_loss(loggers["infer"], valid_obj, valid_acc, model.clock)
logging.info('valid_acc %f', valid_acc)
utils.plot_loss_acc(loggers, args.save)
# model.update_history()
utils.save_file(recoder=model.alphas_normal_history, path=os.path.join(args.save, 'normalalpha'),
steps=loggers["train"]["step"])
utils.save_file(recoder=model.alphas_reduce_history, path=os.path.join(args.save, 'reducealpha'),
steps=loggers["train"]["step"])
utils.save_file(recoder=model.FI_normal_history, path=os.path.join(args.save, 'normalFI'),
steps=loggers["train"]["step"])
utils.save_file(recoder=model.FI_reduce_history, path=os.path.join(args.save, 'reduceFI'),
steps=loggers["train"]["step"])
scaled_FI_normal = scale(model.FI_normal_history, model.alphas_normal_history)
scaled_FI_reduce = scale(model.FI_reduce_history, model.alphas_reduce_history)
utils.save_file(recoder=scaled_FI_normal, path=os.path.join(args.save, 'normalFIscaled'),
steps=loggers["train"]["step"])
utils.save_file(recoder=scaled_FI_reduce, path=os.path.join(args.save, 'reduceFIscaled'),
steps=loggers["train"]["step"])
utils.plot_FI(loggers["train"]["step"], model.FI_history, args.save, "FI", loggers["ath"], loggers['astep'])
utils.plot_FI(loggers["train"]["step"], model.FI_ewma_history, args.save, "FI_ewma", loggers["ath"],
loggers['astep'])
utils.plot_FI(model.FI_alpha_history_step, model.FI_alpha_history, args.save, "FI_alpha", loggers["zuth"],
loggers['zustep'])
utils.save(model, os.path.join(args.save, 'weights.pt'))
genotype = model.genotype()
logging.info('genotype = %s', genotype)
f = open(os.path.join(args.save, 'genotype.txt'), "w")
f.write(str(genotype))
f.close()
def main():
logger.info("Logger is set - training start")
# set default gpu device id
torch.cuda.set_device(config.gpus[0])
# set seed
np.random.seed(config.seed)
torch.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
torch.backends.cudnn.benchmark = True
# get data with meta info
input_size, input_channels, n_classes, train_data, val_dat, test_dat = utils.get_data(
config.dataset,
config.data_path,
cutout_length=0,
validation=True,
validation2=True,
img_resize=config.img_resize)
net_crit = nn.CrossEntropyLoss().to(device)
model = SearchCNNController(input_channels,
config.init_channels,
n_classes,
config.layers,
net_crit,
device_ids=config.gpus)
#comment if generating onnix graph
model = model.to(device)
# weights optimizer
w_optim = torch.optim.SGD(model.weights(),
config.w_lr,
momentum=config.w_momentum,
weight_decay=config.w_weight_decay)
# alphas optimizer
alpha_optim = torch.optim.Adam(model.alphas(),
config.alpha_lr,
betas=(0.5, 0.999),
weight_decay=config.alpha_weight_decay)
#balanced split to train/validation
print(train_data)
# split data to train/validation
n_train = len(train_data) // int(config.data_train_proportion)
n_val = len(val_dat)
n_test = len(test_dat)
split = n_train // 2
indices1 = list(range(n_train))
indices2 = list(range(n_val))
indices3 = list(range(n_test))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices1)
valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices2)
test_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices3)
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=train_sampler,
num_workers=config.workers,
pin_memory=True)
valid_loader = torch.utils.data.DataLoader(val_dat,
batch_size=config.batch_size,
sampler=valid_sampler,
num_workers=config.workers,
pin_memory=True)
test_loader = torch.utils.data.DataLoader(test_dat,
batch_size=config.batch_size,
sampler=test_sampler,
num_workers=config.workers,
pin_memory=True)
#load
if (config.load):
model, config.epochs, w_optim, alpha_optim, net_crit = utils.load_checkpoint(
model, config.epochs, w_optim, alpha_optim, net_crit,
'/content/MyDarts/searchs/custom/checkpoint.pth.tar')
#uncomment if saving onnix graph
"""
dummy_input = Variable(torch.randn(1, 3, 64, 64))
torch.onnx.export(model, dummy_input, "rsdarts.onnx", verbose=True)
input_np = np.random.uniform(0, 1, (1, 3, 64, 64))
input_var = Variable(torch.FloatTensor(input_np))
from pytorch2keras.converter import pytorch_to_keras
# we should specify shape of the input tensor
output = model(input_var)
k_model = pytorch_to_keras(model, input_var, (3, 64, 64,), verbose=True)
error = check_error(output, k_model, input_np)
if max_error < error:
max_error = error
print('Max error: {0}'.format(max_error))
a=2/0
"""
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim, config.epochs, eta_min=config.w_lr_min)
architect = Architect(model, config.w_momentum, config.w_weight_decay)
#model = torch.load('/content/pt.darts/searchs/custom/checkpoint.pth.tar')
#print("Loaded!")
# training loop
best_top1 = 0.
best_top_overall = -999
config.epochs = 300 #BUG, config epochs ta com algum erro
for epoch in range(config.epochs):
lr_scheduler.step()
lr = lr_scheduler.get_lr()[0]
model.print_alphas(logger)
print("###################TRAINING#########################")
# training
#sample rs arch
arch = sample_arch(model)
#import pickle
#arch = pickle.load( open( "best_arch.p", "rb" ) )
train(train_loader, valid_loader, model, arch, w_optim, alpha_optim,
lr, epoch)
print("###################END TRAINING#########################")
# validation
cur_step = (epoch + 1) * len(train_loader)
print("###################VALID#########################")
top1, top_overall, _, _ = validate(valid_loader,
model,
arch,
epoch,
cur_step,
overall=True)
print("###################END VALID#########################")
# test
print("###################TEST#########################")
_, _, preds, targets = validate(test_loader,
model,
arch,
epoch,
cur_step,
overall=True,
debug=True)
s = [preds, targets]
import pickle
pickle.dump(s, open("predictions_" + str(epoch + 1) + ".p", "wb"))
#print("predictions: ",preds)
#print("targets:",targets)
print("###################END TEST#########################")
# log
# genotype
#print("Model Alpha:",model.alpha_normal)
genotype = model.genotype()
logger.info("genotype = {}".format(genotype))
# genotype as a image
plot_path = os.path.join(config.plot_path,
"EP{:02d}".format(epoch + 1))
caption = "Epoch {}".format(epoch + 1)
print("Genotype normal:", genotype.normal)
plot(genotype.normal, plot_path + "-normal", caption)
plot(genotype.reduce, plot_path + "-reduce", caption)
# save
if best_top1 < top1:
best_top1 = top1
best_genotype = genotype
best_arch = arch
is_best = True
import pickle
pickle.dump(best_arch, open("best_arch.p", "wb"))
print('best_arch:', best_arch)
print("saved!")
else:
is_best = False
#save best overall(macro avg of f1 prec and recall)
if (best_top_overall < top_overall):
best_top_overall = top_overall
best_genotype_overall = genotype
is_best_overall = True
else:
is_best_overall = False
utils.save_checkpoint(model, epoch, w_optim, alpha_optim, net_crit,
config.path, is_best, is_best_overall)
logger.info("Final best Prec@1 = {:.4%}".format(best_top1))
logger.info("Best Genotype = {}".format(best_genotype))
logger.info("Best Genotype Overall = {}".format(best_genotype_overall))
def main():
start_time = time.time()
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
torch.cuda.set_device(args.gpu)
reproducibility(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.model_name,
CIFAR_CLASSES,
sub_policies,
args.use_cuda,
args.use_parallel,
temperature=args.temperature,
criterion=criterion)
# model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
# train_transform, valid_transform = utils._data_transforms_cifar10(args)
# train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
# train_data = AugmCIFAR10(
# root=args.data, train=True, download=True,
# transform=train_transform, ops_names=sub_policies, search=True, magnitudes=model.magnitudes)
# valid_data = AugmCIFAR10(
# root=args.data, train=True, download=True,
# transform=train_transform, ops_names=sub_policies, search=False, magnitudes=model.magnitudes)
# num_train = len(train_data)
# indices = list(range(num_train))
# split = int(np.floor(args.train_portion * num_train))
# train_queue = torch.utils.data.DataLoader(
# train_data, batch_size=args.batch_size,
# sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
# pin_memory=True, num_workers=args.num_workers)
# valid_queue = torch.utils.data.DataLoader(
# valid_data, batch_size=args.batch_size,
# sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]),
# pin_memory=True, num_workers=args.num_workers)
train_queue, valid_queue = get_dataloaders(args.dataset,
args.batch_size,
args.num_workers,
args.dataroot,
sub_policies,
model.magnitudes,
args.cutout,
args.cutout_length,
split=args.train_portion,
split_idx=0,
target_lb=-1)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
# logging.info('genotype = %s', genotype)
print_genotype(genotype)
# logging.info('%s' % str(torch.nn.functional.softmax(model.ops_weights, dim=-1)))
probs = model.ops_weights
# logging.info('%s' % str(probs / probs.sum(-1, keepdim=True)))
logging.info('%s' % str(torch.nn.functional.softmax(probs, dim=-1)))
logging.info('%s' % str(model.probabilities.clamp(0, 1)))
logging.info('%s' % str(model.magnitudes.clamp(0, 1)))
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
end_time = time.time()
elapsed = end_time - start_time
logging.info('elapsed time: %.3f Hours' % (elapsed / 3600.))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels,
CIFAR_CLASSES,
args.layers,
criterion,
k=args.k)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.dataset == 'cifar100':
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
else:
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True)
architect = Architect(model, args)
# configure progressive parameter
epoch = 0
ks = [6, 4]
num_keeps = [7, 4]
train_epochs = [2, 2] if 'debug' in args.save else [25, 25]
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(sum(train_epochs)), eta_min=args.learning_rate_min)
for i, current_epochs in enumerate(train_epochs):
for e in range(current_epochs):
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
model.show_arch_parameters()
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr,
e)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
epoch += 1
scheduler.step()
utils.save(model, os.path.join(args.save, 'weights.pt'))
if not i == len(train_epochs) - 1:
model.pruning(num_keeps[i + 1])
# architect.pruning([model.mask_normal, model.mask_reduce])
model.wider(ks[i + 1])
optimizer = configure_optimizer(
optimizer,
torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay))
scheduler = configure_scheduler(
scheduler,
torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer,
float(sum(train_epochs)),
eta_min=args.learning_rate_min))
logging.info('pruning finish, %d ops left per edge',
num_keeps[i + 1])
logging.info('network wider finish, current pc parameter %d',
ks[i + 1])
genotype = model.genotype()
logging.info('genotype = %s', genotype)
model.show_arch_parameters()
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
# torch.cuda.set_device(args.gpu)
gpus = [int(i) for i in args.gpu.split(',')]
if len(gpus) == 1:
torch.cuda.set_device(int(args.gpu))
# cudnn.benchmark = True
torch.manual_seed(args.seed)
# cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %s' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, criterion)
model = model.cuda()
if len(gpus)>1:
print("True")
model = nn.parallel.DataParallel(model, device_ids=gpus, output_device=gpus[0])
model = model.module
arch_params = list(map(id, model.arch_parameters()))
weight_params = filter(lambda p: id(p) not in arch_params,
model.parameters())
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(
# model.parameters(),
weight_params,
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
#optimizer = nn.DataParallel(optimizer, device_ids=gpus)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]),
pin_memory=True, num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, criterion, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
print(F.softmax(model.alphas_normal, dim=-1))
print(F.softmax(model.alphas_reduce, dim=-1))
# training
train_acc, train_obj = train(train_queue, valid_queue, model, architect, criterion, optimizer, lr)
logging.info('train_acc %f', train_acc)
# validation
with torch.no_grad():
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def nas(args: Namespace, task: Task, preprocess_func: Compose) -> Module:
''' Network Architecture Search method
Given task and preprocess function, this method returns a model output by NAS.
The implementation of DARTS is available at https://github.com/alphadl/darts.pytorch1.1
'''
# TODO: Replace model with the output by NAS
args.save = 'search-{}-{}'.format(args.save,
time.strftime("%Y%m%d-%H%M%S"))
utils.create_exp_dir(args.save, scripts_to_save=glob.glob('*.py'))
log_format = '%(asctime)s %(message)s'
logging.basicConfig(stream=sys.stdout,
level=logging.INFO,
format=log_format,
datefmt='%m/%d %I:%M:%S %p')
fh = logging.FileHandler(os.path.join(args.save, 'log.txt'))
fh.setFormatter(logging.Formatter(log_format))
logging.getLogger().addHandler(fh)
CLASSES = task.n_classes
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
# torch.cuda.set_device(args.gpu)
#gpus = [int(args.gpu)]
gpus = [int(i) for i in args.gpu.split(',')]
if len(gpus) == 1:
torch.cuda.set_device(int(args.gpu))
# cudnn.benchmark = True
torch.manual_seed(args.seed)
# cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %s' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels, CLASSES, args.layers, criterion)
model = model.cuda()
if len(gpus) > 1:
print("True")
model = nn.parallel.DataParallel(model,
device_ids=gpus,
output_device=gpus[0])
model = model.module
arch_params = list(map(id, model.arch_parameters()))
weight_params = filter(lambda p: id(p) not in arch_params,
model.parameters())
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(
# model.parameters(),
weight_params,
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
optimizer = nn.DataParallel(optimizer, device_ids=gpus)
if task.name == 'cifar100':
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=preprocess_func)
#train_transform, valid_transform = utils._data_transforms_cifar10(args)
#train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform)
elif task.name == 'cifar10':
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=preprocess_func)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer.module, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, criterion, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
print(F.softmax(model.alphas_normal, dim=-1))
print(F.softmax(model.alphas_reduce, dim=-1))
# training
train_acc, train_obj = train(args, train_queue, valid_queue, model,
architect, criterion, optimizer, lr)
logging.info('train_acc %f', train_acc)
# validation
with torch.no_grad():
valid_acc, valid_obj = infer(args, valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
# return a neural network model (torch.nn.Module)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
model = NetworkClassification(36, task.n_classes, 20, False, genotype)
return model
def main():
path_to_best_loss_eval = "./generator/best_loss_model_{}.csv".format(args.seed)
path_to_best_model = "./generator/best_model_{}.pth".format(args.seed)
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, criterion)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]),
pin_memory=True, num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
# ================= DONAS ==========================
low_flops = args.low_flops
high_flops = args.high_flops
nodes, edges = model.get_arch_param_nums()
lookup_table = LookUpTable(edges, nodes)
arch_param_nums = nodes * edges
generator = get_generator(20)
generator = generator.cuda()
backbone_pool = BackbonePool(nodes, edges, lookup_table, arch_param_nums)
backbone = backbone_pool.get_backbone((low_flops+high_flops)/2)
g_optimizer = torch.optim.Adam(generator.parameters(),
weight_decay=0,
lr=0.001,
betas=(0.5, 0.999))
tau = 5
best_hc_loss = 100000
# ================= DONAS ==========================
architect = Architect(model, generator, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
# training
train_acc, train_obj = train(train_queue, valid_queue, model, architect, criterion, optimizer, lr, low_flops, high_flops, backbone, tau)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion, generator, backbone, (low_flops+high_flops)//2, lookup_table)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
evalulate_metric, total_loss, kendall_tau = evalulate_generator(generator, backbone, lookup_table, low_flops, high_flops)
if total_loss < best_hc_loss:
logger.log("Best hc loss : {}. Save model!".format(total_loss))
save_generator_evaluate_metric(evalulate_metric, path_to_best_loss_eval)
best_hc_loss = total_loss
if valid_acc > best_top1:
logger.log("Best top1-avg : {}. Save model!".format(valid_acc_top1))
save_model(generator, path_to_best_model)
best_top1 = valid_acc
tau *= 0.95
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
if args.random_seed:
args.seed = np.random.randint(0, 1000, 1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
# dataset modelnet
pre_transform, transform = T.NormalizeScale(), T.SamplePoints(
args.num_points)
train_dataset = GeoData.ModelNet(os.path.join(args.data, 'modelnet10'),
'10', True, transform, pre_transform)
train_queue = DenseDataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.batch_size // 2)
test_dataset = GeoData.ModelNet(os.path.join(args.data, 'modelnet10'),
'10', False, transform, pre_transform)
valid_queue = DenseDataLoader(test_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.batch_size // 2)
n_classes = train_queue.dataset.num_classes
criterion = torch.nn.CrossEntropyLoss().cuda()
model = Network(args.init_channels,
n_classes,
args.num_cells,
criterion,
args.n_steps,
in_channels=args.in_channels,
emb_dims=args.emb_dims,
dropout=args.dropout,
k=args.k).cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
num_edges = model._steps * 2
post_train = 5
# import pdb;pdb.set_trace()
args.epochs = args.warmup_dec_epoch + args.decision_freq * (
num_edges - 1) + post_train + 1
logging.info("total epochs: %d", args.epochs)
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
normal_selected_idxs = torch.tensor(len(model.alphas_normal) * [-1],
requires_grad=False,
dtype=torch.int).cuda()
normal_candidate_flags = torch.tensor(len(model.alphas_normal) * [True],
requires_grad=False,
dtype=torch.bool).cuda()
logging.info('normal_selected_idxs: {}'.format(normal_selected_idxs))
logging.info('normal_candidate_flags: {}'.format(normal_candidate_flags))
model.normal_selected_idxs = normal_selected_idxs
model.normal_candidate_flags = normal_candidate_flags
print(F.softmax(torch.stack(model.alphas_normal, dim=0), dim=-1).detach())
count = 0
normal_probs_history = []
train_losses, valid_losses = utils.AverageMeter(), utils.AverageMeter()
for epoch in range(args.epochs):
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
# training
# import pdb;pdb.set_trace()
att = model.show_att()
beta = model.show_beta()
train_acc, train_losses = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr,
train_losses)
valid_overall_acc, valid_class_acc, valid_losses = infer(
valid_queue, model, criterion, valid_losses)
logging.info(
'train_acc %f\tvalid_overall_acc %f \t valid_class_acc %f',
train_acc, valid_overall_acc, valid_class_acc)
logging.info('beta %s', beta.cpu().detach().numpy())
logging.info('att %s', att.cpu().detach().numpy())
# make edge decisions
saved_memory_normal, model.normal_selected_idxs, \
model.normal_candidate_flags = edge_decision('normal',
model.alphas_normal,
model.normal_selected_idxs,
model.normal_candidate_flags,
normal_probs_history,
epoch,
model,
args)
if saved_memory_normal:
del train_queue, valid_queue
torch.cuda.empty_cache()
count += 1
new_batch_size = args.batch_size + args.batch_increase * count
logging.info("new_batch_size = {}".format(new_batch_size))
train_queue = DenseDataLoader(train_dataset,
batch_size=new_batch_size,
shuffle=True,
num_workers=args.batch_size // 2)
valid_queue = DenseDataLoader(test_dataset,
batch_size=new_batch_size,
shuffle=False,
num_workers=args.batch_size // 2)
# post validation
if args.post_val:
post_valid_overall_acc, post_valid_class_acc, valid_losses = infer(
valid_queue, model, criterion, valid_losses)
logging.info('post_valid_overall_acc %f',
post_valid_overall_acc)
writer.add_scalar('stats/train_acc', train_acc, epoch)
writer.add_scalar('stats/valid_overall_acc', valid_overall_acc, epoch)
writer.add_scalar('stats/valid_class_acc', valid_class_acc, epoch)
utils.save(model, os.path.join(args.save, 'weights.pt'))
scheduler.step()
logging.info("#" * 30 + " Done " + "#" * 30)
logging.info('genotype = %s', model.get_genotype())
def main():
parser = argparse.ArgumentParser(description='TrainingContainer')
parser.add_argument('--algorithm-settings',
type=str,
default="",
help="algorithm settings")
parser.add_argument('--search-space',
type=str,
default="",
help="search space for the neural architecture search")
parser.add_argument('--num-layers',
type=str,
default="",
help="number of layers of the neural network")
args = parser.parse_args()
# Get Algorithm Settings
algorithm_settings = args.algorithm_settings.replace("\'", "\"")
algorithm_settings = json.loads(algorithm_settings)
print(">>> Algorithm settings")
for key, value in algorithm_settings.items():
if len(key) > 13:
print("{}\t{}".format(key, value))
elif len(key) < 5:
print("{}\t\t\t{}".format(key, value))
else:
print("{}\t\t{}".format(key, value))
print()
num_epochs = int(algorithm_settings["num_epochs"])
w_lr = float(algorithm_settings["w_lr"])
w_lr_min = float(algorithm_settings["w_lr_min"])
w_momentum = float(algorithm_settings["w_momentum"])
w_weight_decay = float(algorithm_settings["w_weight_decay"])
w_grad_clip = float(algorithm_settings["w_grad_clip"])
alpha_lr = float(algorithm_settings["alpha_lr"])
alpha_weight_decay = float(algorithm_settings["alpha_weight_decay"])
batch_size = int(algorithm_settings["batch_size"])
num_workers = int(algorithm_settings["num_workers"])
init_channels = int(algorithm_settings["init_channels"])
print_step = int(algorithm_settings["print_step"])
num_nodes = int(algorithm_settings["num_nodes"])
stem_multiplier = int(algorithm_settings["stem_multiplier"])
# Get Search Space
search_space = args.search_space.replace("\'", "\"")
search_space = json.loads(search_space)
search_space = SearchSpace(search_space)
# Get Num Layers
num_layers = int(args.num_layers)
print("Number of layers {}\n".format(num_layers))
# Set GPU Device
# Currently use only first available GPU
# TODO: Add multi GPU support
# TODO: Add functionality to select GPU
all_gpus = list(range(torch.cuda.device_count()))
if len(all_gpus) > 0:
device = torch.device("cuda")
torch.cuda.set_device(all_gpus[0])
np.random.seed(2)
torch.manual_seed(2)
torch.cuda.manual_seed_all(2)
torch.backends.cudnn.benchmark = True
print(">>> Use GPU for Training <<<")
print("Device ID: {}".format(torch.cuda.current_device()))
print("Device name: {}".format(torch.cuda.get_device_name(0)))
print("Device availability: {}\n".format(torch.cuda.is_available()))
else:
device = torch.device("cpu")
print(">>> Use CPU for Training <<<")
# Get dataset with meta information
# TODO: Add support for more dataset
input_channels, num_classes, train_data = utils.get_dataset()
criterion = nn.CrossEntropyLoss().to(device)
model = NetworkCNN(init_channels, input_channels, num_classes, num_layers,
criterion, search_space, num_nodes, stem_multiplier)
model = model.to(device)
# Weights optimizer
w_optim = torch.optim.SGD(model.getWeights(),
w_lr,
momentum=w_momentum,
weight_decay=w_weight_decay)
# Alphas optimizer
alpha_optim = torch.optim.Adam(model.getAlphas(),
alpha_lr,
betas=(0.5, 0.999),
weight_decay=alpha_weight_decay)
# Split data to train/validation
num_train = len(train_data)
split = num_train // 2
indices = list(range(num_train))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(
indices[:split])
valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(
indices[split:])
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=batch_size,
sampler=train_sampler,
num_workers=num_workers,
pin_memory=True)
valid_loader = torch.utils.data.DataLoader(train_data,
batch_size=batch_size,
sampler=valid_sampler,
num_workers=num_workers,
pin_memory=True)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(w_optim,
num_epochs,
eta_min=w_lr_min)
architect = Architect(model, w_momentum, w_weight_decay)
# Start training
best_top1 = 0.
for epoch in range(num_epochs):
lr_scheduler.step()
lr = lr_scheduler.get_lr()[0]
model.print_alphas()
# Training
print(">>> Training")
train(train_loader, valid_loader, model, architect, w_optim,
alpha_optim, lr, epoch, num_epochs, device, w_grad_clip,
print_step)
# Validation
print("\n>>> Validation")
cur_step = (epoch + 1) * len(train_loader)
top1 = validate(valid_loader, model, epoch, cur_step, num_epochs,
device, print_step)
# Print genotype
genotype = model.genotype(search_space)
print("\nModel genotype = {}".format(genotype))
# Modify best top1
if top1 > best_top1:
best_top1 = top1
best_genotype = genotype
print("Final best Prec@1 = {:.4%}".format(best_top1))
print("\nBest-Genotype={}".format(str(best_genotype).replace(" ", "")))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, criterion)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10_simple(
args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
print(F.softmax(model.alphas_normal, dim=-1))
print(F.softmax(model.alphas_reduce, dim=-1))
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr)
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
# adversarial testing
adv_acc, adv_obj = infer_minibatch(valid_queue, model, criterion)
logging.info('adv_acc %f', adv_acc)
#infer_minibatch(valid_queue, model, criterion)
utils.save(model,
os.path.join(args.save, 'weights_' + str(epoch) + '.pt'))
def main(config, writer, logger):
logger.info("Logger is set - training search start")
input_size, input_channels, n_classes, train_data = utils.get_data(
config.dataset, config.data_path, cutout_length=0, validation=False)
net_crit = nn.CrossEntropyLoss().to(config.device)
model = SearchCNNController(input_channels, config.init_channels, n_classes, config.layers, net_crit).to(config.device)
# weights optimizer
w_optim = torch.optim.SGD(model.weights(), config.w_lr, momentum=config.w_momentum,
weight_decay=config.w_weight_decay)
# alphas optimizer
alpha_optim = torch.optim.Adam(model.alphas(), config.alpha_lr, betas=(config.alpha_beta1, config.alpha_beta2),
weight_decay=config.alpha_weight_decay)
# split data to train/validation
n_train = len(train_data)
split = n_train // 2
indices = list(range(n_train))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:split])
valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[split:])
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=train_sampler,
num_workers=config.workers,
pin_memory=True)
valid_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=valid_sampler,
num_workers=config.workers,
pin_memory=True)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(w_optim, config.epochs, eta_min=config.w_lr_min)
architect = Architect(model, config.w_momentum, config.w_weight_decay)
# training loop
best_top1 = 0.
for epoch in range(config.epochs):
lr_scheduler.step()
lr = lr_scheduler.get_lr()[0]
model.print_alphas(logger)
# training
train(train_loader, valid_loader, model, architect, w_optim, alpha_optim, lr, epoch, config, writer, logger)
# validation
cur_step = (epoch+1) * len(train_loader)
top1 = validate(valid_loader, model, epoch, cur_step, config, writer, logger)
# log
# genotype
genotype = model.genotype()
logger.info("genotype = {}".format(genotype))
# genotype as a image
plot_path = os.path.join(config.plot_path, "EP{:02d}".format(epoch+1))
caption = "Epoch {}".format(epoch+1)
plot(genotype.normal, plot_path + "-normal", caption)
plot(genotype.reduce, plot_path + "-reduce", caption)
# save
if best_top1 < top1:
best_top1 = top1
best_genotype = genotype
is_best = True
else:
is_best = False
utils.save_checkpoint(model, config.path, is_best)
logger.info("Final best Prec@1 = {:.4%}".format(best_top1))
logger.info("Best Genotype = {}".format(best_genotype))
