def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels, args.n_class, args.layers, criterion)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
_, _, n_classes, train_data, val_dat, test_dat = utils2.get_data(
"custom",
args.train_data_path,
args.val_data_path,
args.test_data_path,
cutout_length=0,
validation=True,
validation2=True,
n_class=args.n_class,
image_size=args.image_size)
#balanced split to train/validation
print(train_data)
# split data to train/validation
num_train = len(train_data)
n_val = len(val_dat)
n_test = len(test_dat)
indices1 = list(range(num_train))
indices2 = list(range(n_val))
indices3 = list(range(n_test))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices1)
valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices2)
test_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices3)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
sampler=train_sampler,
num_workers=2,
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(val_dat,
batch_size=args.batch_size,
sampler=valid_sampler,
num_workers=2,
pin_memory=True)
test_queue = torch.utils.data.DataLoader(test_dat,
batch_size=args.batch_size,
sampler=test_sampler,
num_workers=2,
pin_memory=True)
"""
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.set=='cifar100':
train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform)
else:
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]),
pin_memory=True, num_workers=2)
"""
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
bestMetric = -999
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
#print(F.softmax(model.alphas_normal, dim=-1))
#print(F.softmax(model.alphas_reduce, dim=-1))
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr,
epoch)
logging.info('train_acc %f', train_acc)
# validation
#if args.epochs-epoch<=1:
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
if (valid_acc > bestMetric):
bestMetric = valid_acc
utils.save(model, os.path.join(args.save, 'best_weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info("args = %s", args)
traindir = os.path.join(args.data, 'train')
valdir = os.path.join(args.data, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
#dataset split
train_data = dset.ImageFolder(
traindir,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
valid_data = dset.ImageFolder(
valdir,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
num_train = len(train_data)
num_val = len(valid_data)
print('# images to train network: %d' % num_train)
print('# images to validate network: %d' % num_val)
model = Network(args.init_channels,
CLASSES,
args.layers,
criterion,
k=args.k)
model = nn.DataParallel(model)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
optimizer_a = torch.optim.Adam(model.module.arch_parameters(),
lr=args.arch_learning_rate,
betas=(0.5, 0.999),
weight_decay=args.arch_weight_decay)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=args.workers)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=args.workers)
# configure progressive parameter
epoch = 0
ks = [6, 3]
num_keeps = [7, 4]
train_epochs = [2, 2] if 'debug' in args.save else [25, 25]
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(sum(train_epochs)), eta_min=args.learning_rate_min)
lr = args.learning_rate
for i, current_epochs in enumerate(train_epochs):
for e in range(current_epochs):
current_lr = scheduler.get_lr()[0]
logging.info('Epoch: %d lr: %e', epoch, current_lr)
if epoch < 5 and args.batch_size > 256:
for param_group in optimizer.param_groups:
param_group['lr'] = lr * (epoch + 1) / 5.0
logging.info('Warming-up Epoch: %d, LR: %e', epoch,
lr * (epoch + 1) / 5.0)
print(optimizer)
genotype = model.module.genotype()
logging.info('genotype = %s', genotype)
model.module.show_arch_parameters()
epoch_start = time.time()
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
optimizer, optimizer_a, criterion, e)
logging.info('Train_acc %f', train_acc)
# validation
if epoch >= 47:
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('Valid_acc %f', valid_acc)
#test_acc, test_obj = infer(test_queue, model, criterion)
#logging.info('Test_acc %f', test_acc)
epoch += 1
scheduler.step()
epoch_duration = time.time() - epoch_start
logging.info('Epoch time: %ds.', epoch_duration)
# utils.save(model, os.path.join(args.save, 'weights.pt'))
if not i == len(train_epochs) - 1:
model.module.pruning(num_keeps[i + 1])
model.module.wider(ks[i + 1])
optimizer = configure_optimizer(
optimizer,
torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay))
scheduler = configure_scheduler(
scheduler,
torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer,
float(sum(train_epochs)),
eta_min=args.learning_rate_min))
logging.info('pruning finish, %d ops left per edge',
num_keeps[i + 1])
logging.info('network wider finish, current pc parameter %d',
ks[i + 1])
genotype = model.module.genotype()
logging.info('genotype = %s', genotype)
model.module.show_arch_parameters()
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
#torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
#logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
#dataset_dir = '/cache/'
#pre.split_dataset(dataset_dir)
#sys.exit(1)
# dataset prepare
traindir = data_dir = os.path.join(data_dir, 'train')
valdir = data_dir = os.path.join(data_dir, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
#dataset split
train_data1 = dset.ImageFolder(
traindir,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
train_data2 = dset.ImageFolder(
valdir,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
valid_data = dset.ImageFolder(
valdir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
]))
num_train = len(train_data1)
num_val = len(train_data2)
print('# images to train network: %d' % num_train)
print('# images to validate network: %d' % num_val)
model = Network(args.init_channels, CLASSES, args.layers, criterion)
model = torch.nn.DataParallel(model)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
optimizer_a = torch.optim.Adam(model.module.arch_parameters(),
lr=args.arch_learning_rate,
betas=(0.5, 0.999),
weight_decay=args.arch_weight_decay)
test_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=args.workers)
train_queue = torch.utils.data.DataLoader(train_data1,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=args.workers)
valid_queue = torch.utils.data.DataLoader(train_data2,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=args.workers)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
#architect = Architect(model, args)
lr = args.learning_rate
for epoch in range(args.epochs):
scheduler.step()
current_lr = scheduler.get_lr()[0]
logging.info('Epoch: %d lr: %e', epoch, current_lr)
if epoch < 5 and args.batch_size > 256:
for param_group in optimizer.param_groups:
param_group['lr'] = lr * (epoch + 1) / 5.0
logging.info('Warming-up Epoch: %d, LR: %e', epoch,
lr * (epoch + 1) / 5.0)
print(optimizer)
genotype = model.module.genotype()
logging.info('genotype = %s', genotype)
arch_param = model.module.arch_parameters()
logging.info(F.softmax(arch_param[0], dim=-1))
logging.info(F.softmax(arch_param[1], dim=-1))
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
optimizer, optimizer_a, criterion, lr,
epoch)
logging.info('Train_acc %f', train_acc)
# validation
if epoch >= 47:
valid_acc, valid_obj = infer(valid_queue, model, criterion)
#test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('Valid_acc %f', valid_acc)
def main():
np.random.seed(args.seed)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
if engine.local_rank == 0:
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
#data
traindir = os.path.join(args.data_dir, 'train')
valdir = os.path.join(args.data_dir, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_data1 = dset.ImageFolder(traindir, transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
train_data2 = dset.ImageFolder(valdir, transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
train_data_full=ConcatDataset([train_data1,train_data2])
valid_data = dset.ImageFolder(valdir, transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
]))
num_train = len(train_data1)
num_val = len(train_data2)
if engine.local_rank == 0:
logging.info('# images to train network: %d' % num_train)
logging.info('# images to validate network: %d' % num_val)
train_queue_A = engine.get_dataloader(train_data1)
train_queue_B = engine.get_dataloader(train_data2)
train_queue_Full = engine.get_dataloader(train_data_full)
valid_queue = engine.get_dataloader(valid_data)
model = Network(args.init_channels, CLASSES, args.op_search_layers, criterion)
model.load_state_dict(torch.load("logs/search/search-imagenet_1_002_sub14_32-5448/weights_op_search.pth"))
model.load_arch(torch.load("logs/search/search-imagenet_1_002_sub14_32-5448/arch_op_search.pth"))
start_epoch = 30
model = model.cuda()
if engine.local_rank == 0:
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
model = engine.data_parallel(model)
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, Tp_Pretrain_Start, eta_min=args.learning_rate_min)
arch_optimizer = torch.optim.Adam(model.module.arch_parameters(),
lr=args.arch_learning_rate, betas=(0.9, 0.999),
weight_decay=args.arch_weight_decay)
for epoch in range(start_epoch,args.epochs):
if epoch == Op_Pretrain_Start:
model.module.phase = 'op_pretrain'
if engine.local_rank == 0:
logging.info("Begin operation pretrain!")
elif epoch == Op_Search_Start:
model.module.phase = 'op_search'
if engine.local_rank == 0:
logging.info("Begin operation search!")
elif epoch == Tp_Pretrain_Start:
model.module.__init__(args.init_channels, CLASSES, args.tp_search_layers, criterion, init_arch=False)
model.module.phase = 'tp_pretrain'
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate, # use twice data to update parameters
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
model.module.prune_model()
arch_optimizer = torch.optim.Adam(model.module.arch_parameters(),
lr=args.arch_learning_rate, betas=(0.9, 0.999),
weight_decay=args.arch_weight_decay)
model = model.cuda()
if engine.local_rank == 0:
logging.info("Prune model finish!")
logging.info("Load Prune Architecture finish!")
logging.info("Begin topology pretrain!")
elif epoch == Tp_Search_Start:
model.module.phase = 'tp_search'
if engine.local_rank == 0:
logging.info("Begin topology search!")
else:
pass
if 'pretrain' in model.module.phase:
model.module.T = 1.0
else:
if 'op' in model.module.phase:
model.module.T = 1.0
else:
model.module.T = 10 * pow(Tp_Anneal_Rate, epoch - Tp_Search_Start)
scheduler.step(epoch)
lr = scheduler.get_lr()[0]
if engine.local_rank == 0:
logging.info('epoch:%d phase:%s lr:%e', epoch, model.module.phase, lr)
print_genotype(model)
# training
if 'op' in model.module.phase:
train_queue_A.sampler.set_epoch(epoch)
train_queue_B.sampler.set_epoch(epoch)
train_acc, train_obj = train_op(train_queue_A, train_queue_B, model, criterion, optimizer,arch_optimizer,engine)
else:
train_queue_A.sampler.set_epoch(epoch)
train_queue_Full.sampler.set_epoch(epoch)
train_acc, train_obj = train_tp(train_queue_A, train_queue_Full, model, criterion, optimizer, arch_optimizer,engine)
if engine.local_rank == 0:
logging.info('train_acc %f', train_acc)
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
if engine.local_rank == 0:
logging.info('valid_acc %f', valid_acc)
if engine.local_rank == 0:
utils.save_dist(model, os.path.join(args.save, 'weights_%s.pth' % model.module.phase))
model.module.save_arch(os.path.join(args.save, 'arch_%s.pth' % model.module.phase))
if engine.local_rank == 0:
print_genotype(model)
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
criterion_smooth = CrossEntropyLabelSmooth(CLASSES, 0.1)
criterion_smooth = criterion_smooth.cuda()
if args.pretrain_weight:
model_cifar = Network(16, 10, 8, nn.CrossEntropyLoss().cuda())
utils.load(model_cifar, args.pretrain_weight)
debug_model(model_cifar)
sdict = repeat_weight(model_cifar)
model = Network(16 * args.channel_scale, CLASSES, 8, criterion, 4, 4, 4,
args.residual_wei, args.shrink_channel)
#model = Network(args.init_channels, CLASSES, args.layers, criterion)
model = model.cuda()
logging.info("param size before resize = %fMB",
utils.count_parameters_in_MB(model))
if args.pretrain_weight:
init_weight(sdict, model)
logging.info("param size after resize = %fMB",
utils.count_parameters_in_MB(model))
del sdict
del model_cifar
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
traindir = os.path.join(args.data, 'train')
validdir = os.path.join(args.data, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_transform = transforms.Compose([
transforms.RandomResizedCrop(int(224 * args.input_scale)),
transforms.RandomHorizontalFlip(),
transforms.ColorJitter(brightness=0.4,
contrast=0.4,
saturation=0.4,
hue=0.2),
transforms.ToTensor(),
normalize,
])
valid_transform = transforms.Compose([
transforms.Resize(int(256 * args.input_scale)),
transforms.CenterCrop(int(224 * args.input_scale)),
transforms.ToTensor(),
normalize,
])
train_data = dset.ImageFolder(traindir, train_transform)
'''
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
'''
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True,
num_workers=4 * args.ngpu)
valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True,
num_workers=4 * args.ngpu)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
args.decay_period,
gamma=args.gamma)
#scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
# optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
print(F.softmax(model.alphas_normal, dim=-1))
print(F.softmax(model.alphas_reduce, dim=-1))
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
architect, criterion, optimizer, lr)
logging.info('train_acc %f', train_acc)
if epoch == args.epochs - 1:
# validation
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(initGpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % initGpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels, CLASSES, args.layers, criterion)
torch.cuda.set_device(initGpu)
model = nn.DataParallel(model, device_ids)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
data_dir = args.tmp_data_dir
traindir = os.path.join(data_dir, 'train')
validdir = os.path.join(data_dir, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
train_data = dset.ImageFolder(
traindir,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ColorJitter(
brightness=0.4,
contrast=0.4,
saturation=0.4,
hue=0.2),
transforms.ToTensor(),
normalize,
]))
valid_data = dset.ImageFolder(
validdir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
]))
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(0.5 * num_train))
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True,
num_workers=args.workers)
valid_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]),
pin_memory=True,
num_workers=args.workers)
architect = Architect(model, args)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
##if model is DataParallel
if(hasattr(model, 'module') ):
genotype = model.module.genotype()
else:
genotype = model.genotype()
logging.info('genotype = %s', genotype)
#print(F.softmax(model.alphas_normal, dim=-1))
#print(F.softmax(model.alphas_reduce, dim=-1))
# training
train_acc, train_obj = train(train_queue, valid_queue, model, architect, criterion, optimizer, lr,epoch)
logging.info('train_acc %f', train_acc)
# validation
if args.epochs-epoch<=1:
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
print(args.gpu)
#torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = args.cudnn
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %s' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels, args.n_class, args.layers, criterion)
#model = model.cuda()
model = torch.nn.DataParallel(model).cuda()
#model = torch.nn.DataParallel(model,device_ids=[1]).cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
_, _, n_classes, train_data, test_dat = utils3.get_data(
"custom",
args.data_path,
args.labels_path,
cutout_length=0,
validation=True,
validation2=True,
n_class=args.n_class,
image_size=args.image_size)
#balanced split to train/validation
print(train_data)
# split data to train/validation
num_train = len(train_data)
n_test = len(test_dat)
indices1 = list(range(num_train))
indices3 = list(range(n_test))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices1)
test_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices3)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
sampler=train_sampler,
num_workers=1,
pin_memory=True)
test_queue = torch.utils.data.DataLoader(test_dat,
batch_size=args.batch_size,
sampler=test_sampler,
num_workers=1,
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
bestMetric = -999
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.module.genotype()
logging.info('genotype = %s', genotype)
#print(F.softmax(model.alphas_normal, dim=-1))
#print(F.softmax(model.alphas_reduce, dim=-1))
# training
train_acc, train_obj = train(train_queue, test_queue, model, architect,
criterion, optimizer, lr, epoch)
logging.info('train_acc %f', train_acc)
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, criterion, darts = args.darts, pc = args.pc)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
traindir = os.path.join(args.data, "train")
valdir = os.path.join(args.data, "val")
train_transform, valid_transform = utils._data_transforms_imagenet(args)
train_data1 = dset.ImageNet(traindir, transform=train_transform)
train_data2 = dset.ImageNet(valdir, transform=train_transform)
num_train = len(train_data1)
num_val = len(train_data2)
train_grow = int(np.floor(args.grow_portion * num_train))
valid_grow = int(np.floor(args.grow_portion * (num_val)))
train_indices = range(num_train)
valid_indices = range(num_val)
train_queue = torch.utils.data.DataLoader(
train_data1, batch_size=args.batch_size,
shuffle = True,
pin_memory=True, num_workers=4)
valid_queue = torch.utils.data.DataLoader(
train_data2, batch_size=args.batch_size,
shuffle = True
pin_memory=True, num_workers=4)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
architect = Architect(model, args)
for epoch in range(args.epochs):
lr = scheduler.get_last_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
genotype = model.genotype()
logging.info('genotype = %s', genotype)
if not args.darts:
alphas_reduce = torch.where(model.alphas_reduce==0,torch.FloatTensor([float("-inf")]).cuda(),model.alphas_reduce)
alphas_normal = torch.where(model.alphas_normal==0,torch.FloatTensor([float("-inf")]).cuda(),model.alphas_normal)
logging.info(F.softmax(alphas_normal, dim=-1))
logging.info(F.softmax(alphas_reduce, dim=-1))
else:
logging.info(F.softmax(model.alphas_normal, dim=-1))
logging.info(F.softmax(model.alphas_reduce, dim=-1))
# print("post grow")
# alphas_reduce = torch.where(model.alphas_reduce==0,torch.FloatTensor([float("-inf")]).cuda(),model.alphas_reduce)
# alphas_normal = torch.where(model.alphas_normal==0,torch.FloatTensor([float("-inf")]).cuda(),model.alphas_normal)
# print(F.softmax(alphas_normal, dim=-1))
# print(F.softmax(alphas_reduce, dim=-1))
# training
train_s = time.time()
train_acc, train_obj = train(train_queue, valid_queue, model, architect, criterion, optimizer, lr, epoch)
logging.info('train_acc %f', train_acc)
train_e = time.time()
t_record["train"]+=(train_e-train_s)
if not args.darts and epoch > args.init:
architect.print_arch_grad()
#scheduler update
scheduler.step()
#if architect.scheduler is not None:
# architect.scheduler.step()
# validation
if epoch > 47:
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
if not args.darts and epoch % args.grow_freq == 0 and epoch < args.epochs-args.final and epoch >= args.init:
train_indices_grow = np.random.choice(train_indices, train_grow, replace = False)
valid_indices_grow = np.random.choice(valid_indices, valid_grow, replace = False)
train_grow_queue = torch.utils.data.DataLoader(
train_data1, batch_size=args.grow_batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(train_indices_grow),
pin_memory=True, num_workers=4)
valid_grow_queue = torch.utils.data.DataLoader(
train_data2, batch_size=args.grow_batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(valid_indices_grow),
pin_memory=True, num_workers=4)
grow_s = time.time()
grow(train_grow_queue, valid_grow_queue, model, architect, criterion, optimizer, lr, args.num_grow)
grow_e = time.time()
t_record["grow"]+=(grow_e-grow_s)
for param_group in optimizer.param_groups:
param_group["lr"] = args.learning_rate_middle
param_group["initial_lr"] = args.learning_rate_middle
optimizer.defaults["lr"] = args.learning_rate_middle
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, args.grow_freq, eta_min=args.learning_rate_stable)
if not args.darts and epoch == args.epochs-args.final:
for param_group in optimizer.param_groups:
param_group["lr"] = args.learning_rate_stable
param_group["initial_lr"] = args.learning_rate_stable
optimizer.defaults["lr"]=args.learning_rate_stable
#scheduler = None
#scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
# optimizer, 10.0, eta_min=args.learning_rate_min)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, args.final, eta_min=args.learning_rate_min)
#scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
# optimizer, 10.0, eta_min=args.learning_rate_min)
#for param_group in architect.optimizer.param_groups:
# param_group["lr"] = architect.lr
#architect.scheduler = torch.optim.lr_scheduler.StepLR(architect.optimizer, step_size = 1, gamma=0.9)
torch.save(t_record, "time_record.pt")
utils.save(model, os.path.join(args.save, 'weights.pt'))
logging.info("total train: %f", t_record["train"])
logging.info("total grow: %f", t_record["grow"])
logging.info("total grow search: %f", t_record["grow_search"])
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(initGpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % initGpu)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels, CLASSES, args.layers, criterion)
torch.cuda.set_device(initGpu)
model = nn.DataParallel(model, device_ids)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
data_dir = args.tmp_data_dir
traindir = os.path.join(data_dir, 'train')
validdir = os.path.join(data_dir, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
train_data = dset.ImageFolder(
traindir,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ColorJitter(
brightness=0.4,
contrast=0.4,
saturation=0.4,
hue=0.2),
transforms.ToTensor(),
normalize,
]))
valid_data = dset.ImageFolder(
validdir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
]))
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(0.5 * num_train))
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True,
num_workers=args.workers)
valid_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]),
pin_memory=True,
num_workers=args.workers)
architect = Architect(model, args)
logs,losses = find_lr(model, train_queue, criterion, optimizer)
plt.plot(logs[10:-5],losses[10:-5])
plt.savefig('fing.jpg')
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
model = Network(args.init_channels, CLASSES, args.layers, criterion)
model = torch.nn.DataParallel(model)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
optimizer_a = torch.optim.Adam(model.module.arch_parameters(),
lr=args.arch_learning_rate,
betas=(0.5, 0.999),
weight_decay=args.arch_weight_decay)
#data preparation, we random sample 10% and 2.5% from training set(each class) as train and val, respectively.
#Note that the data sampling can not use torch.utils.data.sampler.SubsetRandomSampler as imagenet is too large.
data_dir = os.path.join(args.tmp_data_dir, 'imagenet_search')
traindir = os.path.join(data_dir, 'train')
validdir = os.path.join(data_dir, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_data = dset.ImageFolder(
traindir,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
valid_data = dset.ImageFolder(
validdir,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=args.workers)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=args.workers)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), eta_min=args.learning_rate_min)
for epoch in range(args.epochs):
scheduler.step()
lr = scheduler.get_lr()[0]
logging.info('epoch %d lr %e', epoch, lr)
if epoch < 5 and args.batch_size > 256:
for param_group in optimizer.param_groups:
param_group['lr'] = lr * (epoch + 1) / 5.0
logging.info('Warming-up Epoch: %d, LR: %e', epoch,
lr * (epoch + 1) / 5.0)
genotype = model.module.genotype()
logging.info('genotype = %s', genotype)
arch_param = model.module.arch_parameters()
logging.info(F.softmax(arch_param[0], dim=-1))
logging.info(F.softmax(arch_param[1], dim=-1))
# training
train_acc, train_obj = train(train_queue, valid_queue, model,
optimizer, optimizer_a, criterion, lr,
epoch)
logging.info('train_acc %f', train_acc)
# validation
if args.epochs - epoch <= 1:
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))