def get_cifar_tuned_model(load_weights=True):
network = NetworkCIFAR(40, CIFAR_CLASSES, 20, True, 0.4, CIFAR_TUNED)
if load_weights:
device = torch.device('cpu')
state_dict = torch.load('weights/cifar_tuned.pt', map_location=device)
network.load_state_dict(state_dict)
return network
def main():
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
cudnn.enabled = True
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
model = Network(args.init_ch, 10, args.layers, True, genotype).cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
checkpoint = torch.load(args.checkpoint + '/top1.pt')
model.load_state_dict(checkpoint['model_state_dict'])
criterion = nn.CrossEntropyLoss().cuda()
CIFAR_MEAN = [0.49139968, 0.48215827, 0.44653124]
CIFAR_STD = [0.24703233, 0.24348505, 0.26158768]
valid_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(CIFAR_MEAN, CIFAR_STD),
])
valid_queue = torch.utils.data.DataLoader(dset.CIFAR10(
root=args.data, train=False, transform=valid_transform),
batch_size=args.batch_size,
shuffle=True,
num_workers=2,
pin_memory=True)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc: %f', valid_acc)
def main():
if not torch.cuda.is_available():
sys.exit(1)
## step 1 construct the selected network
genotype = eval("genotypes.%s" % args.selected_arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
## step 2 load pretrained model parameter
if args.cifar100:
model = torch.nn.DataParallel(model)
model = model.cuda()
model.load_state_dict(torch.load(args.model_path)['net'])
else:
utils.load(model, args.model_path)
model = torch.nn.DataParallel(model)
model = model.cuda()
model.module.drop_path_prob = 0
model.drop_path_prob = 0
print("param size = %fMB" % utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
## step 3 load test data
valid_queue = load_data_cifar(args)
## step 4. inference on test data
valid_acc, valid_obj = infer(valid_queue, model, criterion)
print('-----------------------------------------------')
print('Average Valid_acc: %f ' % valid_acc)
print('-----------------------------------------------')
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
checkpoint = torch.load(args.model_path)
model.load_state_dict(checkpoint['model_state_dict'])
print("param size = {:.1f}MB".format(
floor(utils.count_parameters_in_MB(model), 1)))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
_, test_transform = utils._data_transforms_cifar10(args)
test_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=test_transform)
test_queue = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
model.drop_path_prob = args.drop_path_prob
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
start_epoch = 0
if os.path.isfile((os.path.join(args.save, 'checkpoint.pt'))):
checkpoint = torch.load(os.path.join(args.save, 'checkpoint.pt'))
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
scheduler.load_state_dict(checkpoint['scheduler'])
accuracies = checkpoint['accuracies']
print(
'Load checkpoint from {:} with start-epoch = {:}, acc: {}'.format(
args.save, start_epoch, accuracies))
for epoch in range(start_epoch, args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(epoch, args, model, optimizer, scheduler, valid_acc,
os.path.join(args.save, 'checkpoint.pt'))
from darts.genotypes import DARTS
from darts.operations import *
import sys
sys.path.append('../cnn')
from model import NetworkCIFAR as Network
from chainer.datasets import get_cifar10
c_model = NetworkCIFAR(DARTS)
chainer.serializers.load_npz('model.npz', c_model)
chainer.global_config.train = False
t_model = Network(36, 10, 20, True, DARTS)
t_model.drop_path_prob = 0.2
t_model.load_state_dict(torch.load('cifar10_model.pt'))
t_model.eval()
train, val = get_cifar10()
img, _ = train[0]
x = img[None]
t_x = torch.autograd.Variable(torch.FloatTensor(x))
c_y = c_model(x)
t_y = t_model(t_x)
np.testing.assert_almost_equal(c_y[0].data,
t_y[0].detach().data.numpy(),
decimal=5)
#####################
# c_prev_s = c_model.stem(x)
# c_prev_prev_s = c_prev_s
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype)
model = model.cuda()
if( os.path.exists(args.loadModelPath)):
model.load_state_dict(torch.load(args.loadModelPath))
else:
logging.info('model path not exist')
return
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.set=='cifar100':
train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR100(root=args.data, train=False, download=True, transform=valid_transform)
else:
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
#train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
#valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
best_acc = 0.0
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
if valid_acc > best_acc:
best_acc = valid_acc
logging.info('valid_acc %f, best_acc %f', valid_acc, best_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('No GPU device available')
sys.exit(1)
np.random.seed(args.seed)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info("args = %s", args)
logging.info("unparsed args = %s", unparsed)
num_gpus = torch.cuda.device_count()
genotype = eval("genotypes.%s" % args.arch)
print('---------Genotype---------')
logging.info(genotype)
print('--------------------------')
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
model = torch.nn.DataParallel(model)
model = model.cuda()
start_epch = 0
if args.resume:
MT = torch.load(os.path.join(args.save, 'weight_optimizers.pt'))
model.load_state_dict(MT['net'])
optimizer.load_state_dict(MT['optimizer'])
start_epch = MT['epoch']
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
if args.cifar100:
train_transform, valid_transform = utils._data_transforms_cifar100(
args)
else:
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.cifar100:
train_data = dset.CIFAR100(root=args.data_dir,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR100(root=args.data_dir,
train=False,
download=True,
transform=valid_transform)
else:
train_data = dset.CIFAR10(root=args.data_dir,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data_dir,
train=False,
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=args.workers)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=args.workers)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
best_acc = 0.0
for epoch in range(start_epch, args.epochs):
model.module.drop_path_prob = args.drop_path_prob * epoch / args.epochs
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
start_time = time.time()
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('Train_acc: %f', train_acc)
scheduler.step()
logging.info('Epoch: %d lr %e', epoch, scheduler.get_lr()[0])
valid_acc, valid_obj = infer(valid_queue, model, criterion)
if valid_acc > best_acc:
best_acc = valid_acc
state = {
'net': model.state_dict(),
'optimizer': optimizer.state_dict(),
'epoch': epoch
}
torch.save(state,
os.path.join(args.save, 'best_weight_optimizers.pt'))
logging.info('Valid_acc: %f, best_acc: %f', valid_acc, best_acc)
end_time = time.time()
duration = end_time - start_time
print('Epoch time: %d h.' % (duration * (args.epochs - epoch) / 3600))
if epoch % 50 == 0:
state = {
'net': model.state_dict(),
'optimizer': optimizer.state_dict(),
'epoch': epoch
}
torch.save(state, os.path.join(args.save, 'weight_optimizers.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype)
model = model.to('cuda')
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.to('cuda')
# not apply weight decay to BN layers
if args.bn_no_decay:
logging.info('BN layers are excluded from weight decay')
bn_params, other_params = utils.split_bn_params(model)
logging.debug('bn: %s', [p.dtype for p in bn_params])
logging.debug('other: %s', [p.dtype for p in other_params])
param_group = [{'params': bn_params, 'weight_decay': 0},
{'params': other_params}]
else:
param_group = model.parameters()
optimizer = torch.optim.SGD(
param_group,
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
logging.info('optimizer: %s', optimizer)
train_transform, valid_transform = utils.data_transforms_cifar10(args)
train_data = datasets.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = datasets.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=args.num_workers)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=args.num_workers)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
init_epoch = 0
best_acc = 0
if args.recover:
states = torch.load(args.recover)
model.load_state_dict(states['stete'])
init_epoch = states['epoch'] + 1
best_acc = states['best_acc']
logging.info('checkpoint loaded')
scheduler.step(init_epoch)
logging.info('scheduler is set to epoch %d. learning rate is %s', init_epoch, scheduler.get_lr())
for epoch in range(init_epoch, args.epochs):
logging.info('epoch %d lr %s', epoch, scheduler.get_lr())
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %.4f', train_acc)
with torch.no_grad():
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
logging.info('epoch %03d overall train_acc=%.4f valid_acc=%.4f', epoch, train_acc, valid_acc)
scheduler.step()
# gpu info
utils.gpu_usage(args.debug)
if valid_acc > best_acc:
best_acc = valid_acc
logging.info('best acc: %.4f', best_acc)
utils.save_checkpoint(state={'stete': model.state_dict(),
'epoch': epoch,
'best_acc': best_acc,},
is_best=False, save=args.save)
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
if args.parallel:
model = nn.DataParallel(model).cuda()
else:
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
start_epoch = 0
if args.resume:
checkpoint = torch.load(
os.path.join(args.resume_dir, 'checkpoint.pth.tar'))
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['model_state_dict'])
model = model.cuda()
scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
print('model was loaded. Start training from epoch {}.'.format(
start_epoch))
for epoch in range(start_epoch, args.epochs):
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
with open(os.path.join(args.save, 'learning_rate.txt'), mode='a') as f:
f.write(str(scheduler.get_lr()[0]) + '\n')
if args.parallel:
model.module.drop_path_prob = args.drop_path_prob * epoch / args.epochs
else:
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
with open(os.path.join(args.save, "train_acc.txt"), mode='a') as f:
f.write(str(train_acc) + '\n')
with open(os.path.join(args.save, "train_loss.txt"), mode='a') as f:
f.write(str(train_obj) + '\n')
scheduler.step()
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
with open(os.path.join(args.save, "test_acc.txt"), mode='a') as f:
f.write(str(valid_acc) + '\n')
with open(os.path.join(args.save, "test_loss.txt"), mode='a') as f:
f.write(str(valid_obj) + '\n')
utils.save(model, os.path.join(args.save, 'weights.pt'))
torch.save(scheduler.state_dict(),
os.path.join(args.save, 'scheduler.pth.tar'),
pickle_module=dill)
utils.save_checkpoint(
{
'epoch': epoch + 1,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'scheduler_state_dict': scheduler.state_dict()
},
False,
save=args.save)
def main():
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
cudnn.enabled = True
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
cur_epoch = 0
net = eval(args.arch)
print(net)
code = gen_code_from_list(net, node_num=int((len(net) / 4)))
genotype = translator([code, code], max_node=int((len(net) / 4)))
print(genotype)
model_ema = None
if not continue_train:
print('train from the scratch')
model = Network(args.init_ch, 10, args.layers, args.auxiliary,
genotype).cuda()
print("model init params values:", flatten_params(model))
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = CutMixCrossEntropyLoss(True).cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.wd)
if args.model_ema:
model_ema = ModelEma(
model,
decay=args.model_ema_decay,
device='cpu' if args.model_ema_force_cpu else '')
else:
print('continue train from checkpoint')
model = Network(args.init_ch, 10, args.layers, args.auxiliary,
genotype).cuda()
criterion = CutMixCrossEntropyLoss(True).cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.wd)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
checkpoint = torch.load(args.save + '/model.pt')
model.load_state_dict(checkpoint['model_state_dict'])
cur_epoch = checkpoint['epoch']
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
if args.model_ema:
model_ema = ModelEma(
model,
decay=args.model_ema_decay,
device='cpu' if args.model_ema_force_cpu else '',
resume=args.save + '/model.pt')
train_transform, valid_transform = utils._auto_data_transforms_cifar10(
args)
ds_train = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
args.cv = -1
if args.cv >= 0:
sss = StratifiedShuffleSplit(n_splits=5, test_size=0.2, random_state=0)
sss = sss.split(list(range(len(ds_train))), ds_train.targets)
for _ in range(args.cv + 1):
train_idx, valid_idx = next(sss)
ds_valid = Subset(ds_train, valid_idx)
ds_train = Subset(ds_train, train_idx)
else:
ds_valid = Subset(ds_train, [])
train_queue = torch.utils.data.DataLoader(CutMix(ds_train,
10,
beta=1.0,
prob=0.5,
num_mix=2),
batch_size=args.batch_size,
shuffle=True,
num_workers=2,
pin_memory=True)
valid_queue = torch.utils.data.DataLoader(dset.CIFAR10(
root=args.data, train=False, transform=valid_transform),
batch_size=args.batch_size,
shuffle=True,
num_workers=2,
pin_memory=True)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
best_acc = 0.0
if continue_train:
for i in range(cur_epoch + 1):
scheduler.step()
for epoch in range(cur_epoch, args.epochs):
print('cur_epoch is', epoch)
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
if model_ema is not None:
model_ema.ema.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer,
epoch, model_ema)
logging.info('train_acc: %f', train_acc)
if model_ema is not None and not args.model_ema_force_cpu:
valid_acc_ema, valid_obj_ema = infer(valid_queue,
model_ema.ema,
criterion,
ema=True)
logging.info('valid_acc_ema %f', valid_acc_ema)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc: %f', valid_acc)
if valid_acc > best_acc:
best_acc = valid_acc
print('this model is the best')
torch.save(
{
'epoch': epoch,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, os.path.join(args.save, 'top1.pt'))
print('current best acc is', best_acc)
logging.info('best_acc: %f', best_acc)
if model_ema is not None:
torch.save(
{
'epoch': epoch,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'state_dict_ema': get_state_dict(model_ema)
}, os.path.join(args.save, 'model.pt'))
else:
torch.save(
{
'epoch': epoch,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, os.path.join(args.save, 'model.pt'))
print('saved to: trained.pt')
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
#torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype, args.residual_wei, args.shrink_channel)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
resume = os.path.join(args.save, 'checkpoint.pth.tar')
if os.path.exists(resume):
print("=> loading checkpoint %s" % resume)
#checkpoint = torch.load(resume)
checkpoint = torch.load(resume, map_location = lambda storage, loc: storage.cuda(0))
args.start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
#optimizer.load_state_dict(checkpoint['optimizer'])
optimizer.state_dict()['state'] = checkpoint['optimizer']['state']
print('=> loaded checkpoint epoch %d' % args.start_epoch)
if args.start_epoch >= args.epochs:
print('training finished')
sys.exit(0)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
for epoch in range(args.start_epoch):
scheduler.step()
for epoch in range(args.start_epoch, args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
if epoch == args.epochs - 1:
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_acc_top1': train_acc,
'optimizer' : optimizer.state_dict(),
}, False, args.save)
def main(self, arch, epochs=600, gpu=0, load_weights=False, train_portion=0.7, seed=0, save='EXP'):
# Set up save file and logging
self.save = 'eval-{}-{}'.format(save, time.strftime("%Y%m%d-%H%M%S"))
utils.create_exp_dir(self.save, scripts_to_save=glob.glob('*.py'))
log_format = '%(asctime)s %(message)s'
logging.basicConfig(stream=sys.stdout, level=logging.INFO,
format=log_format, datefmt='%m/%d %I:%M:%S %p')
fh = logging.FileHandler(os.path.join(self.save, 'log.txt'))
fh.setFormatter(logging.Formatter(log_format))
logging.getLogger().addHandler(fh)
self.arch = arch
self.epochs = epochs
self.load_weights = load_weights
self.gpu = gpu
self.train_portion = train_portion
if self.train_portion == 1:
self.validation_set = False
self.seed = seed
print('Train class params')
print('arch: {}, epochs: {}, gpu: {}, load_weights: {}, train_portion: {}'
.format(arch, epochs, gpu, load_weights, train_portion))
# cpu-gpu switch
if not torch.cuda.is_available():
logging.info('no gpu device available')
torch.manual_seed(self.seed)
device = torch.device('cpu')
else:
torch.cuda.manual_seed_all(self.seed)
random.seed(self.seed)
torch.manual_seed(self.seed)
device = torch.device(self.gpu)
cudnn.benchmark = False
cudnn.enabled=True
cudnn.deterministic=True
logging.info('gpu device = %d' % self.gpu)
Genotype = namedtuple('Genotype', 'normal normal_concat reduce reduce_concat')
genotype = eval(self.convert_to_genotype(arch))
model = Network(self.init_channels, self.CIFAR_CLASSES, self.layers, self.auxiliary, genotype)
model = model.to(device)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
total_params = sum(x.data.nelement() for x in model.parameters())
logging.info('Model total parameters: {}'.format(total_params))
criterion = nn.CrossEntropyLoss()
criterion = criterion.to(device)
optimizer = torch.optim.SGD(
model.parameters(),
self.learning_rate,
momentum=self.momentum,
weight_decay=self.weight_decay
)
train_transform, test_transform = utils._data_transforms_cifar10(self.cutout, self.cutout_length)
train_data = dset.CIFAR10(root=self.data, train=True, download=True, transform=train_transform)
test_data = dset.CIFAR10(root=self.data, train=False, download=True, transform=test_transform)
num_train = len(train_data)
indices = list(range(num_train))
if self.validation_set:
split = int(np.floor(self.train_portion * num_train))
else:
split = num_train
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=self.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True, num_workers=0)
if self.validation_set:
valid_queue = torch.utils.data.DataLoader(
train_data, batch_size=self.batch_size // 2,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]),
pin_memory=True, num_workers=0)
test_queue = torch.utils.data.DataLoader(
test_data, batch_size=self.batch_size // 2, shuffle=False, pin_memory=True, num_workers=0)
if self.load_weights:
logging.info('loading saved weights')
ml = 'cuda:{}'.format(self.gpu) if torch.cuda.is_available() else 'cpu'
model.load_state_dict(torch.load('weights.pt', map_location = ml))
logging.info('loaded saved weights')
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(self.epochs))
valid_accs = []
test_accs = []
for epoch in range(self.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = self.drop_path_prob * epoch / self.epochs
train_acc, train_obj = self.train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
if self.validation_set:
valid_acc, valid_obj = self.infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
else:
valid_acc, valid_obj = 0, 0
test_acc, test_obj = self.infer(test_queue, model, criterion, test_data=True)
logging.info('test_acc %f', test_acc)
utils.save(model, os.path.join(self.save, 'weights-new.pt'))
if epoch in list(range(max(0, epochs - 5), epochs)):
valid_accs.append((epoch, valid_acc))
test_accs.append((epoch, test_acc))
return valid_accs, test_accs
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
if args.is_cifar100:
model = Network(args.init_channels, CIFAR100_CLASSES, args.layers,
args.auxiliary, genotype)
else:
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
if args.is_cifar100:
train_transform, valid_transform = utils._data_transforms_cifar100(
args)
else:
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.is_cifar100:
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR100(root=args.data,
train=False,
download=True,
transform=valid_transform)
else:
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
# train_queue = torch.utils.data.DataLoader(
# train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
num_train = len(train_data)
indices = list(range(num_train))
split = 45000
# int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True,
num_workers=2)
train_valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:]),
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
start_epoch = 0
if args.resume:
checkpoint = torch.load(args.resume)
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
scheduler.load_state_dict(checkpoint['scheduler'])
best_acc = 0
for epoch in range(start_epoch, args.epochs):
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
scheduler.step()
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(train_valid_queue, model, criterion)
if valid_acc > best_acc:
best_acc = valid_acc
test_acc, _ = infer(valid_queue, model, criterion)
logging.info('Test_acc: %f', test_acc)
logging.info('Valid_acc: %f', valid_acc)
logging.info('Best_acc: %f', best_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'scheduler': scheduler.state_dict(),
'optimizer': optimizer.state_dict()
})
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
# torch.cuda.set_device(args.gpu)
device = torch.device("cuda")
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
# read data
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.dataset == 'cifar10':
args.data = '/home/work/dataset/cifar'
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
classes = 10
if args.dataset == 'cifar100':
args.data = '/home/work/dataset/cifar100'
train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR100(root=args.data, train=False, download=True, transform=valid_transform)
classes = 100
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
# model
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, classes, args.layers, args.auxiliary, genotype)
model = model.cuda()
model.drop_path_prob = args.drop_path_prob
flops, params = profile(model, inputs=(torch.randn(1, 3, 32, 32).cuda(),), verbose=False)
logging.info('flops = %fM', flops / 1e6)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
best_val_acc = 0.
if args.resume:
# state = torch.load('/home/work/lixudong/code_work/sgas/cnn/full_train_s3_1-20200608/weights.pt')
# state = torch.load('/home/work/lixudong/code_work/sgas/cnn/full_train_s2_factor1-20200609/weights.pt', map_location='cpu')
# state = torch.load('/home/work/lixudong/code_work/sgas/cnn/full_train_s3_factor1-20200609/weights.pt', map_location='cpu')
# state = torch.load('/home/work/lixudong/code_work/sgas/cnn/full_train_s3_0-20200608/weights.pt', map_location='cpu')
# state = torch.load('/home/work/lixudong/code_work/sgas/cnn/full_train_s2_0-20200608/weights.pt', map_location='cpu')
state = torch.load('/home/work/lixudong/code_work/sgas/cnn/full_train_s3_2-20200608/weights.pt', map_location='cpu')
model.load_state_dict(state)
model = model.to(device)
for i in range(args.start_epoch):
scheduler.step()
best_val_acc = 97.19#97.34#97.32#94.92#94.6#97.2
for epoch in range(args.start_epoch, args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
with torch.no_grad():
valid_acc, valid_obj = infer(valid_queue, model, criterion)
if valid_acc > best_val_acc:
best_val_acc = valid_acc
utils.save(model, os.path.join(args.save, 'best_weights.pt'))
# logging.info('valid_acc %f\tbest_val_acc %f', valid_acc, best_val_acc)
logging.info('val_acc: {:.6}, best_val_acc: \033[31m{:.6}\033[0m'.format(valid_acc, best_val_acc))
state = {
'epoch': epoch,
'model_state': model.state_dict(),
'optimizer': optimizer.state_dict(),
'best_val_acc': best_val_acc
}
torch.save(state, os.path.join(args.save, 'weights.pt.tar'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
# device = torch.device("cuda")
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
# read data
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.dataset == 'cifar10':
args.data = '/home/work/dataset/cifar'
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
classes = 10
if args.dataset == 'cifar100':
args.data = '/home/work/dataset/cifar100'
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR100(root=args.data,
train=False,
download=True,
transform=valid_transform)
classes = 100
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
# model
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, classes, args.layers, args.auxiliary,
genotype)
model = model.cuda()
model.drop_path_prob = args.drop_path_prob
flops, params = profile(model,
inputs=(torch.randn(1, 3, 32, 32).cuda(), ),
verbose=False)
logging.info('flops = %fM', flops / 1e6)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
state = torch.load(
'/home/work/lixudong/code_work/sgas/cnn/full_train_s2_0-20200608/best_weights.pt',
map_location='cpu')
model.load_state_dict(state)
model = model.cuda()
valid_acc, valid_obj = infer(valid_queue, model, criterion)
print('val acc: {}, val loss: {}'.format(valid_acc, valid_obj))
print('no gpu device available')
sys.exit(1)
genotype = eval('genotypes.%s' % args.arch)
print('Parsing Genotypes: {}'.format(genotype))
model = Network(36, 10, 20, 0.4, genotype, args.parse_method)
flops, params = profile(model, inputs=(torch.randn(1, 3, 32, 32),), verbose=False)
print('flops = %fM' % (flops / 1e6))
print('param size = %fM' %( params / 1e6))
model = model.cuda()
if args.model_path and os.path.isfile(args.model_path):
checkpoint = torch.load(args.model_path)
model.load_state_dict(checkpoint['state_dict'])
else:
print('The Pre-Trained Model Is InValid!')
sys.exit(-1)
top1 = dutils.AvgrageMeter()
top5 = dutils.AvgrageMeter()
model.eval()
with torch.no_grad():
for step, (input, target) in enumerate(valid_queue):
input = input.cuda(non_blocking=True)
target = target.cuda(non_blocking=True)
logits, _ = model(input)
prec1, prec5 = dutils.accuracy(logits, target, topk=(1, 5))
n = input.size(0)
top1.update(prec1.item(), n)
