def main():
np.random.seed(args.seed)
torch.manual_seed(args.seed)
cudnn.benchmark = True
cudnn.enabled = True
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
# equal to: genotype = genotypes.DARTS_v2
genotype = eval("genotypes.%s" % args.arch)
print('Load genotype:', genotype)
model = Network(args.init_ch, 10, args.layers, args.auxiliary, genotype).cuda()
utils.load(model, args.exp_path)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss().cuda()
_, test_transform = utils._data_transforms_cifar10(args)
test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform)
test_queue = torch.utils.data.DataLoader(
test_data, batch_size=args.batchsz, shuffle=False, pin_memory=True, num_workers=2)
model.drop_path_prob = args.drop_path_prob
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
def _init_data_queue(self):
train_transform, valid_transform = dutils._data_transforms_cifar10(
self.args)
train_data = dset.CIFAR10(root=self.args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=self.args.data,
train=False,
download=True,
transform=valid_transform)
self.num_classes = 10
self.train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=self.args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=4)
self.valid_queue = torch.utils.data.DataLoader(
valid_data,
batch_size=self.args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=4)
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
torch.cuda.set_device(args.gpu)
cudnn.enabled = True
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
utils.load(model, args.model_path)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
_, test_transform = utils._data_transforms_cifar10(args)
test_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=test_transform)
test_queue = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=False,
num_workers=2)
model.drop_path_prob = 0.0
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('Test_acc %f', test_acc)
def main():
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
cudnn.enabled = True
torch.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_ch, 10, args.layers, args.auxiliary,
genotype).cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.wd)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batchsz,
shuffle=True,
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batchsz,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc: %f', valid_acc)
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc: %f', train_acc)
utils.save(model, os.path.join(args.save, 'trained.pt'))
print('saved to: trained.pt')
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
args.seed = np.random.randint(0, 100)
logging.info("----------seed = {}-----------".format(args.seed))
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
logging.info("Arc = {}".format(genotype))
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs),eta_min=args.learning_rate_min)
Best_valid_acc = 0.0
for epoch in range(args.epochs):
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
scheduler.step()
if valid_acc >= Best_valid_acc:
Best_valid_acc = valid_acc
logging.info('Best_valid_acc %f', Best_valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
cudnn.benchmark = True
cudnn.enabled=True
logging.info("args = %s", args)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, genotype)
if args.parallel:
model = MyDataParallel(model).cuda()
else:
model = model.cuda()
bin_op = bin_utils.BinOp(model, args)
_, valid_transform = utils._data_transforms_cifar10(args)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
utils.load(model, args.path_to_weights)
if args.parallel:
model.module.drop_path_prob = args.drop_path_prob * (args.epochs-1) / args.epochs
else:
model.drop_path_prob = args.drop_path_prob * (args.epochs-1) / args.epochs
valid_acc, valid_obj = infer(valid_queue, model, criterion, bin_op)
logging.info('valid_acc %f', valid_acc)
def main():
# logging.info('no gpu device available')
# sys.exit(1)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype)
utils.load(model, args.model_path, strict=False)
model = model.to_device() # to(torch._default_device)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.to_device() # to(torch._default_device)
_, test_transform = utils._data_transforms_cifar10(args)
test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform)
test_queue = torch.utils.data.DataLoader(
test_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
model.drop_path_prob = args.drop_path_prob
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
def load_data_cifar(args):
## step 3 load test data
if args.cifar100:
train_transform, valid_transform = utils._data_transforms_cifar100(
args)
else:
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.cifar100:
# train_data = dset.CIFAR100(root=args.tmp_data_dir, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR100(root=args.data_dir,
train=False,
download=True,
transform=valid_transform)
else:
# train_data = dset.CIFAR10(root=args.tmp_data_dir, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data_dir,
train=False,
download=True,
transform=valid_transform)
# train_queue = torch.utils.data.DataLoader(
# train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=args.workers)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=args.workers)
return valid_queue
def load_dataset(args=DEFAULT_ARGS):
if args.dataset == 'cifar10':
train_transform, valid_transform = utils._data_transforms_cifar10(args.cutout_length if args.cutout else None)
train_data = CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
test_data = CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]),
pin_memory=True, num_workers=2)
test_queue = torch.utils.data.DataLoader(
test_data, batch_size=args.evaluate_batch_size,
shuffle=False, pin_memory=True, num_workers=2)
else:
raise NotImplementedError("Temporary not used.")
return train_queue, valid_queue, test_queue
def _create_dataset():
train_transform, valid_transform = utils._data_transforms_cifar10(
args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
self.loader_train = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[:num_train]),
pin_memory=True,
num_workers=args.num_workers)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
self.loader_test = torch.utils.data.DataLoader(
valid_data,
batch_size=args.batch_size,
pin_memory=True,
num_workers=args.num_workers)
def build_cifar100(model_state_dict=None,
optimizer_state_dict=None,
args=None,
**kwargs):
epoch = kwargs.pop('epoch')
ratio = kwargs.pop('ratio')
train_transform, valid_transform = utils._data_transforms_cifar10(
args.cutout_size)
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=valid_transform)
num_train = len(train_data)
assert num_train == len(valid_data)
indices = list(range(num_train))
split = int(np.floor(ratio * num_train))
np.random.shuffle(indices)
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.child_batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True,
num_workers=16)
valid_queue = torch.utils.data.DataLoader(
valid_data,
batch_size=args.child_eval_batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True,
num_workers=16)
model = NASWSNetworkCIFAR(100, args.child_layers, args.child_nodes,
args.child_channels, args.child_keep_prob,
args.child_drop_path_keep_prob,
args.child_use_aux_head, args.steps)
model = model.cuda()
train_criterion = nn.CrossEntropyLoss().cuda()
eval_criterion = nn.CrossEntropyLoss().cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(
model.parameters(),
args.child_lr_max,
momentum=0.9,
weight_decay=args.child_l2_reg,
)
if model_state_dict is not None:
model.load_state_dict(model_state_dict)
if optimizer_state_dict is not None:
optimizer.load_state_dict(optimizer_state_dict)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, args.child_epochs, args.child_lr_min, epoch)
return train_queue, valid_queue, model, train_criterion, eval_criterion, optimizer, scheduler
def __init__(self, arm):
args = {}
args['data'] = '/home/liamli4465/darts/data/'
args['batch_size'] = 96
args['learning_rate'] = 0.025
args['momentum'] = 0.9
args['weight_decay'] = 3e-4
args['report_freq'] = 50
args['gpu'] = 0
args['init_channels'] = 36
args['layers'] = 20
args['auxiliary'] = True
args['auxiliary_weight'] = 0.4
args['cutout'] = True
args['cutout_length'] = 16
args['drop_path_prob'] = 0.2
args['grad_clip'] = 5
args['epochs'] = 300
args['save'] = arm['dir']
args['seed'] = arm['seed']
args['arch'] = arm['genotype']
args['cuda'] = True
args = AttrDict(args)
self.args = args
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = False
cudnn.enabled = True
cudnn.deterministic = True
torch.cuda.manual_seed_all(args.seed)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=False,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=False,
transform=valid_transform)
self.train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=0)
self.valid_queue = torch.utils.data.DataLoader(
valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=0)
self.epoch = 0
def init_loaddata(self):
train_transform, valid_transform = utils._data_transforms_cifar10(
self.args)
train_data = dset.CIFAR10(root=self.args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=self.args.data,
train=False,
download=True,
transform=valid_transform)
if self.args.seed:
def worker_init_fn():
seed = self.seed
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
return
else:
worker_init_fn = None
if self.args.distributed:
train_sampler = DistributedSampler(train_data)
valid_sampler = DistributedSampler(valid_data)
self.train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=self.args.batch_size // self.world_size,
shuffle=False,
num_workers=0,
pin_memory=False,
sampler=train_sampler)
self.valid_queue = torch.utils.data.DataLoader(
valid_data,
batch_size=self.args.batch_size // self.world_size,
shuffle=False,
num_workers=0,
pin_memory=False,
sampler=valid_sampler)
else:
self.train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=self.args.batch_size,
shuffle=True,
pin_memory=False,
num_workers=2)
self.valid_queue = torch.utils.data.DataLoader(
valid_data,
batch_size=self.args.batch_size,
shuffle=False,
pin_memory=False,
num_workers=2)
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
args.seed += 1
logging.info("args = %s", args)
arch = utils.decode_arch(args.arch)
model = Network(arch, CIFAR_CLASSES)
model = model.cuda()
params = utils.count_parameters_in_MB(model)
logging.info("param size = %fMB", params / 1e6)
flops = utils.count_FLOPs(model)
logging.info("FLOPs = %d", flops)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
train_transform, valid_transform = utils._data_transforms_cifar10()
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, pin_memory=True, num_workers=8, shuffle=True)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, pin_memory=True, num_workers=8, shuffle=False)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
best_acc = 0
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
if valid_acc > best_acc:
best_acc = valid_acc
logging.info('valid_acc %f', valid_acc)
return params, flops, best_acc
def evaluate(individual, num_classes, num_epochs, batch_size, learning_rate):
train_transform, test_transform = utils._data_transforms_cifar10()
train_dataset = torchvision.datasets.CIFAR10(root='../../data',
train=True,
transform=train_transform)
test_dataset = torchvision.datasets.CIFAR10(root='../../data',
train=False,
transform=test_transform)
train_loader = torch.utils.data.DataLoader(
dataset=train_dataset,
batch_size=batch_size,
shuffle=True,
# pin_memory=True,
# num_workers=2
)
test_loader = torch.utils.data.DataLoader(
dataset=test_dataset,
batch_size=batch_size,
shuffle=False,
# pin_memory=True,
# num_workers=2
)
structure = Network(individual.structure, [(3, 32), (32, 128), (128, 128)],
num_classes, (32, 32)).to(device)
individual.size = utils.count_parameters_in_MB(structure)
parameters = filter(lambda p: p.requires_grad, structure.parameters())
cudnn.enabled = True
cudnn.benchmark = True
criterion = torch.nn.CrossEntropyLoss().to(device)
optimizer = torch.optim.SGD(parameters,
lr=learning_rate,
momentum=0.9,
weight_decay=3e-4)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,
num_epochs,
eta_min=0.0)
best_acc = 0
for epoch in range(num_epochs):
print('epoch[{}/{}]:'.format(epoch + 1, num_epochs))
train(train_loader, structure, criterion, optimizer)
scheduler.step()
valid_acc = test(test_loader, structure, criterion)
print()
if valid_acc > best_acc:
best_acc = valid_acc
individual.accuracy = best_acc
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
in_channels = 3
num_classes = 10
stride_for_aux = 3
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, in_channels, stride_for_aux,
num_classes, args.layers, args.auxiliary, genotype)
model = model.cuda()
utils.load(model, args.model_path)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
_, test_transform = utils._data_transforms_cifar10(args)
#test_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=test_transform)
#version = 'v6'
#images, labels = utils_test.load_new_test_data(version)
#num_images = images.shape[0]
data_10 = np.load(
"/home/sivan/darts/cnn/data_cifar_10_1/cifar10.1_v4_data.npy")
labels_10 = np.load(
'/home/sivan/darts/cnn/data_cifar_10_1/cifar10.1_v4_labels.npy')
#
tensor_x = torch.stack([torch.Tensor(i)
for i in data_10]) # transform to torch tensors
tensor_y = torch.stack([torch.Tensor(i) for i in labels_10])
test_data = utils_d.TensorDataset(tensor_x, tensor_y) # create your datset
test_queue = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
model.drop_path_prob = args.drop_path_prob
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
# 得到train_search里å¦å¥½çš„normal cell å’Œreduction cell，genotypes.DARTS就是选的å¦å¥½çš„DARTS_V2
genotype = eval("genotypes.%s" % args.arch)#DARTS_V2 = Genotype(normal=[('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 1), ('skip_connect', 0), ('skip_connect', 0), ('dil_conv_3x3', 2)], normal_concat=[2, 3, 4, 5], reduce=[('max_pool_3x3', 0), ('max_pool_3x3', 1), ('skip_connect', 2), ('max_pool_3x3', 1), ('max_pool_3x3', 0), ('skip_connect', 2), ('skip_connect', 2), ('max_pool_3x3', 1)], reduce_concat=[2, 3, 4, 5])
# 这里的Network用的是model.py的NetworkCIFAR
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def get_data_transforms(dataset_name, args):
train_transform, valid_transform = None, None
if dataset_name == 'cifar10' or dataset_name == 'cifar100':
train_transform, valid_transform = utils._data_transforms_cifar10(args)
elif dataset_name == 'mnist':
train_transform, valid_transform = utils._data_transforms_mnist(args)
elif dataset_name == 'mnistm':
train_transform, valid_transform = utils._data_transforms_mnistm(args)
else:
assert False and f'Unrecognized dataset: {dataset_name}'
return train_transform, valid_transform
def build_cifar100(model_state_dict, optimizer_state_dict, **kwargs):
epoch = kwargs.pop('epoch')
train_transform, valid_transform = utils._data_transforms_cifar10(
args.cutout_size)
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR100(root=args.data,
train=False,
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=16)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.eval_batch_size,
shuffle=False,
pin_memory=True,
num_workers=16)
model = NASNetworkCIFAR(args, 100, args.layers, args.nodes, args.channels,
args.keep_prob, args.drop_path_keep_prob,
args.use_aux_head, args.steps, args.arch)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
logging.info("multi adds = %fM", model.multi_adds / 1000000)
if model_state_dict is not None:
model.load_state_dict(model_state_dict)
if torch.cuda.device_count() > 1:
logging.info("Use %d %s", torch.cuda.device_count(), "GPUs !")
model = nn.DataParallel(model)
model = model.cuda()
train_criterion = nn.CrossEntropyLoss().cuda()
eval_criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.lr_max,
momentum=0.9,
weight_decay=args.l2_reg,
)
if optimizer_state_dict is not None:
optimizer.load_state_dict(optimizer_state_dict)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs), args.lr_min, epoch)
return train_queue, valid_queue, model, train_criterion, eval_criterion, optimizer, scheduler
def main():
args = get_args()
if torch.cuda.is_available():
print('Train on GPU!')
device = torch.device("cuda")
else:
device = torch.device("cpu")
train_transform, valid_transform = _data_transforms_cifar10(args)
trainset = torchvision.datasets.CIFAR10(root=args.data_dir,
train=True,
download=True,
transform=train_transform)
train_loader = torch.utils.data.DataLoader(trainset,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=4)
valset = torchvision.datasets.CIFAR10(root=args.data_dir,
train=False,
download=True,
transform=valid_transform)
val_loader = torch.utils.data.DataLoader(valset,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=4)
model = Network(classes=10, gap_size=1).to(device)
criterion = nn.CrossEntropyLoss().to(device)
optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad,
model.parameters()),
lr=args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.LambdaLR(
optimizer, lambda epoch: 1 - (epoch / args.epochs))
summary(model, (3, 32, 32))
print('Start training!')
for epoch in range(args.epochs):
print('epoch:%d, lr:%f' % (epoch, scheduler.get_lr()[0]))
train(args, epoch, train_loader, device, model, criterion, optimizer,
scheduler)
scheduler.step()
if (epoch + 1) % args.val_interval == 0:
validate(args, epoch, val_loader, device, model)
utils.save_checkpoint({
'state_dict': model.state_dict(),
},
epoch + 1,
tag=args.exp_name)
def __init__(
self,
name,
args,
):
self.name = name
self.args = args
self.criterion = nn.CrossEntropyLoss()
self.criterion = self.criterion.cuda()
CIFAR_CLASSES = 10
self.model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
self.criterion)
self.model = self.model.cuda()
logging.info("param size = %fMB",
utils.count_parameters_in_MB(self.model))
self.optimizer = torch.optim.SGD(self.model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
num_train = len(train_data)
self.num_train = num_train
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
self.train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.train_batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[:split]),
pin_memory=True,
num_workers=2)
self.valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.valid_batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True,
num_workers=2)
self.scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
self.optimizer, float(args.epochs), eta_min=args.learning_rate_min)
def main():
if not torch.cuda.is_available():
logging.info('No GPU device available')
sys.exit(1)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info("args = %s", args)
num_gpus = torch.cuda.device_count()
logging.info('Testing with %d GPU(s)', num_gpus)
model = eval("se_resnet%s(num_classes=CIFAR_CLASSES)" % args.resnet_type)
if num_gpus > 1:
model = torch.nn.DataParallel(model)
model = model.cuda()
try:
utils.load(model, args.model_path)
except:
model = model.module
utils.load(model, args.model_path)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
if args.cifar100:
_, test_transform = utils._data_transforms_cifar100(args)
else:
_, test_transform = utils._data_transforms_cifar10(args)
if args.cifar100:
test_data = dset.CIFAR100(root=args.tmp_data_dir,
train=False,
download=True,
transform=test_transform)
else:
test_data = dset.CIFAR10(root=args.tmp_data_dir,
train=False,
download=True,
transform=test_transform)
test_queue = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('TEST ACCURACY: --- %f% ---', test_acc)
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
graph = [AGraph(g) for g in glob.glob('comp_graphs/new/*.dot')]
_, comp_graph = cell_graph.generate_comp_graph(graph)
conf = arch_config(comp_graph=comp_graph,
channels=args.init_channels,
repeat_list=args.layers,
classes=CIFAR_CLASSES)
model = get_net(conf)
model = model.cuda()
state_dict = torch.load(args.model_path)['state_dict']
if isinstance(state_dict, torch.nn.DataParallel):
state_dict = state_dict.module
model.load_state_dict(state_dict)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
_, test_transform = utils._data_transforms_cifar10(args)
test_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=test_transform)
test_queue = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
model.drop_path_prob = args.drop_path_prob
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
def main():
np.random.seed(args.seed)
gpus = [int(i) for i in args.gpu.split(',')]
if len(gpus) == 1:
torch.cuda.set_device(int(args.gpu))
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %s' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model.cuda()
if len(gpus) > 1:
print("True")
model = nn.parallel.DataParallel(model,
device_ids=gpus,
output_device=gpus[0])
model = model.module
utils.load(model, args.model_path)
print("If the model is running on GPU:", next(model.parameters()).is_cuda)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
_, test_transform = utils._data_transforms_cifar10(args)
test_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=test_transform)
test_queue = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
model.drop_path_prob = args.drop_path_prob
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
checkpoint = torch.load(args.model_path)
model.load_state_dict(checkpoint['model_state_dict'])
print("param size = {:.1f}MB".format(
floor(utils.count_parameters_in_MB(model), 1)))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
_, test_transform = utils._data_transforms_cifar10(args)
test_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=test_transform)
test_queue = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
model.drop_path_prob = args.drop_path_prob
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
def init_loaddata(self):
train_transform, valid_transform = utils._data_transforms_cifar10(
self.args)
train_data = dset.CIFAR10(root=self.args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=self.args.data,
train=False,
download=True,
transform=valid_transform)
if self.args.seed:
def worker_init_fn():
seed = self.seed
np.random.seed(seed)
random.seed(seed)
torch.manual_seed(seed)
return
else:
worker_init_fn = None
num_train = len(train_data)
indices = list(range(num_train))
self.train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=self.args.batch_size,
shuffle=True,
pin_memory=False,
num_workers=2)
self.valid_queue = torch.utils.data.DataLoader(
valid_data,
batch_size=self.args.batch_size,
shuffle=False,
pin_memory=False,
num_workers=2)
def build_validation_data_loader(self) -> DataLoader:
train_transform, valid_transform = utils._data_transforms_cifar10(
AttrDict(self.hparams))
valid_data = dset.CIFAR10(
root="{}/data-rank{}".format(
self.data_config["data_download_dir"],
self.context.distributed.get_rank(),
),
train=False,
download=True,
transform=valid_transform,
)
valid_queue = DataLoader(
valid_data,
batch_size=self.context.get_per_slot_batch_size(),
shuffle=False,
pin_memory=True,
num_workers=2,
)
return valid_queue
def _create_dataset():
train_transform, valid_transform = utils._data_transforms_cifar10(
args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
num_train = len(train_data)
indices = list(range(num_train))
ratio = 1.0 if args.mode == 0 else args.train_portion
split = int(np.floor(ratio * num_train))
self.loader_train = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[:split]),
pin_memory=True,
num_workers=args.num_workers)
if args.mode != 0:
self.loader_eval = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True,
num_workers=args.num_workers)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
self.loader_test = torch.utils.data.DataLoader(
valid_data,
batch_size=args.batch_size,
pin_memory=True,
num_workers=args.num_workers)
def _create_dataset():
train_transform, valid_transform = utils._data_transforms_cifar10(
args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
self.loader_train = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
pin_memory=True,
num_workers=args.num_workers)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
self.loader_test = torch.utils.data.DataLoader(
valid_data,
batch_size=args.batch_size,
pin_memory=True,
num_workers=args.num_workers)
def main():
np.random.seed(args.seed)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info(
"device = %s" %
'cuda:{}'.format(args.gpu) if torch.cuda.is_available() else 'cpu')
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype).to(device)
utils.load(model, args.model_path)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss().to(device)
_, test_transform = utils._data_transforms_cifar10(args)
test_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=test_transform)
test_queue = torch.utils.data.DataLoader(
test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2,
)
model.drop_path_prob = args.drop_path_prob
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info("test_acc %f", test_acc)
