def main():
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
cudnn.enabled = True
torch.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_ch, 10, args.layers, args.auxiliary,
genotype).cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.wd)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batchsz,
shuffle=True,
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batchsz,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc: %f', valid_acc)
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc: %f', train_acc)
utils.save(model, os.path.join(args.save, 'trained.pt'))
print('saved to: trained.pt')
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
in_channels, num_classes, dataset_in_torch, stride_for_aux = utils.dataset_fields(
args, train=False) # new
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, in_channels, stride_for_aux,
num_classes, args.layers, args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_data, valid_data = utils.dataset_split_and_transform(
dataset_in_torch, args, train=False) # new
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
start_time = time.time()
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
end_time = time.time()
duration = end_time - start_time
print('Epoch time: %ds.' %duration)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def _init_model(self):
genotype = eval('genotypes.%s' % self.args.arch)
model = Network(self.args.init_channels, self.num_classes,
self.args.layers, self.args.auxiliary, genotype,
self.args.parse_method)
flops, params = profile(model,
inputs=(torch.randn(1, 3, 32, 32), ),
verbose=False)
self.logger.info('flops = %fM', flops / 1e6)
self.logger.info('param size = %fM', params / 1e6)
# Try move model to multi gpus
if torch.cuda.device_count() > 1 and self.args.multi_gpus:
self.logger.info('use: %d gpus', torch.cuda.device_count())
model = nn.DataParallel(model)
else:
self.logger.info('gpu device = %d' % self.device_id)
torch.cuda.set_device(self.device_id)
self.model = model.to(self.device)
criterion = nn.CrossEntropyLoss()
self.criterion = criterion.to(self.device)
self.optimizer = torch.optim.SGD(model.parameters(),
self.args.learning_rate,
momentum=self.args.momentum,
weight_decay=self.args.weight_decay)
self.best_acc_top1 = 0
# optionally resume from a checkpoint
if self.args.resume:
if os.path.isfile(self.args.resume):
print("=> loading checkpoint {}".format(self.args.resume))
checkpoint = torch.load(self.args.resume,
map_location=self.device)
self.dur_time = checkpoint['dur_time']
self.args.start_epoch = checkpoint['epoch']
self.best_acc_top1 = checkpoint['best_acc_top1']
self.args.drop_path_prob = checkpoint['drop_path_prob']
self.model.load_state_dict(checkpoint['state_dict'])
self.optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
self.args.resume, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(
self.args.resume))
self.scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
self.optimizer,
float(self.args.epochs),
eta_min=0,
last_epoch=-1
if self.args.start_epoch == 0 else self.args.start_epoch)
# reload the scheduler if possible
if self.args.resume and os.path.isfile(self.args.resume):
checkpoint = torch.load(self.args.resume)
self.scheduler.load_state_dict(checkpoint['scheduler'])
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype,output_height=args.img_cropped_height,output_width=args.img_cropped_width)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.MSELoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
train_transform, valid_transform = utils._data_trainsforms_denosining_dataset(args)
train_data = DENOISE_DATASET(root=args.data,train_folder=args.train_data,label_folder=args.label_data,train=True, transform=train_transform,target_transform=train_transform )
#valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]), pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:num_train]), pin_memory=True, num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_obj %f', train_obj)
valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_obj %f', valid_obj)
utils.save(model, os.path.join(args.save, './weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
# 得到train_search里学好的normal cell 和reduction cell，genotypes.DARTS就是选的学好的DARTS_V2
genotype = eval("genotypes.%s" % args.arch)#DARTS_V2 = Genotype(normal=[('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 0), ('sep_conv_3x3', 1), ('sep_conv_3x3', 1), ('skip_connect', 0), ('skip_connect', 0), ('dil_conv_3x3', 2)], normal_concat=[2, 3, 4, 5], reduce=[('max_pool_3x3', 0), ('max_pool_3x3', 1), ('skip_connect', 2), ('max_pool_3x3', 1), ('max_pool_3x3', 0), ('skip_connect', 2), ('skip_connect', 2), ('max_pool_3x3', 1)], reduce_concat=[2, 3, 4, 5])
# 这里的Network用的是model.py的NetworkCIFAR
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
np.random.seed(args.seed)
gpus = [int(i) for i in args.gpu.split(',')]
if len(gpus) == 1:
torch.cuda.set_device(int(args.gpu))
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %s' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model.cuda()
if len(gpus) > 1:
print("True")
model = nn.parallel.DataParallel(model,
device_ids=gpus,
output_device=gpus[0])
model = model.module
utils.load(model, args.model_path)
print("If the model is running on GPU:", next(model.parameters()).is_cuda)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
_, test_transform = utils._data_transforms_cifar10(args)
test_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=test_transform)
test_queue = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
model.drop_path_prob = args.drop_path_prob
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
def build_model(self) -> nn.Module:
genotype = self.get_genotype_from_hps()
model = Network(
self.hparams["init_channels"],
10, # num_classes
self.hparams["layers"],
self.hparams["auxiliary"],
genotype,
)
print("param size = {} MB".format(utils.count_parameters_in_MB(model)))
size = 0
for p in model.parameters():
size += p.nelement()
print("param count: {}".format(size))
# If loading backbone weights, do not call reset_parameters() or
# call before loading the backbone weights.
reset_parameters(model)
return model
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
if args.parallel:
model = nn.DataParallel(model).cuda()
else:
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
start_epoch = 0
if args.resume:
checkpoint = torch.load(
os.path.join(args.resume_dir, 'checkpoint.pth.tar'))
start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['model_state_dict'])
model = model.cuda()
scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
print('model was loaded. Start training from epoch {}.'.format(
start_epoch))
for epoch in range(start_epoch, args.epochs):
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
with open(os.path.join(args.save, 'learning_rate.txt'), mode='a') as f:
f.write(str(scheduler.get_lr()[0]) + '\n')
if args.parallel:
model.module.drop_path_prob = args.drop_path_prob * epoch / args.epochs
else:
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
with open(os.path.join(args.save, "train_acc.txt"), mode='a') as f:
f.write(str(train_acc) + '\n')
with open(os.path.join(args.save, "train_loss.txt"), mode='a') as f:
f.write(str(train_obj) + '\n')
scheduler.step()
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
with open(os.path.join(args.save, "test_acc.txt"), mode='a') as f:
f.write(str(valid_acc) + '\n')
with open(os.path.join(args.save, "test_loss.txt"), mode='a') as f:
f.write(str(valid_obj) + '\n')
utils.save(model, os.path.join(args.save, 'weights.pt'))
torch.save(scheduler.state_dict(),
os.path.join(args.save, 'scheduler.pth.tar'),
pickle_module=dill)
utils.save_checkpoint(
{
'epoch': epoch + 1,
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
'scheduler_state_dict': scheduler.state_dict()
},
False,
save=args.save)
def main():
writerTf = SummaryWriter(comment='writerTf')
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
gpus = [int(i) for i in args.gpu.split(',')]
if len(gpus) == 1:
torch.cuda.set_device(int(args.gpu))
# torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
# cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %s' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
if len(gpus) > 1:
print("True")
model = nn.parallel.DataParallel(model,
device_ids=gpus,
output_device=gpus[0])
model = model.module
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
with torch.no_grad():
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
writerTf.add_scalar('train_acc', train_acc, epoch)
writerTf.add_scalar('train_obj', train_obj, epoch)
writerTf.add_scalar('valid_acc', valid_acc, epoch)
writerTf.add_scalar('valid_obj', valid_obj, epoch)
writerTf.close()
def main():
# Setup
args = cmd_argument_parser()
create_logger(args.save)
# Set the gpu device to be used
# NOTE: Only operates on a single GPU
if torch.cuda.is_available():
torch.cuda.set_device(int(args.gpu))
else:
logging.info('no gpu device available')
sys.exit(1)
# Ensure seeds are set
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
# Hardware specific tuning
torch.backends.cudnn.enabled = True
torch.backends.cudnn.benchmark = True
# Get the specific architecture to train
genotype = genomes[args.arch]
# Create the fixed network
# Note: This differs from the Network used in model_search.py
# TODO: Update the Network class
model = Network(C=args.init_channels,
num_classes=CIFAR_CLASSES,
layers=args.layers,
auxiliary=args.auxiliary,
genotype=genotype)
model = model.cuda()
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
# The loss function
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
# Optimizer used to adjust the models parameters as well as an optimizer
# of the learning rate
optimizer = torch.optim.SGD(params=model.parameters(),
lr=args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
lr_scheduler = CosineAnnealingLR(optimizer=optimizer,
T_max=float(args.epochs))
# Get the transforms for both the train and validation data
train_transform, valid_transform = utils._data_transforms_cifar10(args)
# Get the data from torchvision's datasets
train_data = CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
# Create Dataloaders for both
train_queue = DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=0)
valid_queue = DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=0)
for epoch in range(args.epochs):
logging.info(f'epoch = {epoch}')
logging.info(f'lr = {lr_scheduler.get_last_lr()}')
# More likely to drop a path as epochs progress
model.drop_path_prob = args.drop_path_prob * (epoch / args.epochs)
train_acc, train_obj = train(train_queue, model, criterion, optimizer,
args)
with torch.no_grad():
valid_acc, valid_obj = infer(valid_queue, model, criterion, args)
logging.info(f'train_acc = {train_acc}')
logging.info(f'valid_acc = {valid_acc}')
# Save the model for each epoch
utils.save(model, os.path.join(args.save, 'weights.pt'))
lr_scheduler.step()
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
#print(genotype)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
#stat(model, (1, 16, 16))
model = model.cuda()
# input_size=(1,2,128)
'''
Input = torch.randn(1, 1, 2, 128)
Input = Input.type(torch.cuda.FloatTensor)
macs, params = profile(model, inputs=(Input,), custom_ops={Network: Network.forward})
macs, params = clever_format([macs, params], "%.3f")
print(macs, params)
summary(model,(1,16,16))
'''
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
'''
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.set=='cifar100':
train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR100(root=args.data, train=False, download=True, transform=valid_transform)
else:
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
#train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
#valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
'''
x = loadmat("/data/wx/PC-DARTS/data/datashujudoppler=100.mat")
x = x.get('train_data')
x = np.reshape(x, [-1, 1, 2, 128])
data_num = 22000
y1 = np.zeros([data_num, 1])
y2 = np.ones([data_num, 1])
y3 = np.ones([data_num, 1]) * 2
y4 = np.ones([data_num, 1]) * 3
y5 = np.ones([data_num, 1]) * 4
y6 = np.ones([data_num, 1]) * 5
y7 = np.ones([data_num, 1]) * 6
y8 = np.ones([data_num, 1]) * 7
y = np.vstack((y1, y2, y3, y4, y5, y6, y7, y8))
y = np.array(y)
X_train, X_val, Y_train, Y_val = train_test_split(x,
y,
test_size=0.3,
random_state=30)
X_train = torch.from_numpy(X_train)
Y_train = torch.from_numpy(Y_train)
X_train = X_train.type(torch.FloatTensor)
Y_train = Y_train.type(torch.LongTensor)
Y_train = Y_train.type(torch.LongTensor)
# Y_train=np.reshape(Y_train,(16800,4))
Y_train = Y_train.squeeze()
print(Y_train.type)
print(Y_train)
print(X_train.shape, Y_train.shape)
train_Queue = torch.utils.data.TensorDataset(X_train, Y_train)
print(train_Queue)
X_val = torch.from_numpy(X_val)
Y_val = torch.from_numpy(Y_val)
X_val = X_val.type(torch.FloatTensor)
Y_val = Y_val.type(torch.LongTensor)
# Y_train = one_hot_embedding(Y_train, 4)
Y_val = Y_val.type(torch.LongTensor)
Y_val = Y_val.squeeze()
print(Y_val.type, Y_val)
valid_Queue = torch.utils.data.TensorDataset(X_val, Y_val)
train_queue = torch.utils.data.DataLoader(train_Queue,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(valid_Queue,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
best_acc = 0.0
writer = SummaryWriter('./logs/balei')
'''
Input = torch.randn(1, 1, 2, 128)
Input=Input.type(torch.cuda.FloatTensor)
macs, params = profile(model, inputs=(Input,),custom_ops={Network:Network.forward})
macs, params = clever_format([macs, params], "%.3f")
print(macs, params)
'''
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
writer.add_scalar('Train/acc', train_acc, epoch)
TAcc.write(str(train_acc) + ",")
writer.add_scalar('Train/loss', train_obj, epoch)
Tloss.write(str(train_obj) + ",")
valid_acc, valid_obj = infer(valid_queue, model, criterion)
if valid_acc > best_acc:
best_acc = valid_acc
utils.save(model, os.path.join(args.save, 'weights_best_acc.pt'))
logging.info('valid_acc %f, best_acc %f', valid_acc, best_acc)
writer.add_scalar('Valid/acc', valid_acc, epoch)
VAcc.write(str(valid_acc) + ",")
writer.add_scalar('Valid/loss', valid_obj, epoch)
Vloss.write(str(valid_obj) + ",")
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
# torch.cuda.set_device(args.gpu)
device = torch.device("cuda")
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
# read data
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.dataset == 'cifar10':
args.data = '/home/work/dataset/cifar'
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
classes = 10
if args.dataset == 'cifar100':
args.data = '/home/work/dataset/cifar100'
train_data = dset.CIFAR100(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR100(root=args.data, train=False, download=True, transform=valid_transform)
classes = 100
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
# model
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, classes, args.layers, args.auxiliary, genotype)
model = model.cuda()
model.drop_path_prob = args.drop_path_prob
flops, params = profile(model, inputs=(torch.randn(1, 3, 32, 32).cuda(),), verbose=False)
logging.info('flops = %fM', flops / 1e6)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
best_val_acc = 0.
if args.resume:
# state = torch.load('/home/work/lixudong/code_work/sgas/cnn/full_train_s3_1-20200608/weights.pt')
# state = torch.load('/home/work/lixudong/code_work/sgas/cnn/full_train_s2_factor1-20200609/weights.pt', map_location='cpu')
# state = torch.load('/home/work/lixudong/code_work/sgas/cnn/full_train_s3_factor1-20200609/weights.pt', map_location='cpu')
# state = torch.load('/home/work/lixudong/code_work/sgas/cnn/full_train_s3_0-20200608/weights.pt', map_location='cpu')
# state = torch.load('/home/work/lixudong/code_work/sgas/cnn/full_train_s2_0-20200608/weights.pt', map_location='cpu')
state = torch.load('/home/work/lixudong/code_work/sgas/cnn/full_train_s3_2-20200608/weights.pt', map_location='cpu')
model.load_state_dict(state)
model = model.to(device)
for i in range(args.start_epoch):
scheduler.step()
best_val_acc = 97.19#97.34#97.32#94.92#94.6#97.2
for epoch in range(args.start_epoch, args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
with torch.no_grad():
valid_acc, valid_obj = infer(valid_queue, model, criterion)
if valid_acc > best_val_acc:
best_val_acc = valid_acc
utils.save(model, os.path.join(args.save, 'best_weights.pt'))
# logging.info('valid_acc %f\tbest_val_acc %f', valid_acc, best_val_acc)
logging.info('val_acc: {:.6}, best_val_acc: \033[31m{:.6}\033[0m'.format(valid_acc, best_val_acc))
state = {
'epoch': epoch,
'model_state': model.state_dict(),
'optimizer': optimizer.state_dict(),
'best_val_acc': best_val_acc
}
torch.save(state, os.path.join(args.save, 'weights.pt.tar'))
def main():
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpus
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %s' % args.gpus)
logging.info("args = %s", args)
num_gpus = torch.cuda.device_count()
genotype = eval("genotypes.%s" % args.arch)
print('---------Genotype---------')
logging.info(genotype)
print('--------------------------')
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
if num_gpus > 1:
model = nn.DataParallel(model)
model = model.cuda()
else:
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.set == 'cifar100':
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR100(root=args.data,
train=False,
download=True,
transform=valid_transform)
else:
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
#train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
#valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=args.workers)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=args.workers)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
best_acc = 0.0
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
if num_gpus > 1:
model.module.drop_path_prob = args.drop_path_prob * epoch / args.epochs
else:
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
if valid_acc > best_acc:
best_acc = valid_acc
logging.info('valid_acc %f, best_acc %f', valid_acc, best_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
cudnn.enabled = True
torch.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
if not args.arch:
geno_s = pathlib.Path(geno_path).read_text()
else:
geno_s = "genotypes.%s" % args.arch
genotype = eval(geno_s)
model = Network(args.init_ch, 10, args.layers, args.auxiliary, genotype).cuda()
logging.info(f"seed = {args.seed}")
logging.info(f"geno_s = {geno_s}")
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(
model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.wd
)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batchsz, shuffle=True, pin_memory=True, num_workers=0 if 'pydevd' in sys.modules else 2)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batchsz, shuffle=False, pin_memory=True, num_workers=0 if 'pydevd' in sys.modules else 2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
lines = [f'epoch\ttrain_acc\tval_acc']
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc: %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc: %f', valid_acc)
lines.append(f'{epoch}\t{train_acc}\t{valid_acc}')
timebudget.report()
utils.save(model, os.path.join(args.save, 'trained.pt'))
print('saved to: trained.pt')
pathlib.Path(os.path.join(args.exp_path, 'eval.tsv')).write_text('\n'.join(lines))
def main():
if not torch.cuda.is_available():
logging.info('No GPU device available')
sys.exit(1)
np.random.seed(args.seed)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info("args = %s", args)
logging.info("unparsed args = %s", unparsed)
num_gpus = torch.cuda.device_count()
genotype = eval("genotypes.%s" % args.arch)
print('---------Genotype---------')
logging.info(genotype)
print('--------------------------')
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = torch.nn.DataParallel(model)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
if args.cifar100:
train_transform, valid_transform = utils._data_transforms_cifar100(
args)
else:
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.cifar100:
train_data = dset.CIFAR100(root=args.tmp_data_dir,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR100(root=args.tmp_data_dir,
train=False,
download=True,
transform=valid_transform)
else:
train_data = dset.CIFAR10(root=args.tmp_data_dir,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.tmp_data_dir,
train=False,
download=True,
transform=valid_transform)
# train_queue = torch.utils.data.DataLoader(
# train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=args.workers)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=args.workers)
num_train = len(train_data)
indices = list(range(num_train))
split = 45000
# int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True,
num_workers=args.workers)
train_valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:]),
pin_memory=True,
num_workers=args.workers)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
best_acc = 0.0
for epoch in range(args.epochs):
logging.info('Epoch: %d lr %e', epoch, scheduler.get_lr()[0])
model.module.drop_path_prob = args.drop_path_prob * epoch / args.epochs
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
start_time = time.time()
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
scheduler.step()
logging.info('Train_acc: %f', train_acc)
valid_acc, valid_obj = infer(train_valid_queue, model, criterion)
if valid_acc > best_acc:
best_acc = valid_acc
test_acc, _ = infer(valid_queue, model, criterion)
logging.info('Test_acc: %f', test_acc)
logging.info('Valid_acc: %f', valid_acc)
logging.info('Best_acc: %f', best_acc)
end_time = time.time()
duration = end_time - start_time
print('Epoch time: %ds.' % duration)
utils.save(model.module, os.path.join(args.save, 'weights.pt'))
def main():
wandb.init(
project="automl-gradient-based-nas",
name="hw" + str(args.arch),
config=args,
entity="automl"
)
wandb.config.update(args) # adds all of the arguments as config variables
global is_multi_gpu
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
gpus = [int(i) for i in args.gpu.split(',')]
logging.info('gpus = %s' % gpus)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %s' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype)
if len(gpus) > 1:
print("Let's use", torch.cuda.device_count(), "GPUs!")
# dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs
model = nn.DataParallel(model)
is_multi_gpu = True
model.cuda()
weight_params = model.module.parameters() if is_multi_gpu else model.parameters()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
wandb.run.summary["param_size"] = utils.count_parameters_in_MB(model)
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(
weight_params, # model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
best_accuracy = 0
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
wandb.log({"evaluation_train_acc": train_acc, 'epoch': epoch})
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
wandb.log({"evaluation_valid_acc": valid_acc, 'epoch': epoch})
if valid_acc > best_accuracy:
wandb.run.summary["best_valid_accuracy"] = valid_acc
wandb.run.summary["epoch_of_best_accuracy"] = epoch
best_accuracy = valid_acc
utils.save(model, os.path.join(wandb.run.dir, 'weights-best.pt'))
utils.save(model, os.path.join(wandb.run.dir, 'weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
device = torch.device("cuda:{}".format(args.gpu))
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
cudnn.deterministic = True
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
if args.arch is not None:
genotype = eval("genotypes.%s" % args.arch)
if args.dir is not None:
with open(os.path.join(args.dir, "genotype.pickle"), 'rb') as f:
genotype = pickle.load(f)
print("Unpickling genotype.pickle")
logging.info(genotype)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar100(args)
train_data = dset.CIFAR100(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR100(root=args.data,
train=False,
download=True,
transform=valid_transform)
logging.info("[INFO] len(train_data): {}, len(valid_data): {}".format(
len(train_data), len(valid_data)))
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
test_error = []
best_acc = 0.0
for epoch in range(args.epochs):
logging.info('[INFO] epoch %d lr %e', epoch + 1, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('[INFO] train_acc %f', train_acc)
writer.add_scalar("train_acc", train_acc, epoch + 1)
writer.add_scalar("train_obj", train_obj, epoch + 1)
scheduler.step()
valid_acc, valid_obj = infer(valid_queue, model, criterion)
if valid_acc > best_acc:
best_acc = valid_acc
utils.save(model, os.path.join(args.save, 'best_weights.pt'))
logging.info('[INFO] valid_acc %f', valid_acc)
writer.add_scalar("valid_acc", valid_acc, epoch + 1)
writer.add_scalar("valid_obj", valid_obj, epoch + 1)
writer.add_scalar("test_error", 100 - valid_acc, epoch + 1)
utils.save(model, os.path.join(args.save, 'weights.pt'))
test_error.append(100 - valid_acc)
logging.info('[INFO] best_acc %f', best_acc)
with open("{}/test_error.pickle".format(args.save), 'wb') as f:
pickle.dump(test_error, f)
def main():
np.random.seed(args.seed)
random.seed(args.seed)
if torch.cuda.is_available():
device = torch.device('cuda:{}'.format(args.gpu))
cudnn.benchmark = False
torch.manual_seed(args.seed)
cudnn.enabled = True
cudnn.deterministic = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
else:
device = torch.device('cpu')
logging.info('No gpu device available')
torch.manual_seed(args.seed)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.to(device)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
total_params = sum(x.data.nelement() for x in model.parameters())
logging.info('Model total parameters: {}'.format(total_params))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10(
args.cutout, args.cutout_length)
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=0)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size // 2,
shuffle=False,
pin_memory=True,
num_workers=0)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
for epoch in range(args.epochs):
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer,
args.gpu)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion, args.gpu)
logging.info('valid_acc %f', valid_acc)
scheduler.step()
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %s' % args.gpu)
logging.info("args = %s", args)
genotype = eval(args.arch)
print(genotype)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
total_params = sum(p.numel() for p in model.parameters()
if p.requires_grad)
print(total_params)
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, valid_transform = utils._data_transforms_cifar10()
train_data = dset.CIFAR10(root=args.data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=32,
)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=32)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
max_acc = 0
for epoch in range(args.epochs):
model.drop_path_prob = args.drop_path_prob
# model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
scheduler.step()
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
logging.info('max acc:{}'.format(max_acc))
if valid_acc > max_acc:
logging.info('Update Max Acc')
max_acc = valid_acc
utils.save(model, os.path.join(local_save, 'weights.pt'))
dump_results((genotype, valid_acc / 100))
def main(args):
place = fluid.CUDAPlace(fluid.dygraph.parallel.Env().dev_id) \
if args.use_data_parallel else fluid.CUDAPlace(0)
with fluid.dygraph.guard(place):
genotype = eval("genotypes.%s" % args.arch)
model = Network(
C=args.init_channels,
num_classes=args.class_num,
layers=args.layers,
auxiliary=args.auxiliary,
genotype=genotype)
logger.info("param size = {:.6f}MB".format(
count_parameters_in_MB(model.parameters())))
device_num = fluid.dygraph.parallel.Env().nranks
step_per_epoch = int(args.trainset_num /
(args.batch_size * device_num))
learning_rate = fluid.dygraph.CosineDecay(args.learning_rate,
step_per_epoch, args.epochs)
clip = fluid.clip.GradientClipByGlobalNorm(clip_norm=args.grad_clip)
optimizer = fluid.optimizer.MomentumOptimizer(
learning_rate,
momentum=args.momentum,
regularization=fluid.regularizer.L2Decay(args.weight_decay),
parameter_list=model.parameters(),
grad_clip=clip)
if args.use_data_parallel:
strategy = fluid.dygraph.parallel.prepare_context()
model = fluid.dygraph.parallel.DataParallel(model, strategy)
train_loader = fluid.io.DataLoader.from_generator(
capacity=64,
use_double_buffer=True,
iterable=True,
return_list=True,
use_multiprocess=args.use_multiprocess)
valid_loader = fluid.io.DataLoader.from_generator(
capacity=64,
use_double_buffer=True,
iterable=True,
return_list=True,
use_multiprocess=args.use_multiprocess)
train_reader = reader.train_valid(
batch_size=args.batch_size,
is_train=True,
is_shuffle=True,
args=args)
valid_reader = reader.train_valid(
batch_size=args.batch_size,
is_train=False,
is_shuffle=False,
args=args)
if args.use_data_parallel:
train_reader = fluid.contrib.reader.distributed_batch_reader(
train_reader)
train_loader.set_batch_generator(train_reader, places=place)
valid_loader.set_batch_generator(valid_reader, places=place)
save_parameters = (not args.use_data_parallel) or (
args.use_data_parallel and
fluid.dygraph.parallel.Env().local_rank == 0)
best_acc = 0
for epoch in range(args.epochs):
drop_path_prob = args.drop_path_prob * epoch / args.epochs
logger.info('Epoch {}, lr {:.6f}'.format(
epoch, optimizer.current_step_lr()))
train_top1 = train(model, train_loader, optimizer, epoch,
drop_path_prob, args)
logger.info("Epoch {}, train_acc {:.6f}".format(epoch, train_top1))
valid_top1 = valid(model, valid_loader, epoch, args)
if valid_top1 > best_acc:
best_acc = valid_top1
if save_parameters:
fluid.save_dygraph(model.state_dict(),
args.model_save_dir + "/best_model")
logger.info("Epoch {}, valid_acc {:.6f}, best_valid_acc {:.6f}".
format(epoch, valid_top1, best_acc))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, NTU_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
trainset = dataset_ntuxview.MyDataset(
'/media/lab540/79eff75a-f78c-42f2-8902-9358e88bf654/lab540/Neura_auto_search/datasets/ntu112/ntu_cv/train.txt',
transform=transform.ToTensor())
train_queue = torch.utils.data.DataLoader(trainset,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
validset = dataset_ntuxview.MyDataset(
'/media/lab540/79eff75a-f78c-42f2-8902-9358e88bf654/lab540/Neura_auto_search/datasets/ntu112/ntu_cv/test.txt',
transform=transform.ToTensor())
valid_queue = torch.utils.data.DataLoader(validset,
batch_size=args.batch_size,
shuffle=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
best_acc = 0.0
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
if valid_acc > best_acc:
best_acc = valid_acc
logging.info('valid_acc %f, best_acc %f', valid_acc, best_acc)
end_time = time.time()
duration = end_time - start_time
print('Epoch time: %ds.' % duration)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
random.seed(args.seed)
np.random.seed(
args.data_seed) # cutout and load_corrupted_data use np.random
torch.cuda.set_device(args.gpu)
cudnn.benchmark = False
torch.manual_seed(args.seed)
cudnn.enabled = True
cudnn.deterministic = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
if args.arch == 'resnet':
model = ResNet18(CIFAR_CLASSES).cuda()
args.auxiliary = False
elif args.arch == 'resnet50':
model = ResNet50(CIFAR_CLASSES).cuda()
args.auxiliary = False
elif args.arch == 'resnet34':
model = ResNet34(CIFAR_CLASSES).cuda()
args.auxiliary = False
else:
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, test_transform = utils._data_transforms_cifar10(args)
# Load dataset
if args.dataset == 'cifar10':
noisy_train_data = CIFAR10(root=args.data,
train=True,
gold=False,
gold_fraction=0.0,
corruption_prob=args.corruption_prob,
corruption_type=args.corruption_type,
transform=train_transform,
download=True,
seed=args.data_seed)
gold_train_data = CIFAR10(root=args.data,
train=True,
gold=True,
gold_fraction=1.0,
corruption_prob=args.corruption_prob,
corruption_type=args.corruption_type,
transform=train_transform,
download=True,
seed=args.data_seed)
test_data = dset.CIFAR10(root=args.data,
train=False,
download=True,
transform=test_transform)
elif args.dataset == 'cifar100':
noisy_train_data = CIFAR100(root=args.data,
train=True,
gold=False,
gold_fraction=0.0,
corruption_prob=args.corruption_prob,
corruption_type=args.corruption_type,
transform=train_transform,
download=True,
seed=args.data_seed)
gold_train_data = CIFAR100(root=args.data,
train=True,
gold=True,
gold_fraction=1.0,
corruption_prob=args.corruption_prob,
corruption_type=args.corruption_type,
transform=train_transform,
download=True,
seed=args.data_seed)
test_data = dset.CIFAR100(root=args.data,
train=False,
download=True,
transform=test_transform)
num_train = len(gold_train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
if args.gold_fraction == 1.0:
train_data = gold_train_data
else:
train_data = noisy_train_data
train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True,
num_workers=0)
if args.clean_valid:
valid_data = gold_train_data
else:
valid_data = noisy_train_data
valid_queue = torch.utils.data.DataLoader(
valid_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:]),
pin_memory=True,
num_workers=0)
test_queue = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
if args.loss_func == 'cce':
criterion = nn.CrossEntropyLoss().cuda()
elif args.loss_func == 'rll':
criterion = utils.RobustLogLoss(alpha=args.alpha).cuda()
elif args.loss_func == 'forward_gold':
corruption_matrix = train_data.corruption_matrix
criterion = utils.ForwardGoldLoss(corruption_matrix=corruption_matrix)
else:
assert False, "Invalid loss function '{}' given. Must be in {'cce', 'rll'}".format(
args.loss_func)
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer_valid(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
test_acc, test_obj = infer(test_queue, model, criterion)
logging.info('test_acc %f', test_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
random.seed(args.seed)
if args.arch == 'resnet':
model = ResNet18(CIFAR_CLASSES).cuda()
args.auxiliary = False
elif args.arch == 'resnet50':
model = ResNet50(CIFAR_CLASSES).cuda()
args.auxiliary = False
elif args.arch == 'resnet34':
model = ResNet34(CIFAR_CLASSES).cuda()
args.auxiliary = False
else:
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_transform, test_transform = utils._data_transforms_cifar10(args)
train_data = CIFAR10(root=args.data,
train=True,
gold=False,
gold_fraction=0.0,
corruption_prob=args.corruption_prob,
corruption_type=args.corruption_type,
transform=train_transform,
download=True,
seed=args.seed)
gold_train_data = CIFAR10(root=args.data,
train=True,
gold=True,
gold_fraction=1.0,
corruption_prob=args.corruption_prob,
corruption_type=args.corruption_type,
transform=train_transform,
download=True,
seed=args.seed)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
clean_train_queue = torch.utils.data.DataLoader(
gold_train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True,
num_workers=2)
noisy_train_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True,
num_workers=2)
# clean_train_list = []
# for input, target in clean_train_queue:
# input = Variable(input).cuda()
# target = Variable(target).cuda(async=True)
# clean_train_list.append((input, target))
#
# noisy_train_list = []
# for input, target in noisy_train_queue:
# input = Variable(input).cuda()
# target = Variable(target).cuda(async=True)
# noisy_train_list.append((input, target))
clean_valid_queue = torch.utils.data.DataLoader(
gold_train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:]),
pin_memory=True,
num_workers=2)
noisy_valid_queue = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[split:]),
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
if args.loss_func == 'cce':
criterion = nn.CrossEntropyLoss().cuda()
elif args.loss_func == 'rll':
criterion = utils.RobustLogLoss(alpha=args.alpha).cuda()
elif args.loss_func == 'forward_gold':
corruption_matrix = train_data.corruption_matrix
criterion = utils.ForwardGoldLoss(corruption_matrix=corruption_matrix)
else:
assert False, "Invalid loss function '{}' given. Must be in {'cce', 'rll'}".format(
args.loss_func)
global gain
for epoch in range(args.epochs):
if args.random_weight:
logging.info('Epoch %d, Randomly assign weights', epoch)
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
clean_obj, noisy_obj = infer_random_weight(clean_train_queue,
noisy_train_queue,
model, criterion)
logging.info('clean loss %f, noisy loss %f', clean_obj, noisy_obj)
gain = np.random.randint(1, args.grad_clip, size=1)[0]
else:
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj, another_obj = train(clean_train_queue,
noisy_train_queue, model,
criterion, optimizer)
if args.clean_train:
logging.info('train_acc %f, clean_loss %f, noisy_loss %f',
train_acc, train_obj, another_obj)
else:
logging.info('train_acc %f, clean_loss %f, noisy_loss %f',
train_acc, another_obj, train_obj)
utils.save(model, os.path.join(args.save, 'weights.pt'))
clean_valid_acc, clean_valid_obj = infer_valid(clean_valid_queue,
model, criterion)
logging.info('clean_valid_acc %f, clean_valid_loss %f',
clean_valid_acc, clean_valid_obj)
noisy_valid_acc, noisy_valid_obj = infer_valid(noisy_valid_queue,
model, criterion)
logging.info('noisy_valid_acc %f, noisy_valid_loss %f',
noisy_valid_acc, noisy_valid_obj)
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
"""
train_transform, valid_transform = utils._data_transforms_cifar10(args)
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=2)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=2)
"""
data_dir = '../data/kmnist/'
data_augmentations = transforms.ToTensor()
# Load the Data here
train_dataset = K49(data_dir, True, data_augmentations)
#test_dataset = K49(data_dir, False, data_augmentations)
num_train = len(train_dataset)
indices = list(range(num_train))
split = int(np.floor(args.train_portion * num_train))
train_queue = torch.utils.data.DataLoader(
dataset=train_dataset,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(indices[:split]),
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(
dataset=train_dataset,
batch_size=args.batch_size,
sampler=torch.utils.data.sampler.SubsetRandomSampler(
indices[split:num_train]),
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled=True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype)
model = model.to('cuda')
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.to('cuda')
# not apply weight decay to BN layers
if args.bn_no_decay:
logging.info('BN layers are excluded from weight decay')
bn_params, other_params = utils.split_bn_params(model)
logging.debug('bn: %s', [p.dtype for p in bn_params])
logging.debug('other: %s', [p.dtype for p in other_params])
param_group = [{'params': bn_params, 'weight_decay': 0},
{'params': other_params}]
else:
param_group = model.parameters()
optimizer = torch.optim.SGD(
param_group,
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
logging.info('optimizer: %s', optimizer)
train_transform, valid_transform = utils.data_transforms_cifar10(args)
train_data = datasets.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = datasets.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=args.num_workers)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=args.num_workers)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
init_epoch = 0
best_acc = 0
if args.recover:
states = torch.load(args.recover)
model.load_state_dict(states['stete'])
init_epoch = states['epoch'] + 1
best_acc = states['best_acc']
logging.info('checkpoint loaded')
scheduler.step(init_epoch)
logging.info('scheduler is set to epoch %d. learning rate is %s', init_epoch, scheduler.get_lr())
for epoch in range(init_epoch, args.epochs):
logging.info('epoch %d lr %s', epoch, scheduler.get_lr())
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %.4f', train_acc)
with torch.no_grad():
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
logging.info('epoch %03d overall train_acc=%.4f valid_acc=%.4f', epoch, train_acc, valid_acc)
scheduler.step()
# gpu info
utils.gpu_usage(args.debug)
if valid_acc > best_acc:
best_acc = valid_acc
logging.info('best acc: %.4f', best_acc)
utils.save_checkpoint(state={'stete': model.state_dict(),
'epoch': epoch,
'best_acc': best_acc,},
is_best=False, save=args.save)
def main(args):
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info('gpu device = %d' % args.gpu)
logging.info("args = %s", args)
genotype = eval("genotypes.%s" % args.arch)
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
model = model.cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.MSELoss()
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
train_data = BindingDataset(args.annofile, args.seqfile)
num_train = len(train_data)
indices = list(range(num_train))
split = int(np.floor(0.7 * num_train))
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
for epoch in range(args.epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('train_acc %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
utils.save(model, os.path.join(args.save, 'weights.pt'))
def run(net,
init_ch=32,
layers=20,
auxiliary=True,
lr=0.025,
momentum=0.9,
wd=3e-4,
cutout=True,
cutout_length=16,
data='../data',
batch_size=96,
epochs=600,
drop_path_prob=0.2,
auxiliary_weight=0.4):
save = '/checkpoint/linnanwang/nasnet/' + hashlib.md5(
json.dumps(net).encode()).hexdigest()
utils.create_exp_dir(save, scripts_to_save=glob.glob('*.py'))
log_format = '%(asctime)s %(message)s'
logging.basicConfig(stream=sys.stdout,
level=logging.INFO,
format=log_format,
datefmt='%m/%d %I:%M:%S %p')
fh = logging.FileHandler(os.path.join(save, 'log.txt'))
fh.setFormatter(logging.Formatter(log_format))
logging.getLogger().addHandler(fh)
np.random.seed(0)
torch.cuda.set_device(0)
cudnn.benchmark = True
cudnn.enabled = True
torch.manual_seed(0)
logging.info('gpu device = %d' % 0)
# logging.info("args = %s", args)
genotype = net
model = Network(init_ch, 10, layers, auxiliary, genotype).cuda()
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(),
lr,
momentum=momentum,
weight_decay=wd)
model, optimizer = apex.amp.initialize(model, optimizer, opt_level="O3")
train_transform, valid_transform = utils._data_transforms_cifar10(
cutout, cutout_length)
train_data = dset.CIFAR10(root=data,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=data,
train=False,
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=batch_size,
shuffle=True,
pin_memory=True,
num_workers=2)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(epochs))
best_acc = 0.0
for epoch in range(epochs):
scheduler.step()
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
model.drop_path_prob = drop_path_prob * epoch / epochs
train_acc, train_obj = train(train_queue,
model,
criterion,
optimizer,
auxiliary=auxiliary,
auxiliary_weight=auxiliary_weight)
logging.info('train_acc: %f', train_acc)
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc: %f', valid_acc)
if valid_acc > best_acc and epoch >= 50:
print('this model is the best')
torch.save(model.state_dict(), os.path.join(save, 'model.pt'))
if valid_acc > best_acc:
best_acc = valid_acc
print('current best acc is', best_acc)
if epoch == 100:
break
# utils.save(model, os.path.join(args.save, 'trained.pt'))
print('saved to: model.pt')
return best_acc
def main():
#判断是否有GPU可用
if not torch.cuda.is_available():
logging.info('no gpu device available')
sys.exit(1)
np.random.seed(args.seed)
torch.cuda.set_device(args.gpu) #设置当前设备
cudnn.benchmark = True #加速计算
torch.manual_seed(args.seed) #为cpu设置随机数种子
cudnn.enabled=True #cuDNN是一个GPU加速深层神经网络原语库，开启cudnn
torch.cuda.manual_seed(args.seed)#为当前GPU设置随机种子
#打印日志信息
logging.info('gpu device = %d' % args.gpu)
#gpu device = 0
logging.info("args = %s", args)
'''
args = Namespace(arch='DARTS', auxiliary=False, auxiliary_weight=0.4, batch_size=96,
cutout=False, cutout_length=16, data='../data', drop_path_prob=0.2, epochs=600, gpu=0,
grad_clip=5, init_channels=36, layers=20, learning_rate=0.025, model_path='saved_models',
momentum=0.9, report_freq=50, save='eval-EXP-20190618-170816', seed=0, weight_decay=0.0003)
'''
genotype = eval("genotypes.%s" % args.arch) #应该是输出一个框架类型。eval() 函数用来执行一个字符串表达式，并返回表达式的值
#from model import NetworkCIFAR as Network #文件模块
model = Network(args.init_channels, CIFAR_CLASSES, args.layers, args.auxiliary, genotype)
#model = Network(通道个数=36, CIFAR_CLASSES=10, 总体layers=20, args.auxiliary使用辅助塔, genotype=框架类型)
model = model.cuda()
#打印模型参数的大小，即所占空间
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
#param size = 3.349342MB
criterion = nn.CrossEntropyLoss() #定义损失函数
criterion = criterion.cuda()
#定义优化器
optimizer = torch.optim.SGD(
model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay
)
#获得预处理之后的训练集和验证集
train_transform, valid_transform = utils._data_transforms_cifar10(args)
#获取数据集
train_data = dset.CIFAR10(root=args.data, train=True, download=True, transform=train_transform)
valid_data = dset.CIFAR10(root=args.data, train=False, download=True, transform=valid_transform)
'''
Files already downloaded and verified
Files already downloaded and verified
'''
#对数据进行封装为Tensor，主要用来读取数据集
'''
pin_memory:If True, the data loader will copy tensors into CUDA pinned memory before returning them,在数据返回前，是否将数据复制到CUDA内存中
num_workers:加快数据导入速度,工作者数量，默认是0。使用多少个子进程来导入数据。设置为0，就是使用主进程来导入数据。注意：这个数字必须是大于等于0的，不能太大，2的时候报错
'''
train_queue = torch.utils.data.DataLoader(
train_data, batch_size=args.batch_size, shuffle=True, pin_memory=True, num_workers=1)
valid_queue = torch.utils.data.DataLoader(
valid_data, batch_size=args.batch_size, shuffle=False, pin_memory=True, num_workers=1)
#优化器的学习率调整策略：采用CosineAnnealingLR，余弦退火调整学习率
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, float(args.epochs))
#默认epochs=600
for epoch in range(args.epochs):
scheduler.step() #更新权重
logging.info('epoch %d lr %e', epoch, scheduler.get_lr()[0])
#epoch 0 lr 2.500000e-02
#进行dropout：大小与模型的深度相关，模型深度越深，dropout的概率越大，最大0.2
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
#调用下面定义的函数train()
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
'''
train_queue:要训练的队列
model：采用的model；
criterion：定义的损失函数
optimizer：所采用的优化器
'''
logging.info('train_acc %f', train_acc) #打印当前epoch在训练集上的精度
#计算在验证集上的精度
valid_acc, valid_obj = infer(valid_queue, model, criterion)
logging.info('valid_acc %f', valid_acc)
#保存模型参数
utils.save(model, os.path.join(args.save, 'weights.pt'))
def main():
if not torch.cuda.is_available():
logging.info('No GPU device available')
sys.exit(1)
np.random.seed(args.seed)
cudnn.benchmark = True
torch.manual_seed(args.seed)
cudnn.enabled = True
torch.cuda.manual_seed(args.seed)
logging.info("args = %s", args)
logging.info("unparsed args = %s", unparsed)
num_gpus = torch.cuda.device_count()
genotype = eval("genotypes.%s" % args.arch)
print('---------Genotype---------')
logging.info(genotype)
print('--------------------------')
model = Network(args.init_channels, CIFAR_CLASSES, args.layers,
args.auxiliary, genotype)
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
model = torch.nn.DataParallel(model)
model = model.cuda()
start_epch = 0
if args.resume:
MT = torch.load(os.path.join(args.save, 'weight_optimizers.pt'))
model.load_state_dict(MT['net'])
optimizer.load_state_dict(MT['optimizer'])
start_epch = MT['epoch']
logging.info("param size = %fMB", utils.count_parameters_in_MB(model))
criterion = nn.CrossEntropyLoss()
criterion = criterion.cuda()
if args.cifar100:
train_transform, valid_transform = utils._data_transforms_cifar100(
args)
else:
train_transform, valid_transform = utils._data_transforms_cifar10(args)
if args.cifar100:
train_data = dset.CIFAR100(root=args.data_dir,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR100(root=args.data_dir,
train=False,
download=True,
transform=valid_transform)
else:
train_data = dset.CIFAR10(root=args.data_dir,
train=True,
download=True,
transform=train_transform)
valid_data = dset.CIFAR10(root=args.data_dir,
train=False,
download=True,
transform=valid_transform)
train_queue = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=args.workers)
valid_queue = torch.utils.data.DataLoader(valid_data,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=args.workers)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
optimizer, float(args.epochs))
best_acc = 0.0
for epoch in range(start_epch, args.epochs):
model.module.drop_path_prob = args.drop_path_prob * epoch / args.epochs
model.drop_path_prob = args.drop_path_prob * epoch / args.epochs
start_time = time.time()
train_acc, train_obj = train(train_queue, model, criterion, optimizer)
logging.info('Train_acc: %f', train_acc)
scheduler.step()
logging.info('Epoch: %d lr %e', epoch, scheduler.get_lr()[0])
valid_acc, valid_obj = infer(valid_queue, model, criterion)
if valid_acc > best_acc:
best_acc = valid_acc
state = {
'net': model.state_dict(),
'optimizer': optimizer.state_dict(),
'epoch': epoch
}
torch.save(state,
os.path.join(args.save, 'best_weight_optimizers.pt'))
logging.info('Valid_acc: %f, best_acc: %f', valid_acc, best_acc)
end_time = time.time()
duration = end_time - start_time
print('Epoch time: %d h.' % (duration * (args.epochs - epoch) / 3600))
if epoch % 50 == 0:
state = {
'net': model.state_dict(),
'optimizer': optimizer.state_dict(),
'epoch': epoch
}
torch.save(state, os.path.join(args.save, 'weight_optimizers.pt'))
