def main():
logger.info("Logger is set - training start")
# set default gpu device id
torch.cuda.set_device(config.gpus[0])
# set seed
np.random.seed(config.seed)
torch.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
torch.backends.cudnn.benchmark = True
# get data with meta info
input_size, input_channels, n_classes, train_data, val_dat, test_dat = utils.get_data(
config.dataset,
config.data_path,
cutout_length=0,
validation=True,
validation2=True,
img_resize=config.img_resize)
net_crit = nn.CrossEntropyLoss().to(device)
model = SearchCNNController(input_channels,
config.init_channels,
n_classes,
config.layers,
net_crit,
device_ids=config.gpus)
#comment if generating onnix graph
model = model.to(device)
# weights optimizer
w_optim = torch.optim.SGD(model.weights(),
config.w_lr,
momentum=config.w_momentum,
weight_decay=config.w_weight_decay)
# alphas optimizer
alpha_optim = torch.optim.Adam(model.alphas(),
config.alpha_lr,
betas=(0.5, 0.999),
weight_decay=config.alpha_weight_decay)
#balanced split to train/validation
print(train_data)
# split data to train/validation
n_train = len(train_data) // int(config.data_train_proportion)
n_val = len(val_dat)
n_test = len(test_dat)
split = n_train // 2
indices1 = list(range(n_train))
indices2 = list(range(n_val))
indices3 = list(range(n_test))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices1)
valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices2)
test_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices3)
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=train_sampler,
num_workers=config.workers,
pin_memory=True)
valid_loader = torch.utils.data.DataLoader(val_dat,
batch_size=config.batch_size,
sampler=valid_sampler,
num_workers=config.workers,
pin_memory=True)
test_loader = torch.utils.data.DataLoader(test_dat,
batch_size=config.batch_size,
sampler=test_sampler,
num_workers=config.workers,
pin_memory=True)
#load
if (config.load):
model, config.epochs, w_optim, alpha_optim, net_crit = utils.load_checkpoint(
model, config.epochs, w_optim, alpha_optim, net_crit,
'/content/MyDarts/searchs/custom/checkpoint.pth.tar')
#uncomment if saving onnix graph
"""
dummy_input = Variable(torch.randn(1, 3, 64, 64))
torch.onnx.export(model, dummy_input, "rsdarts.onnx", verbose=True)
input_np = np.random.uniform(0, 1, (1, 3, 64, 64))
input_var = Variable(torch.FloatTensor(input_np))
from pytorch2keras.converter import pytorch_to_keras
# we should specify shape of the input tensor
output = model(input_var)
k_model = pytorch_to_keras(model, input_var, (3, 64, 64,), verbose=True)
error = check_error(output, k_model, input_np)
if max_error < error:
max_error = error
print('Max error: {0}'.format(max_error))
a=2/0
"""
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim, config.epochs, eta_min=config.w_lr_min)
architect = Architect(model, config.w_momentum, config.w_weight_decay)
#model = torch.load('/content/pt.darts/searchs/custom/checkpoint.pth.tar')
#print("Loaded!")
# training loop
best_top1 = 0.
best_top_overall = -999
config.epochs = 300 #BUG, config epochs ta com algum erro
for epoch in range(config.epochs):
lr_scheduler.step()
lr = lr_scheduler.get_lr()[0]
model.print_alphas(logger)
print("###################TRAINING#########################")
# training
#sample rs arch
arch = sample_arch(model)
#import pickle
#arch = pickle.load( open( "best_arch.p", "rb" ) )
train(train_loader, valid_loader, model, arch, w_optim, alpha_optim,
lr, epoch)
print("###################END TRAINING#########################")
# validation
cur_step = (epoch + 1) * len(train_loader)
print("###################VALID#########################")
top1, top_overall, _, _ = validate(valid_loader,
model,
arch,
epoch,
cur_step,
overall=True)
print("###################END VALID#########################")
# test
print("###################TEST#########################")
_, _, preds, targets = validate(test_loader,
model,
arch,
epoch,
cur_step,
overall=True,
debug=True)
s = [preds, targets]
import pickle
pickle.dump(s, open("predictions_" + str(epoch + 1) + ".p", "wb"))
#print("predictions: ",preds)
#print("targets:",targets)
print("###################END TEST#########################")
# log
# genotype
#print("Model Alpha:",model.alpha_normal)
genotype = model.genotype()
logger.info("genotype = {}".format(genotype))
# genotype as a image
plot_path = os.path.join(config.plot_path,
"EP{:02d}".format(epoch + 1))
caption = "Epoch {}".format(epoch + 1)
print("Genotype normal:", genotype.normal)
plot(genotype.normal, plot_path + "-normal", caption)
plot(genotype.reduce, plot_path + "-reduce", caption)
# save
if best_top1 < top1:
best_top1 = top1
best_genotype = genotype
best_arch = arch
is_best = True
import pickle
pickle.dump(best_arch, open("best_arch.p", "wb"))
print('best_arch:', best_arch)
print("saved!")
else:
is_best = False
#save best overall(macro avg of f1 prec and recall)
if (best_top_overall < top_overall):
best_top_overall = top_overall
best_genotype_overall = genotype
is_best_overall = True
else:
is_best_overall = False
utils.save_checkpoint(model, epoch, w_optim, alpha_optim, net_crit,
config.path, is_best, is_best_overall)
logger.info("Final best Prec@1 = {:.4%}".format(best_top1))
logger.info("Best Genotype = {}".format(best_genotype))
logger.info("Best Genotype Overall = {}".format(best_genotype_overall))
def main():
logger.info("Logger is set - training start")
# set default gpu device id
torch.cuda.set_device(config.gpus[0])
# set seed
np.random.seed(config.seed)
torch.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
torch.backends.cudnn.benchmark = True
# get data with meta info
dahai_train_dataset = utils.MyDataset(data_dir=TRAIN_DATA_PATH, )
dahai_dev_dataset = utils.MyDataset(data_dir=DEV_DAHAI_DATA_PATH, )
# zhikang_test_dataset = utils.MyDataset(window_size=WINDOW_SIZE,
# window_step=WINDOW_STEP_DEV,
# data_path=TEST_ZHIKANG_DATA_PATH,
# voice_embed_path=TEST_ZHIKANG_VOICE_EMBEDDING_PATH,
# w2i=w2i,
# sent_max_len=SENT_MAX_LEN,
# )
train_data = utils.DataProvider(batch_size=config.batch_size,
dataset=dahai_train_dataset,
is_cuda=config.is_cuda)
dev_data = utils.DataProvider(batch_size=config.batch_size,
dataset=dahai_dev_dataset,
is_cuda=config.is_cuda)
# test_data = utils.DataProvider(batch_size=config.batch_size, dataset=zhikang_test_dataset, is_cuda=config.is_cuda)
print("train data nums:", len(train_data.dataset), "dev data nums:",
len(dev_data.dataset))
net_crit = nn.CrossEntropyLoss(reduction="none").to(device)
model = SearchCNNController(config.embedding_dim,
config.init_channels,
config.n_classes,
config.layers,
net_crit,
config=config,
n_nodes=config.n_nodes,
device_ids=config.gpus)
model = model.to(device).float()
# weights optimizer
w_optim = torch.optim.SGD(model.weights(),
config.w_lr,
momentum=config.w_momentum,
weight_decay=config.w_weight_decay)
# alphas optimizer
alpha_optim = torch.optim.Adam(model.alphas(),
config.alpha_lr,
betas=(0.5, 0.999),
weight_decay=config.alpha_weight_decay)
###### 余弦退火-调整学习率
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim, config.epochs, eta_min=config.w_lr_min)
architect = Architect(model, config.w_momentum, config.w_weight_decay)
# training loop
best_acc = 0.
best_genotype = model.genotype()
while True:
epoch = train_data.epoch
if epoch > config.epochs - 1:
break
lr_scheduler.step()
lr = lr_scheduler.get_lr()[0]
model.print_alphas(logger)
# training
train(train_data, dev_data, epoch, model, architect, w_optim,
alpha_optim, lr)
# validation
cur_step = train_data.iteration
valid_acc = validate(dev_data, model, epoch, cur_step)
# log
# genotype
genotype = model.genotype()
logger.info("genotype = {}".format(genotype))
# genotype as a image
plot_path = os.path.join(config.plot_path,
"EP{:02d}".format(epoch + 1))
caption = "Epoch {}".format(epoch + 1)
plot(genotype.normal, plot_path + "-normal", caption)
plot(genotype.reduce, plot_path + "-reduce", caption)
# save
if best_acc < valid_acc:
best_acc = valid_acc
best_genotype = genotype
is_best = True
else:
is_best = False
utils.save_checkpoint(model, config.path, is_best)
print("")
logger.info("Final best Prec@1 = {:.4%}".format(best_acc))
logger.info("Best Genotype = {}".format(best_genotype))
def main():
logger.info("Logger is set - training start")
# set default gpu device id
torch.cuda.set_device(config.gpus[0])
# set seed
np.random.seed(config.seed)
torch.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
torch.backends.cudnn.benchmark = True
# get data with meta info
input_size, input_channels, n_classes, train_data = utils.get_data(
config.dataset, config.data_path, cutout_length=0, validation=False)
net_crit = nn.CrossEntropyLoss().to(device)
model = SearchCNNController(input_channels,
config.init_channels,
n_classes,
config.layers,
net_crit,
device_ids=config.gpus)
model = model.to(device)
# weights optimizer
w_optim = torch.optim.SGD(model.weights(),
config.w_lr,
momentum=config.w_momentum,
weight_decay=config.w_weight_decay)
# alphas optimizer
alpha_optim = torch.optim.Adam(model.alphas(),
config.alpha_lr,
betas=(0.5, 0.999),
weight_decay=config.alpha_weight_decay)
# split data to train/validation
n_train = len(train_data)
split = n_train // 2
indices = list(range(n_train))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(
indices[:split])
valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(
indices[split:])
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=train_sampler,
num_workers=config.workers,
pin_memory=False)
valid_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=valid_sampler,
num_workers=config.workers,
pin_memory=False)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim, config.epochs, eta_min=config.w_lr_min)
architect = Architect(model, config.w_momentum, config.w_weight_decay)
# training loop
best_top1 = -1.0
best_epoch = 0
################################ restore from last time #############################################
epoch_restore = config.epoch_restore
if config.restore:
utils.load_state_dict(model,
config.path,
extra='model',
parallel=(len(config.gpus) > 1))
if not config.model_only:
utils.load_state_dict(w_optim,
config.path,
extra='w_optim',
parallel=False)
utils.load_state_dict(alpha_optim,
config.path,
extra='alpha_optim',
parallel=False)
utils.load_state_dict(lr_scheduler,
config.path,
extra='lr_scheduler',
parallel=False)
utils.load_state_dict(epoch_restore,
config.path,
extra='epoch_restore',
parallel=False)
#####################################################################################################
for epoch in range(epoch_restore, config.epochs):
lr_scheduler.step()
lr = lr_scheduler.get_lr()[0]
model.print_alphas(logger)
# training
train(train_loader, valid_loader, model, architect, w_optim,
alpha_optim, lr, epoch)
# validation
cur_step = (epoch + 1) * len(train_loader)
top1 = validate(valid_loader, model, epoch, cur_step)
# top1 = 0.0
# log
# genotype
genotype = model.genotype()
logger.info("genotype = {}".format(genotype))
# genotype as a image
plot_path = os.path.join(config.plot_path,
"EP{:02d}".format(epoch + 1))
caption = "Epoch {}".format(epoch + 1)
plot(genotype.normal, plot_path + "-normal", caption)
plot(genotype.reduce, plot_path + "-reduce", caption)
# save
if best_top1 < top1:
best_top1 = top1
best_genotype = genotype
is_best = True
best_epoch = epoch + 1
else:
is_best = False
utils.save_checkpoint(model, config.path, is_best)
######################################## save all state ###################################################
utils.save_state_dict(model,
config.path,
extra='model',
is_best=is_best,
parallel=(len(config.gpus) > 1),
epoch=epoch + 1,
acc=top1,
last_state=((epoch + 1) >= config.epochs))
utils.save_state_dict(lr_scheduler,
config.path,
extra='lr_scheduler',
is_best=is_best,
parallel=False,
epoch=epoch + 1,
acc=top1,
last_state=((epoch + 1) >= config.epochs))
utils.save_state_dict(alpha_optim,
config.path,
extra='alpha_optim',
is_best=is_best,
parallel=False,
epoch=epoch + 1,
acc=top1,
last_state=((epoch + 1) >= config.epochs))
utils.save_state_dict(w_optim,
config.path,
extra='w_optim',
is_best=is_best,
parallel=False,
epoch=epoch + 1,
acc=top1,
last_state=((epoch + 1) >= config.epochs))
############################################################################################################
print("")
logger.info("Best Genotype at {} epch.".format(best_epoch))
logger.info("Final best Prec@1 = {:.4%}".format(best_top1))
logger.info("Best Genotype = {}".format(best_genotype))
def main():
logger.info("Logger is set - training start")
torch.cuda.set_device(config.gpus[0])
# seed setting
np.random.seed(config.seed)
torch.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
torch.backends.cudnn.benchmark = True
# get data with meta infomation
input_size, input_channels, n_classes, train_data = utils.get_data(
config.dataset, config.data_path, cutout_length=0, validation=False)
# set model
net_crit = nn.CrossEntropyLoss().to(device)
model = SearchCNNController(input_channels,
config.init_channels,
n_classes,
config.layers,
net_crit,
n_nodes=config.nodes,
device_ids=config.gpus)
model = model.to(device)
# weight optim
w_optim = torch.optim.SGD(model.weights(),
config.w_lr,
momentum=config.w_momentum,
weight_decay=config.alpha_weight_decay)
# alpha optim
alpha_optim = torch.optim.Adam(model.alphas(),
config.alpha_lr,
betas=(0.5, 0.999),
weight_decay=config.alpha_weight_decay)
# split data (train,validation)
n_train = len(train_data)
split = n_train // 2
indices = list(range(n_train))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(
indices[:split])
valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(
indices[split:])
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=train_sampler,
num_workers=config.workers,
pin_memory=True)
valid_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=valid_sampler,
num_workers=config.workers,
pin_memory=True)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim, config.epochs, eta_min=config.w_lr_min)
arch = Architect(model, config.w_momentum, config.w_weight_decay)
# training loop-----------------------------------------------------------------------------
best_top1 = 0.
for epoch in range(config.epochs):
lr_scheduler.step()
lr = lr_scheduler.get_lr()[0]
model.print_alphas(logger)
#training
train(train_loader, valid_loader, model, arch, w_optim, alpha_optim,
lr, epoch)
#validation
cur_step = (epoch + 1) * len(train_loader)
top1 = validate(valid_loader, model, epoch, cur_step)
#log
#genotype
genotype = model.genotype()
logger.info("genotype = {}".format(genotype))
# genotype as a image
plot_path = os.path.join(config.plot_path,
"EP{:02d}".format(epoch + 1))
caption = "Epoch {}".format(epoch + 1)
plot(genotype.normal, plot_path + "-normal", caption)
plot(genotype.reduce, plot_path + "-reduce", caption)
# output alpha per epochs to tensorboard data
for i, tensor in enumerate(model.alpha_normal):
for j, lsn in enumerate(F.softmax(tensor, dim=-1)):
tb_writer.add_scalars(
'epoch_alpha_normal/%d ~~ %d' % ((j - 2), i), {
'max_pl3': lsn[0],
'avg_pl3': lsn[1],
'skip_cn': lsn[2],
'sep_conv3': lsn[3],
'sep_conv5': lsn[4],
'dil_conv3': lsn[5],
'dil_conv5': lsn[6],
'none': lsn[7]
}, epoch)
for i, tensor in enumerate(model.alpha_reduce):
for j, lsr in enumerate(F.softmax(tensor, dim=-1)):
tb_writer.add_scalars(
'epoch_alpha_reduce/%d ~~ %d' % ((j - 2), i), {
'max_pl3': lsr[0],
'avg_pl3': lsr[1],
'skip_cn': lsr[2],
'sep_conv3': lsr[3],
'sep_conv5': lsr[4],
'dil_conv3': lsr[5],
'dil_conv5': lsr[6],
'none': lsr[7]
}, epoch)
#save
if best_top1 < top1:
best_top1 = top1
best_genotype = genotype
is_best = True
else:
is_best = False
utils.save_checkpoint(model, config.path, is_best)
print("")
logger.info("Final best Prec@1 = {:.4%}".format(best_top1))
logger.info("Best Genotype is = {}".format(best_genotype))
def main():
logger.info("Logger is set - training start")
# set default gpu device id
torch.cuda.set_device(config.gpus[0])
# get data with meta info
input_size, input_channels, n_classes, train_data = utils.get_data(
config.dataset, config.data_path, cutout_length=0, validation=False)
net_crit = nn.CrossEntropyLoss().to(device)
# set seed
np.random.seed(config.seed)
torch.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
model_1 = SearchCNNController(input_channels,
config.init_channels,
n_classes,
config.layers,
net_crit,
device_ids=config.gpus)
torch.manual_seed(config.seed + 1)
torch.cuda.manual_seed_all(config.seed + 1)
model_2 = SearchCNNController(input_channels,
config.init_channels,
n_classes,
config.layers,
net_crit,
device_ids=config.gpus)
torch.backends.cudnn.benchmark = True
model_1 = model_1.to(device)
model_2 = model_2.to(device)
# weights optimizer
w_optim_1 = torch.optim.SGD(model_1.weights(),
config.w_lr,
momentum=config.w_momentum,
weight_decay=config.w_weight_decay)
# alphas optimizer
alpha_optim_1 = torch.optim.Adam(model_1.alphas(),
config.alpha_lr,
betas=(0.5, 0.999),
weight_decay=config.alpha_weight_decay)
# weights optimizer
w_optim_2 = torch.optim.SGD(model_2.weights(),
config.w_lr,
momentum=config.w_momentum,
weight_decay=config.w_weight_decay)
# alphas optimizer
alpha_optim_2 = torch.optim.Adam(model_2.alphas(),
config.alpha_lr,
betas=(0.5, 0.999),
weight_decay=config.alpha_weight_decay)
# split data to train/validation
n_train = len(train_data)
split = n_train // 2
indices = list(range(n_train))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(
indices[:split])
valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(
indices[split:])
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=train_sampler,
num_workers=config.workers,
pin_memory=True)
valid_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=valid_sampler,
num_workers=config.workers,
pin_memory=True)
lr_scheduler_1 = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim_1, config.epochs, eta_min=config.w_lr_min)
lr_scheduler_2 = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim_2, config.epochs, eta_min=config.w_lr_min)
architect = Architect(model_1, model_2, config.w_momentum,
config.w_weight_decay)
# training loop
best_top1_1 = 0.
best_top1_2 = 0.
for epoch in range(config.epochs):
lr_scheduler_1.step()
lr_1 = lr_scheduler_1.get_lr()[0]
lr_scheduler_2.step()
lr_2 = lr_scheduler_2.get_lr()[0]
model_1.print_alphas(logger)
model_2.print_alphas(logger)
# training
train(train_loader, valid_loader, model_1, model_2, architect,
w_optim_1, w_optim_2, alpha_optim_1, alpha_optim_2, lr_1, lr_2,
epoch, config.lmbda)
# validation
cur_step = (epoch + 1) * len(train_loader)
top1_1, top1_2 = validate(valid_loader, model_1, model_2, epoch,
cur_step)
# log
# genotype
genotype_1 = model_1.genotype()
genotype_2 = model_2.genotype()
logger.info("genotype_1 = {}".format(genotype_1))
logger.info("genotype_2 = {}".format(genotype_2))
# genotype as a image
# plot_path = os.path.join(config.plot_path, "EP{:02d}".format(epoch+1))
# caption = "Epoch {}".format(epoch+1)
# plot(genotype_1.normal, plot_path + "-normal", caption)
# plot(genotype_1.reduce, plot_path + "-reduce", caption)
# plot(genotype_2.normal, plot_path + "-normal", caption)
# plot(genotype_2.reduce, plot_path + "-reduce", caption)
# save
if best_top1_1 < top1_1:
best_top1_1 = top1_1
best_genotype_1 = genotype_1
is_best_1 = True
else:
is_best_1 = False
if best_top1_2 < top1_2:
best_top1_2 = top1_2
best_genotype_2 = genotype_2
is_best_2 = True
else:
is_best_2 = False
utils.save_checkpoint(model_1, config.path, 1, is_best_1)
utils.save_checkpoint(model_2, config.path, 2, is_best_2)
print("")
logger.info("Final best Prec@1_1 = {:.4%}".format(best_top1_1))
logger.info("Best Genotype_1 = {}".format(best_genotype_1))
logger.info("Final best Prec@1_2 = {:.4%}".format(best_top1_2))
logger.info("Best Genotype_2 = {}".format(best_genotype_2))
def main():
logger.info("Logger is set - training start")
# set default gpu device id
torch.cuda.set_device(config.gpus[0])
# set seed
np.random.seed(config.seed)
torch.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
torch.backends.cudnn.benchmark = True
# get data with meta info
input_size, input_channels, n_classes, train_data = utils.get_data(
config.dataset, config.data_path, cutout_length=0, validation=False)
if config.ops_set == 2:
gt.PRIMITIVES = gt.PRIMITIVES2
if config.ops_set == 3:
gt.PRIMITIVES = gt.PRIMITIVES_NO_SKIP
if config.smart_sample:
gt.smart_sample = True
""" Initialize the distributed environment. """
if config.multi_avg_size > 0:
init_processes(config.multi_avg_rank, config.multi_avg_size, backend='Gloo')
net_crit = nn.CrossEntropyLoss().to(device)
model = SearchCNNController(input_channels, config.init_channels, n_classes, config.layers,
net_crit, n_nodes = config.n_nodes, device_ids=config.gpus, proxyless = config.proxyless)
model = model.to(device)
# weights optimizer
w_optim = torch.optim.SGD(model.weights(), config.w_lr, momentum=config.w_momentum,
weight_decay=config.w_weight_decay)
# alphas optimizer
alpha_optim = torch.optim.Adam(model.alphas(), config.alpha_lr, betas=(0.5, 0.999),
weight_decay=config.alpha_weight_decay)
# split data to train/validation
n_train = len(train_data)
split = n_train // 2
indices = list(range(n_train))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:split])
valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[split:])
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=train_sampler,
num_workers=config.workers,
pin_memory=True)
valid_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=valid_sampler,
num_workers=config.workers,
pin_memory=True)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim, config.epochs, eta_min=config.w_lr_min)
architect = Architect(model, config.w_momentum, config.w_weight_decay, config.hv_type)
# training loop
best_top1 = 0.
for epoch in range(config.epochs):
lr_scheduler.step()
lr = lr_scheduler.get_lr()[0]
model.print_alphas(logger)
# training
train(train_loader, valid_loader, model, architect, w_optim, alpha_optim, lr, epoch)
# validation
cur_step = (epoch+1) * len(train_loader)
top1 = validate(valid_loader, model, epoch, cur_step)
# log
# genotype
genotype = model.genotype()
logger.info("genotype = {}".format(genotype))
# genotype as a image
plot_path = os.path.join(config.plot_path, "EP{:02d}".format(epoch+1))
caption = "Epoch {}".format(epoch+1)
#plot(genotype.normal, plot_path + "-normal", caption)
#plot(genotype.reduce, plot_path + "-reduce", caption)
# save
if best_top1 < top1:
best_top1 = top1
best_genotype = genotype
is_best = True
else:
is_best = False
utils.save_checkpoint(model, config.path, is_best)
print("")
logger.info("Final best Prec@1 = {:.4%}".format(best_top1))
logger.info("Best Genotype = {}".format(best_genotype))
def main():
logger.info("Logger is set - training start")
# set default gpu device id
# torch.cuda.set_device(config.gpus[0])
# set seed
np.random.seed(config.seed)
torch.manual_seed(config.seed)
# torch.cuda.manual_seed_all(config.seed)
torch.backends.cudnn.benchmark = True
# get data with meta info
input_size, input_channels, n_classes, train_data = utils.get_data(
config.dataset, config.data_path, cutout_length=0, validation=False
)
net_crit = nn.CrossEntropyLoss().to(device)
model = SearchCNNController(
input_channels,
config.init_channels,
n_classes,
config.layers,
net_crit,
device_ids=config.gpus,
imagenet_mode=config.dataset.lower() in utils.LARGE_DATASETS,
)
model = model.to(device)
# weights optimizer
w_optim = torch.optim.SGD(
model.weights(),
config.w_lr,
momentum=config.w_momentum,
weight_decay=config.w_weight_decay,
)
# alphas optimizer
alpha_optim = torch.optim.Adam(
model.alphas(),
config.alpha_lr,
betas=(0.5, 0.999),
weight_decay=config.alpha_weight_decay,
)
# split data to train/validation
n_train = len(train_data)
split = n_train // 2
indices = list(range(n_train))
random.shuffle(indices)
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:split])
valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[split:])
train_loader = torch.utils.data.DataLoader(
train_data,
batch_size=config.batch_size,
sampler=train_sampler,
num_workers=config.workers,
pin_memory=True,
)
valid_loader = torch.utils.data.DataLoader(
train_data,
batch_size=config.batch_size,
sampler=valid_sampler,
num_workers=config.workers,
pin_memory=True,
)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim, config.epochs, eta_min=config.w_lr_min
)
architect = Architect(model, config.w_momentum, config.w_weight_decay)
# training loop
best_top1 = 0.0
for epoch in range(config.epochs):
lr_scheduler.step()
lr = lr_scheduler.get_lr()[0]
model.print_alphas(logger)
# training
train(
train_loader, valid_loader, model, architect, w_optim, alpha_optim, lr, epoch
)
# validation
cur_step = (epoch + 1) * len(train_loader)
top1 = validate(valid_loader, model, epoch, cur_step)
# log
# genotype
genotype = model.genotype()
logger.info("genotype = {}".format(genotype))
# genotype as a image
plot_path = os.path.join(config.plot_path, "EP{:02d}".format(epoch + 1))
caption = "Epoch {}".format(epoch + 1)
plot(genotype.normal, plot_path + "-normal", caption)
plot(genotype.reduce, plot_path + "-reduce", caption)
# save
if best_top1 < top1:
best_top1 = top1
best_genotype = genotype
is_best = True
else:
is_best = False
utils.save_checkpoint(model, config.path, is_best)
print("")
# restrict skip-co
count = 0
indices = []
for i in range(4):
_, primitive_indices = torch.topk(model.alpha_normal[i][:, :], 1)
for j in range(2 + i):
if primitive_indices[j].item() == 2:
count = count + 1
indices.append((i, j))
while count > 2:
alpha_min, indice_min = model.alpha_normal[indices[0][0]][indices[0][1], 2], 0
for i in range(1, count):
alpha_c = model.alpha_normal[indices[i][0]][indices[i][1], 2]
if alpha_c < alpha_min:
alpha_min, indice_min = alpha_c, i
model.alpha_normal[indices[indice_min][0]][indices[indice_min][1], 2] = 0
indices.pop(indice_min)
print(indices)
count = count - 1
best_genotype = model.genotype()
with open(config.path + "/best_genotype.txt", "w") as f:
f.write(str(best_genotype))
logger.info("Final best Prec@1 = {:.4%}".format(best_top1))
logger.info("Best Genotype = {}".format(best_genotype))
def main():
logger.info("Logger is set - training start")
fileRoot = r'/home/hlu/Data/VIPL'
saveRoot = r'/home/hlu/Data/VIPL_STMap' + str(config.fold_num) + str(
config.fold_index)
n_classes = 1
input_channels = 3
# set default gpu device id
torch.cuda.set_device(config.gpus[0])
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
toTensor = transforms.ToTensor()
resize = transforms.Resize(size=(64, 300))
# set seed
np.random.seed(config.seed)
torch.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
torch.backends.cudnn.benchmark = True # 网络加速
if config.reData == 1:
test_index, train_index = MyDataset.CrossValidation(
fileRoot, fold_num=config.fold_num, fold_index=config.fold_index)
train_data = MyDataset.Data_STMap(root_dir=(saveRoot + '_Train'),
frames_num=300,
transform=transforms.Compose(
[resize, toTensor, normalize]))
net_crit = nn.L1Loss().to(device)
model = SearchCNNController(input_channels,
config.init_channels,
n_classes,
config.layers,
net_crit,
device_ids=config.gpus)
model._init_weight()
model = model.to(device)
# weights optimizer
w_optim = torch.optim.SGD(model.weights(),
config.w_lr,
momentum=config.w_momentum,
weight_decay=config.w_weight_decay)
# w_optim = torch.optim.Adam(model.weights(), config.w_lr)
# alphas optimizer
alpha_optim = torch.optim.Adam(model.alphas(),
config.alpha_lr,
betas=(0.5, 0.999),
weight_decay=config.alpha_weight_decay)
# split data to train/validation
n_train = len(train_data)
split = n_train // 2
indices = list(range(n_train))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(
indices[:split])
valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(
indices[split:])
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=train_sampler,
num_workers=config.workers,
pin_memory=True)
valid_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=valid_sampler,
num_workers=config.workers,
pin_memory=True)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim, config.epochs, eta_min=config.w_lr_min)
architect = Architect(model, config.w_momentum, config.w_weight_decay)
# training loop
best_losses = 100
for epoch in range(config.epochs):
lr_scheduler.step()
lr = lr_scheduler.get_lr()[0]
model.print_alphas(logger)
# training
train(train_loader, valid_loader, model, architect, w_optim,
alpha_optim, lr, epoch)
# validation
cur_step = (epoch + 1) * len(train_loader)
losses = validate(valid_loader, model, epoch, cur_step)
# log
# genotype
genotype = model.genotype()
logger.info("genotype = {}".format(genotype))
# save
if losses < best_losses:
best_losses = losses
best_genotype = genotype
is_best = True
else:
is_best = False
utils.save_checkpoint(model, config.path, is_best)
print("")
logger.info("Best Genotype = {}".format(best_genotype))
async def main():
logger.info("Logger is set - training start")
# set default gpu device id
torch.cuda.set_device(config.gpus[0])
# set seed
np.random.seed(config.seed)
torch.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
torch.backends.cudnn.benchmark = True
# get data with meta info
input_size, input_channels, n_classes, train_data = utils.get_data(
config.dataset, config.data_path, cutout_length=0, validation=False)
net_crit = nn.CrossEntropyLoss().to(default_device)
model = SearchCNNController(input_channels,
config.init_channels,
n_classes,
config.layers,
net_crit,
device_ids=config.gpus)
model = model.to(default_device)
# weights optimizer
w_optim = torch.optim.SGD(model.weights(),
config.w_lr,
momentum=config.w_momentum,
weight_decay=config.w_weight_decay)
# alphas optimizer
alpha_optim = torch.optim.Adam(model.alphas(),
config.alpha_lr,
betas=(0.5, 0.999),
weight_decay=config.alpha_weight_decay)
# split data to train/validation
n_train = len(train_data)
split = n_train // 2
indices = list(range(n_train))
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(
indices[:split])
valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(
indices[split:])
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=train_sampler,
num_workers=config.workers,
pin_memory=True)
valid_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=valid_sampler,
num_workers=config.workers,
pin_memory=True)
for idx, (data, target) in enumerate(train_loader):
wid = idx % len(workers)
data = data.send(workers[wid])
target = target.send(workers[wid])
remote_train_data[wid].append((data, target))
for idx, (data, target) in enumerate(valid_loader):
wid = idx % len(workers)
data = data.send(workers[wid])
target = target.send(workers[wid])
remote_valid_data[wid].append((data, target))
print("finish sampler")
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(
w_optim, config.epochs, eta_min=config.w_lr_min)
architect = Architect(model, config.w_momentum, config.w_weight_decay)
# training loop
best_top1 = 0.
for epoch in range(config.epochs):
lr_scheduler.step()
lr = lr_scheduler.get_lr()[0]
model.print_alphas(logger)
# training
await train(train_loader, valid_loader, model, architect, w_optim,
alpha_optim, lr, epoch)
# validation
cur_step = (epoch + 1) * len(train_loader)
top1 = validate(valid_loader, model, epoch, cur_step)
# log
# genotype
genotype = model.genotype()
logger.info("genotype = {}".format(genotype))
# genotype as a image
plot_path = os.path.join(config.plot_path,
"EP{:02d}".format(epoch + 1))
caption = "Epoch {}".format(epoch + 1)
plot(genotype.normal, plot_path + "-normal", caption)
plot(genotype.reduce, plot_path + "-reduce", caption)
# save
if best_top1 < top1:
best_top1 = top1
best_genotype = genotype
is_best = True
else:
is_best = False
utils.save_checkpoint(model, config.path, is_best)
logger.info("Final best Prec@1 = {:.4%}".format(best_top1))
logger.info("Best Genotype = {}".format(best_genotype))
def main(config, writer, logger):
logger.info("Logger is set - training start")
# set seed
random.seed(config.seed)
np.random.seed(config.seed)
torch.manual_seed(config.seed)
torch.cuda.manual_seed_all(config.seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
# get data with meta info
input_size, input_channels, n_classes, train_data, test_data = utils.get_data(
config.dataset, config.data_path, cutout_length=config.cutout_length, validation=True)
net_crit = nn.CrossEntropyLoss().cuda()
model = SearchCNNController(input_channels, config.init_channels, n_classes, config.n_layers, net_crit,
n_nodes=config.n_nodes, stem_multiplier=config.stem_multiplier,
bn_momentum=config.bn_momentum)
model.cuda()
# weights optimizer
w_optim = optim.SGD(model.weights(), config.w_lr, momentum=config.w_momentum,
weight_decay=config.w_weight_decay)
if not config.search_all_alpha:
# alphas optimizer
alpha_optim = optim.Adam(model.alphas(), config.alpha_lr, betas=(0.5, 0.999),
weight_decay=config.alpha_weight_decay)
# split data to train/validation
n_train = len(train_data)
indices = list(range(n_train))
split = n_train // 2
train_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[:split])
valid_sampler = torch.utils.data.sampler.SubsetRandomSampler(indices[split:])
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=train_sampler,
num_workers=config.workers,
pin_memory=True)
valid_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
sampler=valid_sampler,
num_workers=config.workers,
pin_memory=True)
else:
alpha_optim = SubgraphSearchOptimizer(logger, config, model, w_optim)
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=config.batch_size,
num_workers=config.workers,
pin_memory=True,
shuffle=True)
valid_loader = torch.utils.data.DataLoader(test_data,
batch_size=config.batch_size,
num_workers=config.workers,
pin_memory=True,
shuffle=True)
if config.w_lr_scheduler == "cosine":
lr_scheduler = optim.lr_scheduler.CosineAnnealingLR(w_optim, T_max=config.epochs, eta_min=config.w_lr_min)
elif config.w_lr_scheduler == "plateau":
lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau(w_optim, mode="max", patience=config.w_lr_patience,
factor=config.w_lr_factor, verbose=True)
else:
raise NotImplementedError
architect = Architect(model, config.w_momentum, config.w_weight_decay)
# training loop
best_top1 = 0.
best_genotype = None
final_result_reported = False
for epoch in range(config.epochs):
if config.cutoff_epochs is not None and epoch >= config.cutoff_epochs:
logger.info("Cutoff epochs detected, exiting.")
break
lr = w_optim.param_groups[0]["lr"]
logger.info("Current learning rate: {}".format(lr))
if lr < config.w_lr_min:
logger.info("Learning rate is less than {}, exiting.".format(config.w_lr_min))
break
if not config.search_all_alpha:
model.print_alphas(logger)
valid_loader_for_training = valid_loader
else:
# make dummy input
valid_loader_for_training = itertools.cycle([(torch.tensor(1), torch.tensor(1))])
# training
train(config, writer, logger, train_loader, valid_loader_for_training,
model, architect, w_optim, alpha_optim, lr, epoch, valid_loader)
if config.w_lr_scheduler == "cosine":
lr_scheduler.step()
# validation
if config.validate_epochs == 0 or (epoch + 1) % config.validate_epochs != 0:
logger.info("Valid: Skipping validation for epoch {}".format(epoch + 1))
continue
cur_step = (epoch + 1) * len(train_loader)
if config.search_all_alpha:
top1 = validate_all(config, writer, logger, valid_loader, model, epoch, cur_step, alpha_optim)
if best_top1 < top1:
best_top1 = top1
# checkpoint saving is not supported yet
else:
top1 = validate(config, writer, logger, valid_loader, model, epoch, cur_step)
# log
# genotype
genotype = model.genotype()
logger.info("genotype = {}".format(genotype))
# genotype as a image
plot_path = os.path.join(config.plot_path, "EP{:02d}".format(epoch + 1))
caption = "Epoch {}".format(epoch + 1)
plot(genotype.normal, plot_path + "-normal", caption)
# save
if best_top1 < top1:
best_top1 = top1
best_genotype = genotype
is_best = True
else:
is_best = False
utils.save_checkpoint(model, config.path, is_best)
if config.nni:
nni_tools.report_result(top1, epoch + 1 == config.epochs)
if epoch + 1 == config.epochs:
final_result_reported = True
if config.w_lr_scheduler == "plateau":
lr_scheduler.step(top1)
print("")
if config.nni and not final_result_reported:
try:
nni_tools.report_result(top1, True)
except:
logger.warning("Final result not reported and top1 not found")
logger.info("Final best Prec@1 = {:.4%}".format(best_top1))
if best_genotype is not None:
logger.info("Best Genotype = {}".format(best_genotype))
print("")