def main():
parser = argparse.ArgumentParser(description='Proxyless-NAS augment')
parser.add_argument('-n',
'--name',
type=str,
required=True,
help="name of the model")
parser.add_argument('-c',
'--config',
type=str,
default='./config/default.yaml',
help="yaml config file")
parser.add_argument('-p',
'--chkpt',
type=str,
default=None,
help="path of checkpoint pt file")
parser.add_argument('-d',
'--device',
type=str,
default="all",
help="override device ids")
parser.add_argument('-g',
'--genotype',
type=str,
default=None,
help="override genotype file")
args = parser.parse_args()
hp = HParam(args.config)
pt_path = os.path.join('.', hp.log.chkpt_dir)
out_dir = os.path.join(pt_path, args.name)
os.makedirs(out_dir, exist_ok=True)
log_dir = os.path.join('.', hp.log.log_dir)
log_dir = os.path.join(log_dir, args.name)
os.makedirs(log_dir, exist_ok=True)
logger = utils.get_logger(log_dir, args.name)
if utils.check_config(hp, args.name):
raise Exception("Config error.")
writer = utils.get_writer(log_dir, hp.log.writer)
dev, dev_list = utils.init_device(hp.device, args.device)
trn_loader = load_data(hp.augment.data, validation=False)
val_loader = load_data(hp.augment.data, validation=True)
gt.set_primitives(hp.genotypes)
# load genotype
genotype = utils.get_genotype(hp.augment, args.genotype)
model, arch = get_model(hp.model, dev, dev_list, genotype)
augment(out_dir, args.chkpt, trn_loader, val_loader, model, writer, logger,
dev, hp.augment)
def main():
logger.info('=> PyTorch Version: {}'.format(torch.__version__))
# Environment initialization
device, pin_memory = init_device(args.seed, args.cpu, args.gpu,
args.cpu_affinity)
# Create the data loader
train_loader, val_loader, test_loader = Cost2100DataLoader(
root=args.data_dir,
batch_size=args.batch_size,
num_workers=args.workers,
pin_memory=pin_memory,
scenario=args.scenario)()
# Define model
model = init_model(args)
model.to(device)
# Define loss function
criterion = nn.MSELoss().to(device)
# Inference mode
if args.evaluate:
Tester(model, device, criterion)(test_loader)
return
# Define optimizer and scheduler
lr_init = 1e-3 if args.scheduler == 'const' else 2e-3
optimizer = torch.optim.Adam(model.parameters(), lr_init)
if args.scheduler == 'const':
scheduler = FakeLR(optimizer=optimizer)
else:
scheduler = WarmUpCosineAnnealingLR(optimizer=optimizer,
T_max=args.epochs *
len(train_loader),
T_warmup=30 * len(train_loader),
eta_min=5e-5)
# Define the training pipeline
trainer = Trainer(model=model,
device=device,
optimizer=optimizer,
criterion=criterion,
scheduler=scheduler,
resume=args.resume)
# Start training
trainer.loop(args.epochs, train_loader, val_loader, test_loader)
# Final testing
loss, rho, nmse = Tester(model, device, criterion)(test_loader)
print(f"\n=! Final test loss: {loss:.3e}"
f"\n test rho: {rho:.3e}"
f"\n test NMSE: {nmse:.3e}\n")
def compute_loss_by_model():
device = init_device()
train_data, valid_data, test_data, word_to_id, id_2_word = ptb_raw_data(data_path='data')
vocab_size = len(word_to_id)
loss_fn = nn.CrossEntropyLoss()
# Models from 4_1
models = [ModelInfo('RNN', 'ADAM', 0.0001, 20, 35, 1500, 2, 0.35),
ModelInfo('GRU','SGD_LR_SCHEDULE', 10, 20, 35, 1500, 2, 0.35),
ModelInfo('TRANSFORMER', 'ADAM', 20, 128, 35, 512, 6, 0.9)]
losses_by_model = []
for model_info in models:
model = load_model(model_info, device, vocab_size)
loss_per_step = compute_loss(model, model_info, device, valid_data, loss_fn)
losses_by_model.append(loss_per_step)
plot_loss(models, losses_by_model)
def main():
logger.info('=> PyTorch Version: {}'.format(torch.__version__))
# Environment initialization
device = init_device(args.seed, args.cpu, args.gpu, args.cpu_affinity)
# Create the test data loader
test_loader = Cost2100DataLoader(root=args.data_dir,
batch_size=args.batch_size,
num_workers=args.workers,
scenario=args.scenario)()
# Define model
model = init_model(args)
model.to(device)
# Define loss function
criterion = nn.MSELoss().to(device)
# Inference
Tester(model, device, criterion, print_freq=20)(test_loader)
help='use single GPU')
args = parser.parse_args()
if args.dropoutl < 0:
args.dropoutl = args.dropouth
if args.small_batch_size < 0:
args.small_batch_size = args.batch_size
# Logger init and set for utils
logger = get_logger(args, filename="finetune.log")
set_utils_logger(logger)
# Set the random seed manually for reproducibility.
np.random.seed(args.seed)
torch.manual_seed(args.seed)
# Sets the `args.device` and CUDA seed.
init_device(args)
# Save infos
save_args(args)
save_commit_id(args)
# Tensorboard
tb = TensorBoard(args.model_dir)
logger.info('finetune load path: {}/model.pt. '.format(args.model_dir))
logger.info('log save path: {}/finetune_log.txt'.format(args.model_dir))
logger.info('model save path: {}/finetune_model.pt'.format(args.model_dir))
###############################################################################
# Load data
###############################################################################
corpus = data.Corpus(args.data)
vocab_size=vocab_size,
load_on_device=False)
hidden = torch.zeros(model.num_layers, batch_size, model.hidden_size)
hidden = repackage_hidden(hidden)
samples = model.generate(start, hidden, seq_len)
generated_sentences = []
for i, sample in enumerate(samples.to("cpu"), 1):
sentence = " ".join([id_2_word[int(word)] for word in sample])
generated_sentences.append(sentence)
with open('%s_%s_generated_samples.txt' % (model_info.model, str(seq_len)),
'w') as f:
for sentence in generated_sentences:
f.write("%s\n\n" % sentence)
if __name__ == "__main__":
seq_lens = [35, 70]
generations_per_seq_len = 10
starting_word = "<eos>"
models = [
ModelInfo('RNN', 'ADAM', 0.0001, 20, 35, 1500, 2, 0.35),
ModelInfo('GRU', 'SGD_LR_SCHEDULE', 10, 20, 35, 1500, 2, 0.35)
]
device = init_device()
for m, s in product(models, seq_lens):
generate_sentences(m, device, s, generations_per_seq_len,
starting_word)