def test_e2e_restore_with_dataloader(num_workers):
dset = DummyDataset()
# Expecting 10 batches in total.
sampler = SingleCutSampler(CUTS,
max_duration=10.0,
shuffle=True,
drop_last=True)
sampler.set_epoch(1)
# Note: not testing with num_workers > 1 as it will randomize the order of batches.
dloader = DataLoader(dset,
batch_size=None,
sampler=sampler,
num_workers=num_workers)
expected_batches = []
for idx, b in enumerate(dloader):
if idx == 4:
# Save the training loop state at step 4 to resume from later.
state = dloader.sampler.state_dict()
if idx > 4:
# Continue iterating to see what batches should be sampled after restoring.
expected_batches.append(b)
# Restore the sampler to its state from the dloader.
restored_sampler = SingleCutSampler(CUTS)
restored_sampler.load_state_dict(state)
# Initialize a new dloader with the restored sampler.
restored_dloader = DataLoader(dset,
batch_size=None,
sampler=restored_sampler,
num_workers=num_workers)
batches = []
for b in restored_dloader:
batches.append(b)
# Check that the results are the same.
assert len(expected_batches) == 5
if num_workers == 0:
assert len(batches) == 5
assert batches == expected_batches
else:
# We "lost" 2 batches due to prefetching (i.e., the sampler's state was ahead by 2 batches
# and we cannot recover from it for now)
assert len(batches) == 3
assert batches == expected_batches[2:]
def main():
fix_random_seed(42)
start_epoch = 0
num_epochs = 8
exp_dir = 'exp-lstm-adam-ctc-musan'
setup_logger('{}/log/log-train'.format(exp_dir))
tb_writer = SummaryWriter(log_dir=f'{exp_dir}/tensorboard')
# load L, G, symbol_table
lang_dir = Path('data/lang_nosp')
phone_symbol_table = k2.SymbolTable.from_file(lang_dir / 'phones.txt')
word_symbol_table = k2.SymbolTable.from_file(lang_dir / 'words.txt')
logging.info("Loading L.fst")
if (lang_dir / 'Linv.pt').exists():
L_inv = k2.Fsa.from_dict(torch.load(lang_dir / 'Linv.pt'))
else:
with open(lang_dir / 'L.fst.txt') as f:
L = k2.Fsa.from_openfst(f.read(), acceptor=False)
L_inv = k2.arc_sort(L.invert_())
torch.save(L_inv.as_dict(), lang_dir / 'Linv.pt')
graph_compiler = CtcTrainingGraphCompiler(
L_inv=L_inv,
phones=phone_symbol_table,
words=word_symbol_table
)
phone_ids = get_phone_symbols(phone_symbol_table)
# load dataset
feature_dir = Path('exp/data')
logging.info("About to get train cuts")
cuts_train = CutSet.from_json(feature_dir /
'cuts_train-clean-100.json.gz')
logging.info("About to get dev cuts")
cuts_dev = CutSet.from_json(feature_dir / 'cuts_dev-clean.json.gz')
logging.info("About to get Musan cuts")
cuts_musan = CutSet.from_json(feature_dir / 'cuts_musan.json.gz')
logging.info("About to create train dataset")
train = K2SpeechRecognitionDataset(
cuts_train,
cut_transforms=[
CutConcatenate(),
CutMix(
cuts=cuts_musan,
prob=0.5,
snr=(10, 20)
)
]
)
train_sampler = SingleCutSampler(
cuts_train,
max_frames=90000,
shuffle=True,
)
logging.info("About to create train dataloader")
train_dl = torch.utils.data.DataLoader(
train,
sampler=train_sampler,
batch_size=None,
num_workers=4
)
logging.info("About to create dev dataset")
validate = K2SpeechRecognitionDataset(cuts_dev)
valid_sampler = SingleCutSampler(cuts_dev, max_frames=90000)
logging.info("About to create dev dataloader")
valid_dl = torch.utils.data.DataLoader(
validate,
sampler=valid_sampler,
batch_size=None,
num_workers=1
)
if not torch.cuda.is_available():
logging.error('No GPU detected!')
sys.exit(-1)
logging.info("About to create model")
device_id = 0
device = torch.device('cuda', device_id)
model = TdnnLstm1b(
num_features=40,
num_classes=len(phone_ids) + 1, # +1 for the blank symbol
subsampling_factor=3)
model.to(device)
describe(model)
learning_rate = 1e-3
optimizer = optim.AdamW(model.parameters(),
lr=learning_rate,
weight_decay=5e-4)
best_objf = np.inf
best_valid_objf = np.inf
best_epoch = start_epoch
best_model_path = os.path.join(exp_dir, 'best_model.pt')
best_epoch_info_filename = os.path.join(exp_dir, 'best-epoch-info')
global_batch_idx_train = 0 # for logging only
if start_epoch > 0:
model_path = os.path.join(exp_dir, 'epoch-{}.pt'.format(start_epoch - 1))
ckpt = load_checkpoint(filename=model_path, model=model, optimizer=optimizer)
best_objf = ckpt['objf']
best_valid_objf = ckpt['valid_objf']
global_batch_idx_train = ckpt['global_batch_idx_train']
logging.info(f"epoch = {ckpt['epoch']}, objf = {best_objf}, valid_objf = {best_valid_objf}")
for epoch in range(start_epoch, num_epochs):
train_sampler.set_epoch(epoch)
curr_learning_rate = 1e-3
# curr_learning_rate = learning_rate * pow(0.4, epoch)
# for param_group in optimizer.param_groups:
# param_group['lr'] = curr_learning_rate
tb_writer.add_scalar('learning_rate', curr_learning_rate, epoch)
logging.info('epoch {}, learning rate {}'.format(
epoch, curr_learning_rate))
objf, valid_objf, global_batch_idx_train = train_one_epoch(dataloader=train_dl,
valid_dataloader=valid_dl,
model=model,
device=device,
graph_compiler=graph_compiler,
optimizer=optimizer,
current_epoch=epoch,
tb_writer=tb_writer,
num_epochs=num_epochs,
global_batch_idx_train=global_batch_idx_train)
# the lower, the better
if valid_objf < best_valid_objf:
best_valid_objf = valid_objf
best_objf = objf
best_epoch = epoch
save_checkpoint(filename=best_model_path,
model=model,
epoch=epoch,
optimizer=None,
scheduler=None,
learning_rate=curr_learning_rate,
objf=objf,
valid_objf=valid_objf,
global_batch_idx_train=global_batch_idx_train)
save_training_info(filename=best_epoch_info_filename,
model_path=best_model_path,
current_epoch=epoch,
learning_rate=curr_learning_rate,
objf=best_objf,
best_objf=best_objf,
valid_objf=valid_objf,
best_valid_objf=best_valid_objf,
best_epoch=best_epoch)
# we always save the model for every epoch
model_path = os.path.join(exp_dir, 'epoch-{}.pt'.format(epoch))
save_checkpoint(filename=model_path,
model=model,
optimizer=optimizer,
scheduler=None,
epoch=epoch,
learning_rate=curr_learning_rate,
objf=objf,
valid_objf=valid_objf,
global_batch_idx_train=global_batch_idx_train)
epoch_info_filename = os.path.join(exp_dir,
'epoch-{}-info'.format(epoch))
save_training_info(filename=epoch_info_filename,
model_path=model_path,
current_epoch=epoch,
learning_rate=curr_learning_rate,
objf=objf,
best_objf=best_objf,
valid_objf=valid_objf,
best_valid_objf=best_valid_objf,
best_epoch=best_epoch)
logging.warning('Done')
def main():
args = get_parser().parse_args()
print('World size:', args.world_size, 'Rank:', args.local_rank)
setup_dist(rank=args.local_rank,
world_size=args.world_size,
master_port=args.master_port)
fix_random_seed(42)
start_epoch = 0
num_epochs = 10
use_adam = True
exp_dir = f'exp-lstm-adam-mmi-bigram-musan'
setup_logger('{}/log/log-train'.format(exp_dir))
tb_writer = SummaryWriter(log_dir=f'{exp_dir}/tensorboard')
# load L, G, symbol_table
lang_dir = Path('data/lang_nosp')
lexicon = Lexicon(lang_dir)
device_id = args.local_rank
device = torch.device('cuda', device_id)
phone_ids = lexicon.phone_symbols()
if not Path(lang_dir / 'P.pt').is_file():
logging.debug(f'Loading P from {lang_dir}/P.fst.txt')
with open(lang_dir / 'P.fst.txt') as f:
# P is not an acceptor because there is
# a back-off state, whose incoming arcs
# have label #0 and aux_label eps.
P = k2.Fsa.from_openfst(f.read(), acceptor=False)
phone_symbol_table = k2.SymbolTable.from_file(lang_dir / 'phones.txt')
first_phone_disambig_id = find_first_disambig_symbol(
phone_symbol_table)
# P.aux_labels is not needed in later computations, so
# remove it here.
del P.aux_labels
# CAUTION(fangjun): The following line is crucial.
# Arcs entering the back-off state have label equal to #0.
# We have to change it to 0 here.
P.labels[P.labels >= first_phone_disambig_id] = 0
P = k2.remove_epsilon(P)
P = k2.arc_sort(P)
torch.save(P.as_dict(), lang_dir / 'P.pt')
else:
logging.debug('Loading pre-compiled P')
d = torch.load(lang_dir / 'P.pt')
P = k2.Fsa.from_dict(d)
graph_compiler = MmiTrainingGraphCompiler(
lexicon=lexicon,
P=P,
device=device,
)
# load dataset
feature_dir = Path('exp/data')
logging.info("About to get train cuts")
cuts_train = CutSet.from_json(feature_dir / 'cuts_train.json.gz')
logging.info("About to get dev cuts")
cuts_dev = CutSet.from_json(feature_dir / 'cuts_dev.json.gz')
logging.info("About to get Musan cuts")
cuts_musan = CutSet.from_json(feature_dir / 'cuts_musan.json.gz')
logging.info("About to create train dataset")
train = K2SpeechRecognitionDataset(cuts_train,
cut_transforms=[
CutConcatenate(),
CutMix(cuts=cuts_musan,
prob=0.5,
snr=(10, 20))
])
train_sampler = SingleCutSampler(
cuts_train,
max_frames=40000,
shuffle=True,
)
logging.info("About to create train dataloader")
train_dl = torch.utils.data.DataLoader(train,
sampler=train_sampler,
batch_size=None,
num_workers=4)
logging.info("About to create dev dataset")
validate = K2SpeechRecognitionDataset(cuts_dev)
valid_sampler = SingleCutSampler(cuts_dev, max_frames=12000)
logging.info("About to create dev dataloader")
valid_dl = torch.utils.data.DataLoader(validate,
sampler=valid_sampler,
batch_size=None,
num_workers=1)
if not torch.cuda.is_available():
logging.error('No GPU detected!')
sys.exit(-1)
logging.info("About to create model")
device_id = 0
device = torch.device('cuda', device_id)
model = TdnnLstm1b(
num_features=40,
num_classes=len(phone_ids) + 1, # +1 for the blank symbol
subsampling_factor=3)
model.P_scores = nn.Parameter(P.scores.clone(), requires_grad=True)
model.to(device)
describe(model)
if use_adam:
learning_rate = 1e-3
weight_decay = 5e-4
optimizer = optim.AdamW(model.parameters(),
lr=learning_rate,
weight_decay=weight_decay)
# Equivalent to the following in the epoch loop:
# if epoch > 6:
# curr_learning_rate *= 0.8
lr_scheduler = optim.lr_scheduler.LambdaLR(
optimizer, lambda ep: 1.0 if ep < 7 else 0.8**(ep - 6))
else:
learning_rate = 5e-5
weight_decay = 1e-5
momentum = 0.9
lr_schedule_gamma = 0.7
optimizer = optim.SGD(model.parameters(),
lr=learning_rate,
momentum=momentum,
weight_decay=weight_decay)
lr_scheduler = optim.lr_scheduler.ExponentialLR(
optimizer=optimizer, gamma=lr_schedule_gamma)
best_objf = np.inf
best_valid_objf = np.inf
best_epoch = start_epoch
best_model_path = os.path.join(exp_dir, 'best_model.pt')
best_epoch_info_filename = os.path.join(exp_dir, 'best-epoch-info')
global_batch_idx_train = 0 # for logging only
if start_epoch > 0:
model_path = os.path.join(exp_dir,
'epoch-{}.pt'.format(start_epoch - 1))
ckpt = load_checkpoint(filename=model_path,
model=model,
optimizer=optimizer,
scheduler=lr_scheduler)
best_objf = ckpt['objf']
best_valid_objf = ckpt['valid_objf']
global_batch_idx_train = ckpt['global_batch_idx_train']
logging.info(
f"epoch = {ckpt['epoch']}, objf = {best_objf}, valid_objf = {best_valid_objf}"
)
for epoch in range(start_epoch, num_epochs):
train_sampler.set_epoch(epoch)
# LR scheduler can hold multiple learning rates for multiple parameter groups;
# For now we report just the first LR which we assume concerns most of the parameters.
curr_learning_rate = lr_scheduler.get_last_lr()[0]
tb_writer.add_scalar('train/learning_rate', curr_learning_rate,
global_batch_idx_train)
tb_writer.add_scalar('train/epoch', epoch, global_batch_idx_train)
logging.info('epoch {}, learning rate {}'.format(
epoch, curr_learning_rate))
objf, valid_objf, global_batch_idx_train = train_one_epoch(
dataloader=train_dl,
valid_dataloader=valid_dl,
model=model,
device=device,
graph_compiler=graph_compiler,
optimizer=optimizer,
current_epoch=epoch,
tb_writer=tb_writer,
num_epochs=num_epochs,
global_batch_idx_train=global_batch_idx_train,
)
lr_scheduler.step()
# the lower, the better
if valid_objf < best_valid_objf:
best_valid_objf = valid_objf
best_objf = objf
best_epoch = epoch
save_checkpoint(filename=best_model_path,
model=model,
optimizer=None,
scheduler=None,
epoch=epoch,
learning_rate=curr_learning_rate,
objf=objf,
valid_objf=valid_objf,
global_batch_idx_train=global_batch_idx_train)
save_training_info(filename=best_epoch_info_filename,
model_path=best_model_path,
current_epoch=epoch,
learning_rate=curr_learning_rate,
objf=objf,
best_objf=best_objf,
valid_objf=valid_objf,
best_valid_objf=best_valid_objf,
best_epoch=best_epoch)
# we always save the model for every epoch
model_path = os.path.join(exp_dir, 'epoch-{}.pt'.format(epoch))
save_checkpoint(filename=model_path,
model=model,
optimizer=optimizer,
scheduler=lr_scheduler,
epoch=epoch,
learning_rate=curr_learning_rate,
objf=objf,
valid_objf=valid_objf,
global_batch_idx_train=global_batch_idx_train)
epoch_info_filename = os.path.join(exp_dir,
'epoch-{}-info'.format(epoch))
save_training_info(filename=epoch_info_filename,
model_path=model_path,
current_epoch=epoch,
learning_rate=curr_learning_rate,
objf=objf,
best_objf=best_objf,
valid_objf=valid_objf,
best_valid_objf=best_valid_objf,
best_epoch=best_epoch)
logging.warning('Done')
def main():
fix_random_seed(42)
start_epoch = 0
num_epochs = 10
use_adam = True
exp_dir = f'exp-lstm-adam-mmi-bigram-musan'
setup_logger('{}/log/log-train'.format(exp_dir))
tb_writer = SummaryWriter(log_dir=f'{exp_dir}/tensorboard')
# load L, G, symbol_table
lang_dir = Path('data/lang_nosp')
phone_symbol_table = k2.SymbolTable.from_file(lang_dir / 'phones.txt')
word_symbol_table = k2.SymbolTable.from_file(lang_dir / 'words.txt')
logging.info("Loading L.fst")
if (lang_dir / 'Linv.pt').exists():
L_inv = k2.Fsa.from_dict(torch.load(lang_dir / 'Linv.pt'))
else:
with open(lang_dir / 'L.fst.txt') as f:
L = k2.Fsa.from_openfst(f.read(), acceptor=False)
L_inv = k2.arc_sort(L.invert_())
torch.save(L_inv.as_dict(), lang_dir / 'Linv.pt')
graph_compiler = MmiTrainingGraphCompiler(L_inv=L_inv,
phones=phone_symbol_table,
words=word_symbol_table)
phone_ids = get_phone_symbols(phone_symbol_table)
P = create_bigram_phone_lm(phone_ids)
P.scores = torch.zeros_like(P.scores)
# load dataset
feature_dir = Path('exp/data')
logging.info("About to get train cuts")
cuts_train = CutSet.from_json(feature_dir / 'cuts_train.json.gz')
logging.info("About to get dev cuts")
cuts_dev = CutSet.from_json(feature_dir / 'cuts_dev.json.gz')
logging.info("About to get Musan cuts")
cuts_musan = CutSet.from_json(feature_dir / 'cuts_musan.json.gz')
logging.info("About to create train dataset")
train = K2SpeechRecognitionDataset(cuts_train,
cut_transforms=[
CutConcatenate(),
CutMix(cuts=cuts_musan,
prob=0.5,
snr=(10, 20))
])
train_sampler = SingleCutSampler(
cuts_train,
max_frames=12000,
shuffle=True,
)
logging.info("About to create train dataloader")
train_dl = torch.utils.data.DataLoader(train,
sampler=train_sampler,
batch_size=None,
num_workers=4)
logging.info("About to create dev dataset")
validate = K2SpeechRecognitionDataset(cuts_dev)
valid_sampler = SingleCutSampler(cuts_dev, max_frames=12000)
logging.info("About to create dev dataloader")
valid_dl = torch.utils.data.DataLoader(validate,
sampler=valid_sampler,
batch_size=None,
num_workers=1)
if not torch.cuda.is_available():
logging.error('No GPU detected!')
sys.exit(-1)
logging.info("About to create model")
device_id = 0
device = torch.device('cuda', device_id)
model = TdnnLstm1b(
num_features=40,
num_classes=len(phone_ids) + 1, # +1 for the blank symbol
subsampling_factor=3)
model.P_scores = nn.Parameter(P.scores.clone(), requires_grad=True)
model.to(device)
describe(model)
if use_adam:
learning_rate = 1e-3
weight_decay = 5e-4
optimizer = optim.AdamW(model.parameters(),
lr=learning_rate,
weight_decay=weight_decay)
# Equivalent to the following in the epoch loop:
# if epoch > 6:
# curr_learning_rate *= 0.8
lr_scheduler = optim.lr_scheduler.LambdaLR(
optimizer, lambda ep: 1.0 if ep < 7 else 0.8**(ep - 6))
else:
learning_rate = 5e-5
weight_decay = 1e-5
momentum = 0.9
lr_schedule_gamma = 0.7
optimizer = optim.SGD(model.parameters(),
lr=learning_rate,
momentum=momentum,
weight_decay=weight_decay)
lr_scheduler = optim.lr_scheduler.ExponentialLR(
optimizer=optimizer, gamma=lr_schedule_gamma)
best_objf = np.inf
best_valid_objf = np.inf
best_epoch = start_epoch
best_model_path = os.path.join(exp_dir, 'best_model.pt')
best_epoch_info_filename = os.path.join(exp_dir, 'best-epoch-info')
global_batch_idx_train = 0 # for logging only
if start_epoch > 0:
model_path = os.path.join(exp_dir,
'epoch-{}.pt'.format(start_epoch - 1))
ckpt = load_checkpoint(filename=model_path,
model=model,
optimizer=optimizer,
scheduler=lr_scheduler)
best_objf = ckpt['objf']
best_valid_objf = ckpt['valid_objf']
global_batch_idx_train = ckpt['global_batch_idx_train']
logging.info(
f"epoch = {ckpt['epoch']}, objf = {best_objf}, valid_objf = {best_valid_objf}"
)
for epoch in range(start_epoch, num_epochs):
train_sampler.set_epoch(epoch)
# LR scheduler can hold multiple learning rates for multiple parameter groups;
# For now we report just the first LR which we assume concerns most of the parameters.
curr_learning_rate = lr_scheduler.get_last_lr()[0]
tb_writer.add_scalar('train/learning_rate', curr_learning_rate,
global_batch_idx_train)
tb_writer.add_scalar('train/epoch', epoch, global_batch_idx_train)
logging.info('epoch {}, learning rate {}'.format(
epoch, curr_learning_rate))
objf, valid_objf, global_batch_idx_train = train_one_epoch(
dataloader=train_dl,
valid_dataloader=valid_dl,
model=model,
P=P,
device=device,
graph_compiler=graph_compiler,
optimizer=optimizer,
current_epoch=epoch,
tb_writer=tb_writer,
num_epochs=num_epochs,
global_batch_idx_train=global_batch_idx_train,
)
lr_scheduler.step()
# the lower, the better
if valid_objf < best_valid_objf:
best_valid_objf = valid_objf
best_objf = objf
best_epoch = epoch
save_checkpoint(filename=best_model_path,
model=model,
optimizer=None,
scheduler=None,
epoch=epoch,
learning_rate=curr_learning_rate,
objf=objf,
valid_objf=valid_objf,
global_batch_idx_train=global_batch_idx_train)
save_training_info(filename=best_epoch_info_filename,
model_path=best_model_path,
current_epoch=epoch,
learning_rate=curr_learning_rate,
objf=objf,
best_objf=best_objf,
valid_objf=valid_objf,
best_valid_objf=best_valid_objf,
best_epoch=best_epoch)
# we always save the model for every epoch
model_path = os.path.join(exp_dir, 'epoch-{}.pt'.format(epoch))
save_checkpoint(filename=model_path,
model=model,
optimizer=optimizer,
scheduler=lr_scheduler,
epoch=epoch,
learning_rate=curr_learning_rate,
objf=objf,
valid_objf=valid_objf,
global_batch_idx_train=global_batch_idx_train)
epoch_info_filename = os.path.join(exp_dir,
'epoch-{}-info'.format(epoch))
save_training_info(filename=epoch_info_filename,
model_path=model_path,
current_epoch=epoch,
learning_rate=curr_learning_rate,
objf=objf,
best_objf=best_objf,
valid_objf=valid_objf,
best_valid_objf=best_valid_objf,
best_epoch=best_epoch)
logging.warning('Done')
def main():
fix_random_seed(42)
if not torch.cuda.is_available():
logging.error("No GPU detected!")
sys.exit(-1)
device_id = 0
device = torch.device("cuda", device_id)
# Reserve the GPU with a dummy variable
reserve_variable = torch.ones(1).to(device)
start_epoch = 0
num_epochs = 100
exp_dir = "exp-tl1a-adam-xent"
setup_logger("{}/log/log-train".format(exp_dir))
tb_writer = SummaryWriter(log_dir=f"{exp_dir}/tensorboard")
# load dataset
feature_dir = Path("exp/data")
logging.info("About to get train cuts")
cuts_train = CutSet.from_json(feature_dir / "cuts_train.json.gz")
logging.info("About to get dev cuts")
cuts_dev = CutSet.from_json(feature_dir / "cuts_dev.json.gz")
logging.info("About to create train dataset")
train = K2VadDataset(cuts_train)
train_sampler = SingleCutSampler(
cuts_train,
max_frames=90000,
shuffle=True,
)
logging.info("About to create train dataloader")
train_dl = torch.utils.data.DataLoader(train,
sampler=train_sampler,
batch_size=None,
num_workers=4)
logging.info("About to create dev dataset")
validate = K2VadDataset(cuts_dev)
valid_sampler = SingleCutSampler(cuts_dev, max_frames=90000)
logging.info("About to create dev dataloader")
valid_dl = torch.utils.data.DataLoader(validate,
sampler=valid_sampler,
batch_size=None,
num_workers=1)
logging.info("About to create model")
model = TdnnLstm1a(
num_features=80,
num_classes=2, # speech/silence
subsampling_factor=1,
)
model.to(device)
describe(model)
learning_rate = 1e-4
optimizer = optim.AdamW(model.parameters(),
lr=learning_rate,
weight_decay=5e-4)
best_objf = np.inf
best_valid_objf = np.inf
best_epoch = start_epoch
best_model_path = os.path.join(exp_dir, "best_model.pt")
best_epoch_info_filename = os.path.join(exp_dir, "best-epoch-info")
global_batch_idx_train = 0 # for logging only
if start_epoch > 0:
model_path = os.path.join(exp_dir,
"epoch-{}.pt".format(start_epoch - 1))
ckpt = load_checkpoint(filename=model_path,
model=model,
optimizer=optimizer)
best_objf = ckpt["objf"]
best_valid_objf = ckpt["valid_objf"]
global_batch_idx_train = ckpt["global_batch_idx_train"]
logging.info(
f"epoch = {ckpt['epoch']}, objf = {best_objf}, valid_objf = {best_valid_objf}"
)
for epoch in range(start_epoch, num_epochs):
train_sampler.set_epoch(epoch)
curr_learning_rate = learning_rate
tb_writer.add_scalar("learning_rate", curr_learning_rate, epoch)
logging.info("epoch {}, learning rate {}".format(
epoch, curr_learning_rate))
objf, valid_objf, global_batch_idx_train = train_one_epoch(
dataloader=train_dl,
valid_dataloader=valid_dl,
model=model,
device=device,
optimizer=optimizer,
current_epoch=epoch,
tb_writer=tb_writer,
num_epochs=num_epochs,
global_batch_idx_train=global_batch_idx_train,
)
# the lower, the better
if valid_objf < best_valid_objf:
best_valid_objf = valid_objf
best_objf = objf
best_epoch = epoch
save_checkpoint(
filename=best_model_path,
model=model,
epoch=epoch,
optimizer=None,
scheduler=None,
learning_rate=curr_learning_rate,
objf=objf,
valid_objf=valid_objf,
global_batch_idx_train=global_batch_idx_train,
)
save_training_info(
filename=best_epoch_info_filename,
model_path=best_model_path,
current_epoch=epoch,
learning_rate=curr_learning_rate,
objf=best_objf,
best_objf=best_objf,
valid_objf=valid_objf,
best_valid_objf=best_valid_objf,
best_epoch=best_epoch,
)
# we always save the model for every epoch
model_path = os.path.join(exp_dir, "epoch-{}.pt".format(epoch))
save_checkpoint(
filename=model_path,
model=model,
optimizer=optimizer,
scheduler=None,
epoch=epoch,
learning_rate=curr_learning_rate,
objf=objf,
valid_objf=valid_objf,
global_batch_idx_train=global_batch_idx_train,
)
epoch_info_filename = os.path.join(exp_dir,
"epoch-{}-info".format(epoch))
save_training_info(
filename=epoch_info_filename,
model_path=model_path,
current_epoch=epoch,
learning_rate=curr_learning_rate,
objf=objf,
best_objf=best_objf,
valid_objf=valid_objf,
best_valid_objf=best_valid_objf,
best_epoch=best_epoch,
)
logging.warning("Done")
