def get_model(settings_model: MutableMapping[str, Union[str, MutableMapping]],
settings_io: MutableMapping[str, Union[str, MutableMapping]],
output_classes: int,
device: str) \
-> Module:
"""Creates and returns the model for the process.
:param settings_model: Model specific settings to be used.
:type settings_model: dict
:param settings_io: File I/O settings to be used.
:type settings_io: dict
:param output_classes: Amount of output classes.
:type output_classes: int
:param device: Device to use for the model weights.
:type device: str|torch.device
:return: Model.
:rtype: torch.nn.Module
"""
encoder_settings = settings_model['encoder']
decoder_settings = settings_model['decoder']
decoder_settings.update({'nb_classes': output_classes})
kwargs = {**encoder_settings, **decoder_settings}
model = BaselineDCASE(**kwargs)
if settings_model['use_pre_trained_model']:
model.load_state_dict(
pt_load(Path(settings_io['root_dirs']['outputs'],
settings_io['model']['model_dir'],
settings_io['model']['pre_trained_model_name']),
map_location=device))
return model
def _do_training(model: Module,
settings_training: MutableMapping[
str, Union[Any, MutableMapping[str, Any]]],
settings_data: MutableMapping[
str, Union[Any, MutableMapping[str, Any]]],
settings_io: MutableMapping[
str, Union[Any, MutableMapping[str, Any]]],
model_file_name: str,
model_dir: Path,
indices_list: MutableSequence[str]) \
-> None:
"""Optimization of the model.
:param model: Model to optimize.
:type model: torch.nn.Module
:param settings_training: Training settings to use.
:type settings_training: dict
:param settings_data: Training data settings to use.
:type settings_data: dict
:param settings_io: Data I/O settings to use.
:type settings_io: dict
:param model_file_name: File name of the model.
:type model_file_name: str
:param model_dir: Directory to serialize the model to.
:type model_dir: pathlib.Path
:param indices_list: A sequence with the words.
:type indices_list: list[str]
"""
# Initialize variables for the training process
prv_training_loss = 1e8
patience: int = settings_training['patience']
loss_thr: float = settings_training['loss_thr']
patience_counter = 0
best_epoch = 0
# Initialize logger
logger_main = logger.bind(is_caption=False, indent=1)
# Inform that we start getting the data
logger_main.info('Getting training data')
# Get training data and count the amount of batches
training_data = get_clotho_loader(
settings_io['dataset']['features_dirs']['development'],
is_training=True,
settings_data=settings_data,
settings_io=settings_io)
logger_main.info('Done')
# Initialize loss and optimizer objects
objective = CrossEntropyLoss()
optimizer = Adam(params=model.parameters(),
lr=settings_training['optimizer']['lr'])
# Inform that we start training
logger_main.info('Starting training')
model.train()
for epoch in range(settings_training['nb_epochs']):
# Log starting time
start_time = time()
# Do a complete pass over our training data
epoch_output = module_epoch_passing(
data=training_data,
module=model,
objective=objective,
optimizer=optimizer,
grad_norm=settings_training['grad_norm']['norm'],
grad_norm_val=settings_training['grad_norm']['value'])
objective_output, output_y_hat, output_y, f_names = epoch_output
# Get mean loss of training and print it with logger
training_loss = objective_output.mean().item()
logger_main.info(f'Epoch: {epoch:05d} -- '
f'Training loss: {training_loss:>7.4f} | '
f'Time: {time() - start_time:>5.3f}')
# Check if we have to decode captions for the current epoch
if divmod(epoch + 1,
settings_training['text_output_every_nb_epochs'])[-1] == 0:
# Get the subset of files for decoding their captions
sampling_indices = sorted(
randperm(len(output_y_hat))
[:settings_training['nb_examples_to_sample']].tolist())
# Do the decoding
_decode_outputs(*zip(*[[output_y_hat[i], output_y[i]]
for i in sampling_indices]),
indices_object=indices_list,
file_names=[
Path(f_names[i_f_name])
for i_f_name in sampling_indices
],
eos_token='<eos>',
print_to_console=False)
# Check improvement of loss
if prv_training_loss - training_loss > loss_thr:
# Log the current loss
prv_training_loss = training_loss
# Log the current epoch
best_epoch = epoch
# Serialize the model keeping the epoch
pt_save(
model.state_dict(),
str(
model_dir.joinpath(
f'epoch_{best_epoch:05d}_{model_file_name}')))
# Zero out the patience
patience_counter = 0
else:
# Increase patience counter
patience_counter += 1
# Serialize the model and optimizer.
for pt_obj, save_str in zip([model, optimizer], ['', '_optimizer']):
pt_save(
pt_obj.state_dict(),
str(model_dir.joinpath(f'latest{save_str}_{model_file_name}')))
# Check for stopping criteria
if patience_counter >= patience:
logger_main.info('No lower training loss for '
f'{patience_counter} epochs. '
'Training stops.')
break
# Inform that we are done
logger_main.info('Training done')
# Load best model
model.load_state_dict(
pt_load(
str(model_dir.joinpath(
f'epoch_{best_epoch:05d}_{model_file_name}'))))
print("load settings end")
print("load pre-trained models start")
pretrain_emb = align_word_embedding(r'data/pickles/words_list.p',
r'outputs/models/w2v_192.mod', ntoken,
nhid)
# pretrain_cnn = torch.load(Path('outputs/models/TagModel_60.pt'), map_location='cuda')
model = TransformerModel(ntoken,
ninp,
nhead,
nhid,
nlayers,
batch_size,
dropout=0.2,
pretrain_cnn=None,
pretrain_emb=pretrain_emb,
freeze_cnn=False)
model.load_state_dict(
pt_load('outputs/models/epoch_090_transformer_model.pt',
map_location=device))
# model.load_encoder(pretrain_cnn, pretrain_emb)
model.to(device)
print('--------------------------------------------------------')
print("load pre-trained models end")
print("evaluation start")
def load(file): # ensure tensors are not loaded on the GPU return pt_load(file, map_location='cpu')
map_location='cuda')
model = TransformerModel(param.ntoken,
param.ninp,
param.nhead,
param.nhid,
param.nlayers,
param.batch_size,
dropout=0.2,
pretrain_cnn=pretrain_cnn,
pretrain_emb=None,
freeze_cnn=False)
if param.load_model:
model.load_state_dict(
pt_load('outputs/models/epoch_042_fine_tuning_model.pt',
map_location=device))
# model.load_encoder(pretrain_cnn, pretrain_emb)
model.to(device)
print('--------------------------------------------------------')
criterion = LabelSmoothingLoss(param.ntoken, smoothing=0.1)
optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad,
model.parameters()),
lr=param.lr,
weight_decay=1e-6)
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, 0.98)
if param.is_training:
epoch = 1
while epoch < param.training_epochs + 1:
scheduler.step(epoch)