def get_model(settings_model: MutableMapping[str, Union[str, MutableMapping]], settings_io: MutableMapping[str, Union[str, MutableMapping]], output_classes: int, device: str) \ -> Module: """Creates and returns the model for the process. :param settings_model: Model specific settings to be used. :type settings_model: dict :param settings_io: File I/O settings to be used. :type settings_io: dict :param output_classes: Amount of output classes. :type output_classes: int :param device: Device to use for the model weights. :type device: str|torch.device :return: Model. :rtype: torch.nn.Module """ encoder_settings = settings_model['encoder'] decoder_settings = settings_model['decoder'] decoder_settings.update({'nb_classes': output_classes}) kwargs = {**encoder_settings, **decoder_settings} model = BaselineDCASE(**kwargs) if settings_model['use_pre_trained_model']: model.load_state_dict( pt_load(Path(settings_io['root_dirs']['outputs'], settings_io['model']['model_dir'], settings_io['model']['pre_trained_model_name']), map_location=device)) return model
def _do_training(model: Module, settings_training: MutableMapping[ str, Union[Any, MutableMapping[str, Any]]], settings_data: MutableMapping[ str, Union[Any, MutableMapping[str, Any]]], settings_io: MutableMapping[ str, Union[Any, MutableMapping[str, Any]]], model_file_name: str, model_dir: Path, indices_list: MutableSequence[str]) \ -> None: """Optimization of the model. :param model: Model to optimize. :type model: torch.nn.Module :param settings_training: Training settings to use. :type settings_training: dict :param settings_data: Training data settings to use. :type settings_data: dict :param settings_io: Data I/O settings to use. :type settings_io: dict :param model_file_name: File name of the model. :type model_file_name: str :param model_dir: Directory to serialize the model to. :type model_dir: pathlib.Path :param indices_list: A sequence with the words. :type indices_list: list[str] """ # Initialize variables for the training process prv_training_loss = 1e8 patience: int = settings_training['patience'] loss_thr: float = settings_training['loss_thr'] patience_counter = 0 best_epoch = 0 # Initialize logger logger_main = logger.bind(is_caption=False, indent=1) # Inform that we start getting the data logger_main.info('Getting training data') # Get training data and count the amount of batches training_data = get_clotho_loader( settings_io['dataset']['features_dirs']['development'], is_training=True, settings_data=settings_data, settings_io=settings_io) logger_main.info('Done') # Initialize loss and optimizer objects objective = CrossEntropyLoss() optimizer = Adam(params=model.parameters(), lr=settings_training['optimizer']['lr']) # Inform that we start training logger_main.info('Starting training') model.train() for epoch in range(settings_training['nb_epochs']): # Log starting time start_time = time() # Do a complete pass over our training data epoch_output = module_epoch_passing( data=training_data, module=model, objective=objective, optimizer=optimizer, grad_norm=settings_training['grad_norm']['norm'], grad_norm_val=settings_training['grad_norm']['value']) objective_output, output_y_hat, output_y, f_names = epoch_output # Get mean loss of training and print it with logger training_loss = objective_output.mean().item() logger_main.info(f'Epoch: {epoch:05d} -- ' f'Training loss: {training_loss:>7.4f} | ' f'Time: {time() - start_time:>5.3f}') # Check if we have to decode captions for the current epoch if divmod(epoch + 1, settings_training['text_output_every_nb_epochs'])[-1] == 0: # Get the subset of files for decoding their captions sampling_indices = sorted( randperm(len(output_y_hat)) [:settings_training['nb_examples_to_sample']].tolist()) # Do the decoding _decode_outputs(*zip(*[[output_y_hat[i], output_y[i]] for i in sampling_indices]), indices_object=indices_list, file_names=[ Path(f_names[i_f_name]) for i_f_name in sampling_indices ], eos_token='<eos>', print_to_console=False) # Check improvement of loss if prv_training_loss - training_loss > loss_thr: # Log the current loss prv_training_loss = training_loss # Log the current epoch best_epoch = epoch # Serialize the model keeping the epoch pt_save( model.state_dict(), str( model_dir.joinpath( f'epoch_{best_epoch:05d}_{model_file_name}'))) # Zero out the patience patience_counter = 0 else: # Increase patience counter patience_counter += 1 # Serialize the model and optimizer. for pt_obj, save_str in zip([model, optimizer], ['', '_optimizer']): pt_save( pt_obj.state_dict(), str(model_dir.joinpath(f'latest{save_str}_{model_file_name}'))) # Check for stopping criteria if patience_counter >= patience: logger_main.info('No lower training loss for ' f'{patience_counter} epochs. ' 'Training stops.') break # Inform that we are done logger_main.info('Training done') # Load best model model.load_state_dict( pt_load( str(model_dir.joinpath( f'epoch_{best_epoch:05d}_{model_file_name}'))))
print("load settings end") print("load pre-trained models start") pretrain_emb = align_word_embedding(r'data/pickles/words_list.p', r'outputs/models/w2v_192.mod', ntoken, nhid) # pretrain_cnn = torch.load(Path('outputs/models/TagModel_60.pt'), map_location='cuda') model = TransformerModel(ntoken, ninp, nhead, nhid, nlayers, batch_size, dropout=0.2, pretrain_cnn=None, pretrain_emb=pretrain_emb, freeze_cnn=False) model.load_state_dict( pt_load('outputs/models/epoch_090_transformer_model.pt', map_location=device)) # model.load_encoder(pretrain_cnn, pretrain_emb) model.to(device) print('--------------------------------------------------------') print("load pre-trained models end") print("evaluation start")
def load(file): # ensure tensors are not loaded on the GPU return pt_load(file, map_location='cpu')
map_location='cuda') model = TransformerModel(param.ntoken, param.ninp, param.nhead, param.nhid, param.nlayers, param.batch_size, dropout=0.2, pretrain_cnn=pretrain_cnn, pretrain_emb=None, freeze_cnn=False) if param.load_model: model.load_state_dict( pt_load('outputs/models/epoch_042_fine_tuning_model.pt', map_location=device)) # model.load_encoder(pretrain_cnn, pretrain_emb) model.to(device) print('--------------------------------------------------------') criterion = LabelSmoothingLoss(param.ntoken, smoothing=0.1) optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=param.lr, weight_decay=1e-6) scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, 0.98) if param.is_training: epoch = 1 while epoch < param.training_epochs + 1: scheduler.step(epoch)