cfg.lrate.get_next_rate(current_error = 100 * numpy.mean(valid_error)) # output nnet parameters and lrate, for training resume if cfg.lrate.epoch % cfg.model_save_step == 0: _nnet2file(dnn.layers, filename=wdir + '/nnet.tmp') _lrate2file(cfg.lrate, wdir + '/training_state.tmp') # save the model and network configuration for i in range(len(cfg.hidden_layers_sizes)): if i==0: n_params = (cfg.n_ins + 1) * cfg.hidden_layers_sizes[i] else: n_params += (cfg.hidden_layers_sizes[i-1] + 1) * cfg.hidden_layers_sizes[i] n_params += cfg.n_outs * (cfg.hidden_layers_sizes[i-1] + 1) ratio = float(n_params) / float(n_data_train) log('-->> Ratio Parameters / Data : ' + str(ratio)) if cfg.param_output_file != '': _nnet2file(dnn.layers, filename=cfg.param_output_file, input_factor = cfg.input_dropout_factor, factor = cfg.dropout_factor) log('> ... the final PDNN model parameter is ' + cfg.param_output_file) if cfg.cfg_output_file != '': _cfg2file(dnn.cfg, filename=cfg.cfg_output_file) log('> ... the final PDNN model config is ' + cfg.cfg_output_file) # output the model into Kaldi-compatible format if cfg.kaldi_output_file != '': dnn.write_model_to_kaldi(cfg.kaldi_output_file) log('> ... the final Kaldi model is ' + cfg.kaldi_output_file) # remove the tmp files (which have been generated from resuming training) os.remove(wdir + '/nnet.tmp') os.remove(wdir + '/training_state.tmp')
log('> ... finetuning the model') while (cfg.lrate.get_rate() != 0): # one epoch of sgd training train_error = train_sgd(train_fn, cfg) log('> epoch %d, training error %f ' % (cfg.lrate.epoch, 100*numpy.mean(train_error)) + '(%)') # validation valid_error = validate_by_minibatch(valid_fn, cfg) log('> epoch %d, lrate %f, validation error %f ' % (cfg.lrate.epoch, cfg.lrate.get_rate(), 100*numpy.mean(valid_error)) + '(%)') cfg.lrate.get_next_rate(current_error = 100*numpy.mean(valid_error)) # output nnet parameters and lrate, for training resume if cfg.lrate.epoch % cfg.model_save_step == 0: _nnet2file(dnn.layers, filename=wdir + '/nnet.tmp') _lrate2file(cfg.lrate, wdir + '/training_state.tmp') # save the model and network configuration if cfg.param_output_file != '': _nnet2file(dnn.layers, filename=cfg.param_output_file, input_factor = cfg.input_dropout_factor, factor = cfg.dropout_factor) log('> ... the final PDNN model parameter is ' + cfg.param_output_file) if cfg.cfg_output_file != '': _cfg2file(dnn.cfg, filename=cfg.cfg_output_file) log('> ... the final PDNN model config is ' + cfg.cfg_output_file) # output the model into Kaldi-compatible format if cfg.kaldi_output_file != '': dnn.write_model_to_kaldi(cfg.kaldi_output_file) log('> ... the final Kaldi model is ' + cfg.kaldi_output_file) # remove the tmp files (which have been generated from resuming training) os.remove(wdir + '/nnet.tmp') os.remove(wdir + '/training_state.tmp')