def reset_parameters(self, args): """Initialize parameters.""" # initialize parameters initialize(self, args.transformer_init) if self.mt_weight > 0: torch.nn.init.normal_(self.encoder_mt.embed[0].weight, mean=0, std=args.adim ** -0.5) torch.nn.init.constant_(self.encoder_mt.embed[0].weight[self.pad], 0)
def _reset_parameters(self, init_type, init_enc_alpha=1.0, init_dec_alpha=1.0): # initialize parameters initialize(self, init_type) # initialize alpha in scaled positional encoding if self.use_scaled_pos_enc: self.encoder.embed[-1].alpha.data = torch.tensor(init_enc_alpha) self.decoder.embed[-1].alpha.data = torch.tensor(init_dec_alpha)
def _reset_parameters(self, args): # initialize parameters initialize(self, args.transformer_init) # initialize alpha in scaled positional encoding if self.use_scaled_pos_enc: self.encoder.embed[-1].alpha.data = torch.tensor( args.initial_encoder_alpha) self.decoder.embed[-1].alpha.data = torch.tensor( args.initial_decoder_alpha)
def reset_parameters(self, args): """Initialize parameters.""" if args.pretrained_model: path = args.pretrained_model logging.warning("load pretrained asr model from {}".format(path)) if 'snapshot' in path: model_state_dict = torch.load(path, map_location=lambda storage, loc: storage)['model'] else: model_state_dict = torch.load(path, map_location=lambda storage, loc: storage) self.load_state_dict(model_state_dict, strict=False) del model_state_dict else: initialize(self, args.transformer_init)
def reset_parameters(self, args): """Initialize parameters.""" # initialize parameters if args.pretrained_ctc_model: path = args.pretrained_ctc_model logging.warning("load pretrained asr model from {}".format(path)) if 'snapshot' in path: model_state_dict = torch.load( path, map_location=lambda storage, loc: storage)['model'] else: model_state_dict = torch.load( path, map_location=lambda storage, loc: storage) # In lid-classifier-mode, encoder+lid_lo params is utilized, # thus we do not remove any keys in model_state_dict and simply set strict=False self.load_state_dict(model_state_dict, strict=False) # logging.warning("load state dict params:") # logging.warning(model_state_dict.keys()) del model_state_dict else: initialize(self, args.transformer_init)
def initializer(model, args): """Initialize transducer model. Args: model (torch.nn.Module): transducer instance args (Namespace): argument Namespace containing options """ if args.dtype != 'transformer': if args.etype == 'transformer': initialize(model.encoder, args.transformer_init) lecun_normal_init_parameters(model.dec) else: lecun_normal_init_parameters(model) model.dec.embed.weight.data.normal_(0, 1) for l in six.moves.range(len(model.dec.decoder)): set_forget_bias_to_one(model.dec.decoder[l].bias_ih) else: if args.etype == 'transformer': initialize(model, args.transformer_init) else: lecun_normal_init_parameters(model.encoder) initialize(model.decoder, args.transformer_init)
def initializer(model, args): """Initialize transducer model. Args: model (torch.nn.Module): transducer instance args (Namespace): argument Namespace containing options """ if "custom" not in args.dtype: if "custom" in args.etype: initialize(model.encoder, args.transformer_init) lecun_normal_init_parameters(model.dec) else: lecun_normal_init_parameters(model) model.dec.embed.weight.data.normal_(0, 1) for i in range(model.dec.dlayers): set_forget_bias_to_one(getattr(model.dec.decoder[i], "bias_ih_l0")) set_forget_bias_to_one(getattr(model.dec.decoder[i], "bias_hh_l0")) else: if "custom" in args.etype: initialize(model, args.transformer_init) else: lecun_normal_init_parameters(model.enc) initialize(model.decoder, args.transformer_init)
def reset_parameters(self, args): """Initialize parameters.""" # load state_dict, and keeps only encoder part # note that self.ctc.ctc_lo is also removed # prefix is added to meet the needs of moe structure def load_state_dict_encoder(path, prefix=''): if 'snapshot' in path: model_state_dict = torch.load( path, map_location=lambda storage, loc: storage)['model'] else: model_state_dict = torch.load( path, map_location=lambda storage, loc: storage) for k in list(model_state_dict.keys()): if not 'encoder' in k: # remove this key del model_state_dict[k] else: new_k = k.replace('encoder.', prefix + 'encoder.') model_state_dict[new_k] = model_state_dict.pop(k) return model_state_dict # initialize parameters if args.pretrained_cn_ctc_model and args.pretrained_en_ctc_model: logging.warning( "loading pretrained ctc model for parallel encoder") # still need to initialize the 'other' params initialize(self, args.transformer_init) cn_state_dict = load_state_dict_encoder( args.pretrained_cn_ctc_model, prefix='cn_') self.load_state_dict(cn_state_dict, strict=False) del cn_state_dict en_state_dict = load_state_dict_encoder( args.pretrained_en_ctc_model, prefix='en_') self.load_state_dict(en_state_dict, strict=False) del en_state_dict else: initialize(self, args.transformer_init)
def initializer(model, args): """Initialize transducer model. Args: model (torch.nn.Module): transducer instance args (Namespace): argument Namespace containing options """ if args.dtype != 'transformer': if args.etype == 'transformer': initialize(model.encoder, args.transformer_init) lecun_normal_init_parameters(model.decoder) else: lecun_normal_init_parameters(model) model.decoder.embed.weight.data.normal_(0, 1) else: if args.etype == 'transformer': initialize(model, args.transformer_init) else: lecun_normal_init_parameters(model.encoder) initialize(model.decoder, args.transformer_init)
def reset_parameters(self, args): """Initialize parameters.""" # initialize parameters initialize(self, args.transformer_init)
def __init__(self, num_time_mask=2, num_freq_mask=2, freq_mask_length=15, time_mask_length=15, feature_dim=320, model_size=512, feed_forward_size=1024, hidden_size=64, dropout=0.1, num_head=8, num_encoder_layer=6, num_decoder_layer=6, vocab_path='testing_vocab.model', max_feature_length=1024, max_token_length=50, enable_spec_augment=True, share_weight=True, smoothing=0.1, restrict_left_length=20, restrict_right_length=20, mtlalpha=0.2, report_wer=True): super(Transformer, self).__init__() self.enable_spec_augment = enable_spec_augment self.max_token_length = max_token_length self.restrict_left_length = restrict_left_length self.restrict_right_length = restrict_right_length self.vocab = Vocab(vocab_path) self.sos = self.vocab.bos_id self.eos = self.vocab.eos_id self.adim = model_size self.odim = self.vocab.vocab_size self.ignore_id = self.vocab.pad_id if enable_spec_augment: self.spec_augment = SpecAugment( num_time_mask=num_time_mask, num_freq_mask=num_freq_mask, freq_mask_length=freq_mask_length, time_mask_length=time_mask_length, max_sequence_length=max_feature_length) self.encoder = Encoder(idim=feature_dim, attention_dim=model_size, attention_heads=num_head, linear_units=feed_forward_size, num_blocks=num_encoder_layer, dropout_rate=dropout, positional_dropout_rate=dropout, attention_dropout_rate=dropout, input_layer='linear', padding_idx=self.vocab.pad_id) self.decoder = Decoder(odim=self.vocab.vocab_size, attention_dim=model_size, attention_heads=num_head, linear_units=feed_forward_size, num_blocks=num_decoder_layer, dropout_rate=dropout, positional_dropout_rate=dropout, self_attention_dropout_rate=dropout, src_attention_dropout_rate=0, input_layer='embed', use_output_layer=False) self.decoder_linear = t.nn.Linear(model_size, self.vocab.vocab_size, bias=True) self.decoder_switch_linear = t.nn.Linear(model_size, 4, bias=True) self.criterion = LabelSmoothingLoss(size=self.odim, smoothing=smoothing, padding_idx=self.vocab.pad_id, normalize_length=True) self.switch_criterion = LabelSmoothingLoss( size=4, smoothing=0, padding_idx=self.vocab.pad_id, normalize_length=True) self.mtlalpha = mtlalpha if mtlalpha > 0.0: self.ctc = CTC(self.odim, eprojs=self.adim, dropout_rate=dropout, ctc_type='builtin', reduce=False) else: self.ctc = None if report_wer: from espnet.nets.e2e_asr_common import ErrorCalculator def load_token_list(path=vocab_path.replace('.model', '.vocab')): with open(path) as reader: data = reader.readlines() data = [i.split('\t')[0] for i in data] return data self.char_list = load_token_list() self.error_calculator = ErrorCalculator( char_list=self.char_list, sym_space=' ', sym_blank=self.vocab.blank_token, report_wer=True) else: self.error_calculator = None self.rnnlm = None self.reporter = Reporter() self.switch_loss = LabelSmoothingLoss(size=4, smoothing=0, padding_idx=0) print('initing') initialize(self, init_type='xavier_normal') print('inited')