def encode(self, xs, task='all', flip=False, use_cache=False, streaming=False): """Encode acoustic or text features. Args: xs (list): A list of length `[B]`, which contains Tensor of size `[T, input_dim]` task (str): all/ys*/ys_sub1*/ys_sub2* flip (bool): if True, flip acoustic features in the time-dimension use_cache (bool): use the cached forward encoder state in the previous chunk as the initial state streaming (bool): streaming encoding Returns: eout_dict (dict): """ if self.input_type == 'speech': # Frame stacking if self.n_stacks > 1: xs = [stack_frame(x, self.n_stacks, self.n_skips) for x in xs] # Splicing if self.n_splices > 1: xs = [splice(x, self.n_splices, self.n_stacks) for x in xs] xlens = torch.IntTensor([len(x) for x in xs]) # Flip acoustic features in the reverse order if flip: xs = [torch.from_numpy(np.flip(x, axis=0).copy()).float().cuda(self.device_id) for x in xs] else: xs = [np2tensor(x, self.device_id).float() for x in xs] xs = pad_list(xs, 0.) # SpecAugment if self.use_specaug and self.training: xs = self.specaug(xs) # Gaussian noise injection if self.gaussian_noise: xs = add_gaussian_noise(xs) # Sequence summary network if self.ssn is not None: xs += self.ssn(xs, xlens) elif self.input_type == 'text': xlens = torch.IntTensor([len(x) for x in xs]) xs = [np2tensor(np.fromiter(x, dtype=np.int64), self.device_id) for x in xs] xs = pad_list(xs, self.pad) xs = self.dropout_emb(self.embed(xs)) # TODO(hirofumi): fix for Transformer # encoder eout_dict = self.enc(xs, xlens, task.split('.')[0], use_cache, streaming) if self.main_weight < 1 and self.enc_type in ['conv', 'tds', 'gated_conv', 'transformer', 'conv_transformer']: for sub in ['sub1', 'sub2']: eout_dict['ys_' + sub]['xs'] = eout_dict['ys']['xs'].clone() eout_dict['ys_' + sub]['xlens'] = eout_dict['ys']['xlens'][:] return eout_dict
def encode(self, xs, task='all', flip=False): """Encode acoustic or text features. Args: xs (list): A list of length `[B]`, which contains Tensor of size `[T, input_dim]` task (str): all or ys* or ys_sub1* or ys_sub2* flip (bool): if True, flip acoustic features in the time-dimension Returns: enc_outs (dict): """ if self.input_type == 'speech': xlens = torch.IntTensor([len(x) for x in xs]) # Flip acoustic features in the reverse order if flip: xs = [torch.from_numpy(np.flip(x, axis=0).copy()).float().cuda(self.device_id) for x in xs] else: xs = [np2tensor(x, self.device_id).float() for x in xs] xs = pad_list(xs, 0.0) # SpecAugment if self.is_specaug and self.training: xs = self.specaug(xs) # Gaussian noise injection if self.gaussian_noise: xs = add_gaussian_noise(xs) # Sequence summary network if self.ssn is not None: xs += self.ssn(xs, xlens) elif self.input_type == 'text': xlens = torch.IntTensor([len(x) for x in xs]) xs = [np2tensor(np.fromiter(x, dtype=np.int64), self.device_id) for x in xs] xs = pad_list(xs, self.pad) xs = self.embed(xs) # encoder enc_outs = self.enc(xs, xlens, task.split('.')[0]) if self.main_weight < 1 and self.enc_type in ['conv', 'tds', 'gated_conv', 'transformer', 'conv_transformer']: for sub in ['sub1', 'sub2']: enc_outs['ys_' + sub]['xs'] = enc_outs['ys']['xs'].clone() enc_outs['ys_' + sub]['xlens'] = enc_outs['ys']['xlens'][:] del xs return enc_outs