def __init__( self, idim, enc_arch, input_layer="linear", repeat_block=0, self_attn_type="selfattn", positional_encoding_type="abs_pos", positionwise_layer_type="linear", positionwise_activation_type="relu", conv_mod_activation_type="relu", normalize_before=True, padding_idx=-1, ): """Construct an Transformer encoder object.""" super(Encoder, self).__init__() self.embed, self.encoders, self.enc_out = build_blocks( "encoder", idim, input_layer, enc_arch, repeat_block=repeat_block, self_attn_type=self_attn_type, positional_encoding_type=positional_encoding_type, positionwise_layer_type=positionwise_layer_type, positionwise_activation_type=positionwise_activation_type, conv_mod_activation_type=conv_mod_activation_type, padding_idx=padding_idx, ) self.normalize_before = normalize_before if self.normalize_before: self.after_norm = LayerNorm(self.enc_out)
def test_invalid_input_layer_type(): architecture = [ { "type": "transformer", "d_hidden": 2, "d_ff": 2, "heads": 1, }, ] with pytest.raises(NotImplementedError): _, _, _ = build_blocks("encoder", 4, "foo", architecture)
def test_invalid_block_arguments(): with pytest.raises(ValueError): _, _, _ = build_blocks("encoder", 4, "linear", [{ "type": "transformer" }]) with pytest.raises(ValueError): _, _, _ = build_blocks("encoder", 4, "linear", [{"type": "conformer"}]) with pytest.raises(ValueError): _, _, _ = build_blocks( "encoder", 4, "linear", [{ "type": "conformer", "d_hidden": 4, "d_ff": 8, "heads": 1, "macaron_style": False, "use_conv_mod": True, }], ) with pytest.raises(ValueError): _, _, _ = build_blocks("decoder", 4, "embed", [{"type": "conformer"}]) with pytest.raises(ValueError): _, _, _ = build_blocks("encoder", 4, "linear", [{"type": "tdnn"}]) with pytest.raises(ValueError): _, _, _ = build_blocks("decoder", 4, "embed", [{ "type": "causal-conv1d" }]) with pytest.raises(ValueError): _, _, _ = build_blocks( "encoder", 4, "embed", [ { "type": "transformer", "d_hidden": 2, "d_ff": 8, "heads": 1, }, ], positional_encoding_type="rel_pos", self_attn_type="self_attn", )
def __init__( self, idim: int, enc_arch: List, input_layer: str = "linear", repeat_block: int = 1, self_attn_type: str = "selfattn", positional_encoding_type: str = "abs_pos", positionwise_layer_type: str = "linear", positionwise_activation_type: str = "relu", conv_mod_activation_type: str = "relu", aux_enc_output_layers: List = [], input_layer_dropout_rate: float = 0.0, input_layer_pos_enc_dropout_rate: float = 0.0, padding_idx: int = -1, ): """Construct an CustomEncoder object.""" super().__init__() ( self.embed, self.encoders, self.enc_out, self.conv_subsampling_factor, ) = build_blocks( "encoder", idim, input_layer, enc_arch, repeat_block=repeat_block, self_attn_type=self_attn_type, positional_encoding_type=positional_encoding_type, positionwise_layer_type=positionwise_layer_type, positionwise_activation_type=positionwise_activation_type, conv_mod_activation_type=conv_mod_activation_type, input_layer_dropout_rate=input_layer_dropout_rate, input_layer_pos_enc_dropout_rate=input_layer_pos_enc_dropout_rate, padding_idx=padding_idx, ) self.after_norm = LayerNorm(self.enc_out) self.n_blocks = len(enc_arch) * repeat_block self.aux_enc_output_layers = aux_enc_output_layers
def __init__( self, idim, enc_arch, input_layer="linear", repeat_block=0, self_attn_type="selfattn", positional_encoding_type="abs_pos", positionwise_layer_type="linear", positionwise_activation_type="relu", conv_mod_activation_type="relu", normalize_before=True, aux_task_layer_list=[], padding_idx=-1, ): """Construct an CustomEncoder object.""" super().__init__() ( self.embed, self.encoders, self.enc_out, self.conv_subsampling_factor, ) = build_blocks( "encoder", idim, input_layer, enc_arch, repeat_block=repeat_block, self_attn_type=self_attn_type, positional_encoding_type=positional_encoding_type, positionwise_layer_type=positionwise_layer_type, positionwise_activation_type=positionwise_activation_type, conv_mod_activation_type=conv_mod_activation_type, padding_idx=padding_idx, ) self.normalize_before = normalize_before if self.normalize_before: self.after_norm = LayerNorm(self.enc_out) self.n_blocks = len(enc_arch) * repeat_block self.aux_task_layer_list = aux_task_layer_list
def __init__( self, odim, edim, jdim, dec_arch, input_layer="embed", repeat_block=0, joint_activation_type="tanh", positional_encoding_type="abs_pos", positionwise_layer_type="linear", positionwise_activation_type="relu", dropout_rate_embed=0.0, blank=0, ): """Construct a Decoder object for transformer-transducer models.""" torch.nn.Module.__init__(self) self.embed, self.decoders, ddim = build_blocks( "decoder", odim, input_layer, dec_arch, repeat_block=repeat_block, positional_encoding_type=positional_encoding_type, positionwise_layer_type=positionwise_layer_type, positionwise_activation_type=positionwise_activation_type, dropout_rate_embed=dropout_rate_embed, padding_idx=blank, ) self.after_norm = LayerNorm(ddim) self.lin_enc = torch.nn.Linear(edim, jdim) self.lin_dec = torch.nn.Linear(ddim, jdim, bias=False) self.lin_out = torch.nn.Linear(jdim, odim) self.joint_activation = get_activation(joint_activation_type) self.dunits = ddim self.odim = odim self.blank = blank
def test_invalid_block_io(): with pytest.raises(ValueError): _, _, _ = build_blocks( "encoder", 4, "linear", [ { "type": "transformer", "d_hidden": 2, "d_ff": 8, "heads": 1, }, { "type": "transformer", "d_hidden": 4, "d_ff": 8, "heads": 1, }, ], )
def __init__( self, odim, edim, jdim, dec_arch, input_layer="embed", repeat_block=0, joint_activation_type="tanh", positional_encoding_type="abs_pos", positionwise_layer_type="linear", positionwise_activation_type="relu", dropout_rate_embed=0.0, blank=0, ): """Construct a CustomDecoder object.""" torch.nn.Module.__init__(self) self.embed, self.decoders, ddim, _ = build_blocks( "decoder", odim, input_layer, dec_arch, repeat_block=repeat_block, positional_encoding_type=positional_encoding_type, positionwise_layer_type=positionwise_layer_type, positionwise_activation_type=positionwise_activation_type, dropout_rate_embed=dropout_rate_embed, padding_idx=blank, ) self.after_norm = LayerNorm(ddim) self.joint_network = JointNetwork(odim, edim, ddim, jdim, joint_activation_type) self.dunits = ddim self.odim = odim self.blank = blank
def __init__( self, odim: int, dec_arch: List, input_layer: str = "embed", repeat_block: int = 0, joint_activation_type: str = "tanh", positional_encoding_type: str = "abs_pos", positionwise_layer_type: str = "linear", positionwise_activation_type: str = "relu", input_layer_dropout_rate: float = 0.0, blank_id: int = 0, ): """Construct a CustomDecoder object.""" torch.nn.Module.__init__(self) self.embed, self.decoders, ddim, _ = build_blocks( "decoder", odim, input_layer, dec_arch, repeat_block=repeat_block, positional_encoding_type=positional_encoding_type, positionwise_layer_type=positionwise_layer_type, positionwise_activation_type=positionwise_activation_type, input_layer_dropout_rate=input_layer_dropout_rate, padding_idx=blank_id, ) self.after_norm = LayerNorm(ddim) self.dlayers = len(self.decoders) self.dunits = ddim self.odim = odim self.blank_id = blank_id
def test_invalid_block(): with pytest.raises(ValueError): _, _, _ = build_blocks("encoder", 4, "linear", [{"foo": "foo"}])
def test_invalid_architecture_layer_type(): with pytest.raises(NotImplementedError): _, _, _ = build_blocks("encoder", 4, "linear", [{"type": "foo"}])