def __init__(
self,
idim,
enc_arch,
input_layer="linear",
repeat_block=0,
self_attn_type="selfattn",
positional_encoding_type="abs_pos",
positionwise_layer_type="linear",
positionwise_activation_type="relu",
conv_mod_activation_type="relu",
normalize_before=True,
padding_idx=-1,
):
"""Construct an Transformer encoder object."""
super(Encoder, self).__init__()
self.embed, self.encoders, self.enc_out = build_blocks(
"encoder",
idim,
input_layer,
enc_arch,
repeat_block=repeat_block,
self_attn_type=self_attn_type,
positional_encoding_type=positional_encoding_type,
positionwise_layer_type=positionwise_layer_type,
positionwise_activation_type=positionwise_activation_type,
conv_mod_activation_type=conv_mod_activation_type,
padding_idx=padding_idx,
)
self.normalize_before = normalize_before
if self.normalize_before:
self.after_norm = LayerNorm(self.enc_out)
def test_invalid_input_layer_type():
architecture = [
{
"type": "transformer",
"d_hidden": 2,
"d_ff": 2,
"heads": 1,
},
]
with pytest.raises(NotImplementedError):
_, _, _ = build_blocks("encoder", 4, "foo", architecture)
def test_invalid_block_arguments():
with pytest.raises(ValueError):
_, _, _ = build_blocks("encoder", 4, "linear", [{
"type": "transformer"
}])
with pytest.raises(ValueError):
_, _, _ = build_blocks("encoder", 4, "linear", [{"type": "conformer"}])
with pytest.raises(ValueError):
_, _, _ = build_blocks(
"encoder",
4,
"linear",
[{
"type": "conformer",
"d_hidden": 4,
"d_ff": 8,
"heads": 1,
"macaron_style": False,
"use_conv_mod": True,
}],
)
with pytest.raises(ValueError):
_, _, _ = build_blocks("decoder", 4, "embed", [{"type": "conformer"}])
with pytest.raises(ValueError):
_, _, _ = build_blocks("encoder", 4, "linear", [{"type": "tdnn"}])
with pytest.raises(ValueError):
_, _, _ = build_blocks("decoder", 4, "embed", [{
"type": "causal-conv1d"
}])
with pytest.raises(ValueError):
_, _, _ = build_blocks(
"encoder",
4,
"embed",
[
{
"type": "transformer",
"d_hidden": 2,
"d_ff": 8,
"heads": 1,
},
],
positional_encoding_type="rel_pos",
self_attn_type="self_attn",
)
def __init__(
self,
idim: int,
enc_arch: List,
input_layer: str = "linear",
repeat_block: int = 1,
self_attn_type: str = "selfattn",
positional_encoding_type: str = "abs_pos",
positionwise_layer_type: str = "linear",
positionwise_activation_type: str = "relu",
conv_mod_activation_type: str = "relu",
aux_enc_output_layers: List = [],
input_layer_dropout_rate: float = 0.0,
input_layer_pos_enc_dropout_rate: float = 0.0,
padding_idx: int = -1,
):
"""Construct an CustomEncoder object."""
super().__init__()
(
self.embed,
self.encoders,
self.enc_out,
self.conv_subsampling_factor,
) = build_blocks(
"encoder",
idim,
input_layer,
enc_arch,
repeat_block=repeat_block,
self_attn_type=self_attn_type,
positional_encoding_type=positional_encoding_type,
positionwise_layer_type=positionwise_layer_type,
positionwise_activation_type=positionwise_activation_type,
conv_mod_activation_type=conv_mod_activation_type,
input_layer_dropout_rate=input_layer_dropout_rate,
input_layer_pos_enc_dropout_rate=input_layer_pos_enc_dropout_rate,
padding_idx=padding_idx,
)
self.after_norm = LayerNorm(self.enc_out)
self.n_blocks = len(enc_arch) * repeat_block
self.aux_enc_output_layers = aux_enc_output_layers
def __init__(
self,
idim,
enc_arch,
input_layer="linear",
repeat_block=0,
self_attn_type="selfattn",
positional_encoding_type="abs_pos",
positionwise_layer_type="linear",
positionwise_activation_type="relu",
conv_mod_activation_type="relu",
normalize_before=True,
aux_task_layer_list=[],
padding_idx=-1,
):
"""Construct an CustomEncoder object."""
super().__init__()
(
self.embed,
self.encoders,
self.enc_out,
self.conv_subsampling_factor,
) = build_blocks(
"encoder",
idim,
input_layer,
enc_arch,
repeat_block=repeat_block,
self_attn_type=self_attn_type,
positional_encoding_type=positional_encoding_type,
positionwise_layer_type=positionwise_layer_type,
positionwise_activation_type=positionwise_activation_type,
conv_mod_activation_type=conv_mod_activation_type,
padding_idx=padding_idx,
)
self.normalize_before = normalize_before
if self.normalize_before:
self.after_norm = LayerNorm(self.enc_out)
self.n_blocks = len(enc_arch) * repeat_block
self.aux_task_layer_list = aux_task_layer_list
def __init__(
self,
odim,
edim,
jdim,
dec_arch,
input_layer="embed",
repeat_block=0,
joint_activation_type="tanh",
positional_encoding_type="abs_pos",
positionwise_layer_type="linear",
positionwise_activation_type="relu",
dropout_rate_embed=0.0,
blank=0,
):
"""Construct a Decoder object for transformer-transducer models."""
torch.nn.Module.__init__(self)
self.embed, self.decoders, ddim = build_blocks(
"decoder",
odim,
input_layer,
dec_arch,
repeat_block=repeat_block,
positional_encoding_type=positional_encoding_type,
positionwise_layer_type=positionwise_layer_type,
positionwise_activation_type=positionwise_activation_type,
dropout_rate_embed=dropout_rate_embed,
padding_idx=blank,
)
self.after_norm = LayerNorm(ddim)
self.lin_enc = torch.nn.Linear(edim, jdim)
self.lin_dec = torch.nn.Linear(ddim, jdim, bias=False)
self.lin_out = torch.nn.Linear(jdim, odim)
self.joint_activation = get_activation(joint_activation_type)
self.dunits = ddim
self.odim = odim
self.blank = blank
def test_invalid_block_io():
with pytest.raises(ValueError):
_, _, _ = build_blocks(
"encoder",
4,
"linear",
[
{
"type": "transformer",
"d_hidden": 2,
"d_ff": 8,
"heads": 1,
},
{
"type": "transformer",
"d_hidden": 4,
"d_ff": 8,
"heads": 1,
},
],
)
def __init__(
self,
odim,
edim,
jdim,
dec_arch,
input_layer="embed",
repeat_block=0,
joint_activation_type="tanh",
positional_encoding_type="abs_pos",
positionwise_layer_type="linear",
positionwise_activation_type="relu",
dropout_rate_embed=0.0,
blank=0,
):
"""Construct a CustomDecoder object."""
torch.nn.Module.__init__(self)
self.embed, self.decoders, ddim, _ = build_blocks(
"decoder",
odim,
input_layer,
dec_arch,
repeat_block=repeat_block,
positional_encoding_type=positional_encoding_type,
positionwise_layer_type=positionwise_layer_type,
positionwise_activation_type=positionwise_activation_type,
dropout_rate_embed=dropout_rate_embed,
padding_idx=blank,
)
self.after_norm = LayerNorm(ddim)
self.joint_network = JointNetwork(odim, edim, ddim, jdim, joint_activation_type)
self.dunits = ddim
self.odim = odim
self.blank = blank
def __init__(
self,
odim: int,
dec_arch: List,
input_layer: str = "embed",
repeat_block: int = 0,
joint_activation_type: str = "tanh",
positional_encoding_type: str = "abs_pos",
positionwise_layer_type: str = "linear",
positionwise_activation_type: str = "relu",
input_layer_dropout_rate: float = 0.0,
blank_id: int = 0,
):
"""Construct a CustomDecoder object."""
torch.nn.Module.__init__(self)
self.embed, self.decoders, ddim, _ = build_blocks(
"decoder",
odim,
input_layer,
dec_arch,
repeat_block=repeat_block,
positional_encoding_type=positional_encoding_type,
positionwise_layer_type=positionwise_layer_type,
positionwise_activation_type=positionwise_activation_type,
input_layer_dropout_rate=input_layer_dropout_rate,
padding_idx=blank_id,
)
self.after_norm = LayerNorm(ddim)
self.dlayers = len(self.decoders)
self.dunits = ddim
self.odim = odim
self.blank_id = blank_id
def test_invalid_block():
with pytest.raises(ValueError):
_, _, _ = build_blocks("encoder", 4, "linear", [{"foo": "foo"}])
def test_invalid_architecture_layer_type():
with pytest.raises(NotImplementedError):
_, _, _ = build_blocks("encoder", 4, "linear", [{"type": "foo"}])