def __init__(self, vocab_src, vocab_tgt, embed_dim, hidden_size,
num_layers, dropout, input_feed):
super(ResponseGenerator, self).__init__()
self.vocab_src = vocab_src
self.vocab_tgt = vocab_tgt
self.encoder_src = LSTMencoder(vocab_src,
embed_dim=embed_dim,
hidden_size=hidden_size // 2,
num_layers=num_layers,
dropout_in=dropout,
dropout_out=dropout,
bidirectional=True,
pretrained_embed=None)
self.embed_tgt = Embedding(vocab_tgt.size, embed_dim,
vocab_tgt.padding_idx)
self.encoder_ske = LSTMencoder(vocab_tgt,
embed_dim=embed_dim,
hidden_size=hidden_size // 2,
num_layers=num_layers,
dropout_in=dropout,
dropout_out=dropout,
bidirectional=True,
pretrained_embed=self.embed_tgt)
self.decoder = LSTMdecoder(vocab_tgt,
embed_dim=embed_dim,
hidden_size=hidden_size,
num_layers=num_layers,
dropout_in=dropout,
dropout_out=dropout,
encoder_hidden_size=hidden_size,
pretrained_embed=self.embed_tgt,
input_feed=input_feed)
self.copy = Linear(hidden_size, 1)
self.generate = Linear(hidden_size, vocab_tgt.size)
def __init__(self, cfg):
super(PCNN, self).__init__()
self.use_pcnn = cfg.use_pcnn
if cfg.dim_strategy == 'cat':
cfg.in_channels = cfg.word_dim + 2 * cfg.pos_dim
else:
cfg.in_channels = cfg.word_dim
self.embedding = Embedding(cfg)
self.cnn = CNN(cfg)
self.fc1 = nn.Linear(
len(cfg.kernel_sizes) * cfg.out_channels, cfg.intermediate)
self.fc2 = nn.Linear(cfg.intermediate, cfg.num_relations)
self.dropout = nn.Dropout(cfg.dropout)
if self.use_pcnn:
self.fc_pcnn = nn.Linear(
3 * len(cfg.kernel_sizes) * cfg.out_channels,
len(cfg.kernel_sizes) * cfg.out_channels)
self.pcnn_mask_embedding = nn.Embedding(4, 3)
masks = torch.tensor([[0, 0, 0], [100, 0, 0], [0, 100, 0],
[0, 0, 100]])
self.pcnn_mask_embedding.weight.data.copy_(masks)
self.pcnn_mask_embedding.weight.requires_grad = False
def test_Embedding_sum():
config.dim_strategy = 'sum'
embed = Embedding(config)
feature = embed((x, x_pos))
dim = config.word_dim
assert feature.shape == torch.Size((3, 5, dim))
def __init__(self, cfg):
super(GCN, self).__init__()
if cfg.dim_strategy == 'cat':
cfg.input_size = cfg.word_dim + 2 * cfg.pos_dim
else:
cfg.input_size = cfg.word_dim
self.embedding = Embedding(cfg)
self.gcn = GCNBlock(cfg)
self.fc = nn.Linear(cfg.hidden_size, cfg.num_attributes)
def __init__(self, cfg):
super(Transformer, self).__init__()
if cfg.dim_strategy == 'cat':
cfg.hidden_size = cfg.word_dim + 2 * cfg.pos_dim
else:
cfg.hidden_size = cfg.word_dim
self.embedding = Embedding(cfg)
self.transformer = TransformerBlock(cfg)
self.fc = nn.Linear(cfg.hidden_size, cfg.num_relations)
def __init__(self, cfg):
super(BiLSTM, self).__init__()
if cfg.dim_strategy == 'cat':
cfg.input_size = cfg.word_dim + 2 * cfg.pos_dim
else:
cfg.input_size = cfg.word_dim
self.embedding = Embedding(cfg)
self.bilstm = RNN(cfg)
self.fc = nn.Linear(cfg.hidden_size, cfg.num_attributes)
self.dropout = nn.Dropout(cfg.dropout)
def __init__(self):
super().__init__()
self.embedding = Embedding(3, 128)
self.sa1 = SA(128)
self.sa2 = SA(128)
self.sa3 = SA(128)
self.sa4 = SA(128)
self.linear = nn.Sequential(
nn.Conv1d(512, 1024, kernel_size=1, bias=False),
nn.BatchNorm1d(1024),
nn.LeakyReLU(negative_slope=0.2)
)
def __init__(self,
space_dim=2,
embed_hidden=128,
enc_stacks=3,
ff_hidden=512,
enc_heads=16,
query_hidden=360,
att_hidden=256,
crit_hidden=256,
n_history=3,
p_dropout=0.1):
"""Agent, made of an encoder + decoder for the actor part, and a critic
part.
Args:
space_dim (int, optional): Dimension for the cities coordinates.
Defaults to 2.
embed_hidden (int, optional): Embeddings hidden size. Defaults to 128.
enc_stacks (int, optional): Number of encoder layers. Defaults to 3.
ff_hidden (int, optional): Hidden size for the FF part of the encoder.
Defaults to 512.
enc_heads (int, optional): Number of attention heads for the encoder.
Defaults to 16.
query_hidden (int, optional): Query hidden size. Defaults to 360.
att_hidden (int, optional): Attention hidden size. Defaults to 256.
crit_hidden (int, optional): Critic hidden size. Defaults to 256.
n_history (int, optional): Size of history (memory size of the
decoder). Defaults to 3.
p_dropout (float, optional): Dropout rate. Defaults to 0.1.
"""
super().__init__()
# Actor
self.embedding = Embedding(in_dim=space_dim, out_dim=embed_hidden)
self.encoder = Encoder(num_layers=enc_stacks,
n_hidden=embed_hidden,
ff_hidden=ff_hidden,
num_heads=enc_heads,
p_dropout=p_dropout)
self.decoder = Decoder(n_hidden=embed_hidden,
att_dim=att_hidden,
query_dim=query_hidden,
n_history=n_history)
# Critic
self.critic = Critic(n_hidden=embed_hidden,
att_hidden=att_hidden,
crit_hidden=crit_hidden)
def __init__(self, cfg):
super(Capsule, self).__init__()
if cfg.dim_strategy == 'cat':
cfg.in_channels = cfg.word_dim + 2 * cfg.pos_dim
else:
cfg.in_channels = cfg.word_dim
# capsule config
cfg.input_dim_capsule = cfg.out_channels
cfg.num_capsule = cfg.num_relations
self.num_relations = cfg.num_relations
self.embedding = Embedding(cfg)
self.cnn = CNN(cfg)
self.capsule = CapsuleLayer(cfg)
def __init__(self, vocab, embed_dim=512, hidden_size=512,
num_layers=1, dropout_in=0.1, dropout_out=0.1,
bidirectional = True, pretrained_embed = None):
super(LSTMencoder, self).__init__()
self.vocab = vocab
self.embed_dim = embed_dim
self.hidden_size = hidden_size
self.num_layers = num_layers
self.dropout_in = dropout_in
self.dropout_out = dropout_out
self.bidirectional = bidirectional
if pretrained_embed is not None:
self.embed_tokens = pretrained_embed
else:
self.embed_tokens = Embedding(vocab.size, embed_dim, vocab.padding_idx)
self.lstm = LSTM(
input_size=embed_dim,
hidden_size=hidden_size,
num_layers=num_layers,
dropout=self.dropout_out if num_layers > 1 else 0.,
bidirectional=bidirectional
)
def __init__(self,
vocab,
embed_dim=512,
hidden_size=512,
num_layers=1,
dropout_in=0.1,
dropout_out=0.1,
encoder_hidden_size=512,
pretrained_embed=None,
input_feed=False):
super(LSTMdecoder, self).__init__()
self.embed_dim = embed_dim
self.hidden_size = hidden_size
self.num_layers = num_layers
self.dropout_in = dropout_in
self.dropout_out = dropout_out
self.input_feed = input_feed
if pretrained_embed is not None:
self.embed_tokens = pretrained_embed
else:
self.embed_tokens = Embedding(vocab.size, embed_dim,
vocab.padding_idx)
self.lstm = LSTM(
input_size=embed_dim + (hidden_size if self.input_feed else 0),
hidden_size=hidden_size,
num_layers=num_layers,
dropout=self.dropout_out if num_layers > 1 else 0.,
)
self.attention_src = GeneralAttention(hidden_size, encoder_hidden_size,
encoder_hidden_size)
self.proj_with_src = Linear(encoder_hidden_size + hidden_size,
hidden_size)
self.attention_ske = GeneralAttention(hidden_size, hidden_size,
hidden_size)
self.gate_with_ske = Linear(2 * hidden_size, hidden_size)
def test_Embedding_cat():
embed = Embedding(config)
feature = embed((x, x_pos))
dim = config.word_dim + config.pos_dim
assert feature.shape == torch.Size((3, 5, dim))
def init_model(dim, normfac, classnums, embeddims, N, max_state_dur,
output_initial_bias, device):
incode_embedding = Embedding(classnums, embeddims, device)
model = Model(dim, normfac, incode_embedding, N, max_state_dur,
output_initial_bias, device)
return model
