def __init__(self, opts, word2syllable, pretrained_emb_weights):
super(ModelBase, self).__init__()
self.opts = opts
self.PAD_token = opts.PAD_token
self.SOS_token = opts.SOS_token
self.EOS_token = opts.EOS_token
self.UNK_token = opts.UNK_token
self.SEP_token = opts.SEP_token
self.embedding = _build_embeddings(opts, word2syllable,
pretrained_emb_weights)
self.latent_module = VADLatentModule(opts.dec_hidden_size,
opts.latent_dim,
opts.latent_use_tanh)
if opts.use_bow_loss:
self.bow_proj_layer = nn.Linear(
opts.latent_dim + opts.dec_hidden_size, opts.vocab_size)
self.emb2states_fwd = nn.Linear(
opts.emb_out_dim, opts.dec_num_layers * opts.dec_hidden_size)
self.emb2states_bwd = nn.Linear(
opts.emb_out_dim, opts.dec_num_layers * opts.dec_hidden_size)
self.fwd_decoder = StackedGRUCell(opts.emb_out_dim + opts.latent_dim,
opts.dec_hidden_size,
opts.dec_num_layers, opts.dropout,
opts.use_layer_norm,
opts.layer_norm_trainable)
self.bwd_decoder = StackedGRUCell(opts.emb_out_dim,
opts.dec_hidden_size,
opts.dec_num_layers, opts.dropout,
opts.use_layer_norm,
opts.layer_norm_trainable)
out_dim = opts.dec_hidden_size
if opts.latent_out_attach:
out_dim += opts.latent_dim
self.fwd_out_proj = nn.Linear(out_dim, opts.vocab_size, bias=False)
if opts.need_bwd_out_proj_layer:
self.bwd_out_proj = nn.Linear(opts.dec_hidden_size,
opts.vocab_size,
bias=False)
if opts.fwd_use_logits_mask:
self.fwd_logits_mask_layer = LogitsMaskLayer(
self.embedding.get_word2syllable_buffer(), opts.SEP_token,
opts.UNK_token)
if opts.bwd_use_logits_mask:
self.bwd_logits_mask_layer = LogitsMaskLayer(
self.embedding.get_word2syllable_buffer(), opts.SEP_token,
opts.UNK_token)
self._reset_parameters()
def __init__(self, opts, word2syllable, pretrained_emb_weights):
super(TransformerCLM, self).__init__()
self.opts = opts
self.PAD_token = opts.PAD_token
self.SOS_token = opts.SOS_token
self.EOS_token = opts.EOS_token
self.UNK_token = opts.UNK_token
self.SEP_token = opts.SEP_token
self.KEYWORD_token = opts.KEYWORD_token
self.keyword_approaches = opts.keyword_approaches
self.embedding = _build_embeddings(opts, word2syllable, pretrained_emb_weights, use_pos_emb=True)
self.decoder = _build_stacked_transformer_layers(opts, opts.num_layers, opts.num_self_attn_per_layer)
self.hidden2emb, self.out_proj = _build_output_layers(
opts, self.embedding.get_word_emb_out_proj_weight(), self.embedding.get_word_emb_weight())
if opts.use_logits_mask:
self.logits_mask_layer = LogitsMaskLayer(
self.embedding.get_word2syllable_buffer(),
opts.SEP_token, opts.UNK_token, opts.KEYWORD_token)
self._reset_parameters()
def __init__(self, opts, word2syllable, pretrained_emb_weights):
super(SequentialTransformerCVAE, self).__init__()
self.opts = opts
self.PAD_token = opts.PAD_token
self.SOS_token = opts.SOS_token
self.EOS_token = opts.EOS_token
self.UNK_token = opts.UNK_token
self.SEP_token = opts.SEP_token
self.KEYWORD_token = opts.KEYWORD_token
self.keyword_approaches = opts.keyword_approaches
self.embedding = _build_embeddings(opts, word2syllable, pretrained_emb_weights, use_pos_emb=True)
self.prior_encoder = _build_stacked_transformer_layers(
opts,
opts.num_layers_before_latent,
opts.num_self_attn_per_layer_before_latent)
self.recognition_encoder = _build_stacked_transformer_layers(
opts,
opts.num_layers_before_latent,
opts.num_self_attn_per_layer_before_latent,
prebuilt_layers=self.prior_encoder)
self.latent_module = VTLatentModule(opts.d_model, opts.latent_dim, opts.latent_use_tanh)
self.drop = nn.Dropout(opts.dropout)
self.norm = nn.LayerNorm(opts.d_model)
if opts.use_bow_loss:
self.bow_proj_layer = nn.Linear(opts.latent_dim + opts.d_model, opts.vocab_size)
self.decoder = _build_stacked_transformer_layers(
opts,
opts.num_layers - opts.num_layers_before_latent,
opts.num_self_attn_per_layer_after_latent)
self.hidden2emb, self.out_proj = _build_output_layers(
opts, self.embedding.get_word_emb_out_proj_weight(), self.embedding.get_word_emb_weight())
if opts.use_logits_mask:
self.logits_mask_layer = LogitsMaskLayer(
self.embedding.get_word2syllable_buffer(),
opts.SEP_token, opts.UNK_token, opts.KEYWORD_token)
self._reset_parameters()
def __init__(self, opts, word2syllable, pretrained_emb_weights):
super(TransformerCVAE, self).__init__()
self.opts = opts
self.PAD_token = opts.PAD_token
self.SOS_token = opts.SOS_token
self.EOS_token = opts.EOS_token
self.UNK_token = opts.UNK_token
self.SEP_token = opts.SEP_token
self.CLS_token = opts.CLS_token
self.KEYWORD_token = opts.KEYWORD_token
self.keyword_approaches = opts.keyword_approaches
self.embedding = _build_embeddings(opts, word2syllable, pretrained_emb_weights, use_pos_emb=True)
self.encoder = _build_stacked_transformer_layers(
opts,
opts.num_layers_before_latent,
opts.num_self_attn_per_layer_before_latent)
self.latent_module = TLatentModule(
opts.d_model, opts.d_model, opts.latent_dim, opts.latent_use_tanh, "klatent" in opts.keyword_approaches)
if opts.use_bow_loss:
bow_inp_dim = opts.latent_dim
if "klatent" in opts.keyword_approaches:
bow_inp_dim += opts.d_model
self.bow_proj_layer = nn.Linear(bow_inp_dim, opts.vocab_size)
self.decoder = _build_stacked_transformer_layers(
opts,
opts.num_layers - opts.num_layers_before_latent,
opts.num_self_attn_per_layer_after_latent)
self.hidden2emb, self.out_proj = _build_output_layers(
opts, self.embedding.get_word_emb_out_proj_weight(), self.embedding.get_word_emb_weight())
if opts.use_logits_mask:
self.logits_mask_layer = LogitsMaskLayer(
self.embedding.get_word2syllable_buffer(),
opts.SEP_token, opts.UNK_token, opts.KEYWORD_token)
self._reset_parameters()