def build_embedding(dictionary, embed_dim, path=None):
num_embeddings = len(dictionary)
padding_idx = dictionary.pad()
emb = Embedding(num_embeddings, embed_dim, padding_idx)
# if provided, load from preloaded dictionaries
if path:
embed_dict = utils.parse_embedding(path)
utils.load_embedding(embed_dict, dictionary, emb)
return emb
def build_embedding(dictionary, embed_dim, path=None):
num_embeddings = len(dictionary)
padding_idx = dictionary.pad()
emb = Embedding(num_embeddings, embed_dim, padding_idx)
# if provided, load from preloaded dictionaries
if path:
embed_dict = utils.parse_embedding(path)
utils.load_embedding(embed_dict, dictionary, emb)
return emb
def build_embedding(dictionary, embed_dim, path=None, freeze=False):
num_embeddings = len(dictionary)
padding_idx = dictionary.pad()
emb = TransformerTokenEmbedding(num_embeddings, embed_dim, padding_idx,
freeze)
# if provided, load from preloaded dictionaries
if path:
embed_dict = utils.parse_embedding(path)
utils.load_embedding(embed_dict, dictionary, emb)
return emb
def load_pretrained_embedding_from_file(embed_path, dictionary, embed_dim):
num_embedding = len(dictionary)
padding_idx = dictionary.pad()
embed_tokens = Embedding(num_embedding, embed_dim, padding_idx)
embed_dict = utils.parse_embedding(embed_path)
utils.print_embed_overlap(embed_dict, dictionary)
# embed_keys = set(embed_dict.keys())
# vocab_keys = set(dictionary.symbols))
# print(vocab_keys - embed_keys)
return utils.load_embedding(embed_dict, dictionary,
embed_tokens), embed_dict
def build_embedding(dictionary, embed_dim, args, mask_file=None, path=None):
num_embeddings = len(dictionary)
padding_idx = dictionary.pad()
if args.one_emb:
emb = OneEmbedding(num_embeddings, embed_dim, padding_idx, args.one_emb, args.one_emb_dropout, args.one_emb_std, args.codenum, args.codebooknum, args.one_emb_layernum, args.one_emb_inter_dim, args.one_emb_relu_dropout, mask_file)
else:
emb = Embedding(num_embeddings, embed_dim, padding_idx)
# if provided, load from preloaded dictionaries
if path:
embed_dict = utils.parse_embedding(path)
utils.load_embedding(embed_dict, dictionary, emb)
return emb
def load_pretrained_embedding_from_file(embed_path, dictionary,
embed_dim):
from fairseq import utils
num_embeddings = len(dictionary)
padding_idx = dictionary.pad()
embed_tokens = Embedding(num_embeddings, embed_dim,
padding_idx)
embed_dict = utils.parse_embedding(embed_path)
utils.print_embed_overlap(embed_dict, dictionary)
return utils.load_embedding(embed_dict, dictionary,
embed_tokens)
def build_embedding(dictionary, embed_dim, path=None, sde=False):
num_embeddings = len(dictionary)
padding_idx = dictionary.pad()
if sde:
emb = SDEembedding(char_vsize=num_embeddings, d_vec=embed_dim, padding_idx=padding_idx)
else:
emb = Embedding(num_embeddings, embed_dim, padding_idx, fix_norm=args.fix_norm)
# if provided, load from preloaded dictionaries
if path:
embed_dict = utils.parse_embedding(path)
utils.load_embedding(embed_dict, dictionary, emb)
return emb
def build_model(cls, args, src_dict, dst_dict):
"""Build a new model instance."""
if not hasattr(args, 'max_source_positions'):
args.max_source_positions = args.max_positions
args.max_target_positions = args.max_positions
if not hasattr(args, 'share_input_output_embed'):
args.share_input_output_embed = False
if not hasattr(args, 'encoder_embed_path'):
args.encoder_embed_path = None
if not hasattr(args, 'decoder_embed_path'):
args.decoder_embed_path = None
encoder_embed_dict = None
if args.encoder_embed_path:
encoder_embed_dict = utils.parse_embedding(args.encoder_embed_path)
utils.print_embed_overlap(encoder_embed_dict, src_dict)
decoder_embed_dict = None
if args.decoder_embed_path:
decoder_embed_dict = utils.parse_embedding(args.decoder_embed_path)
utils.print_embed_overlap(decoder_embed_dict, dst_dict)
encoder = FConvEncoder(
src_dict,
embed_dim=args.encoder_embed_dim,
embed_dict=encoder_embed_dict,
convolutions=eval(args.encoder_layers),
dropout=args.dropout,
max_positions=args.max_source_positions,
)
decoder = FConvDecoder(dst_dict,
embed_dim=args.decoder_embed_dim,
embed_dict=decoder_embed_dict,
convolutions=eval(args.decoder_layers),
out_embed_dim=args.decoder_out_embed_dim,
attention=eval(args.decoder_attention),
dropout=args.dropout,
max_positions=args.max_target_positions,
share_embed=args.share_input_output_embed)
return FConvModel(encoder, decoder)
def build_model(cls, args, task):
"""Build a new model instance."""
# make sure that all args are properly defaulted (in case there are any new ones)
base_architecture(args)
encoder_embed_dict = None
if args.encoder_embed_path:
encoder_embed_dict = utils.parse_embedding(args.encoder_embed_path)
utils.print_embed_overlap(encoder_embed_dict,
task.source_dictionary)
decoder_embed_dict = None
if args.decoder_embed_path:
decoder_embed_dict = utils.parse_embedding(args.decoder_embed_path)
utils.print_embed_overlap(decoder_embed_dict,
task.target_dictionary)
encoder = FConvEncoder(
dictionary=task.source_dictionary,
embed_dim=args.encoder_embed_dim,
embed_dict=encoder_embed_dict,
convolutions=eval(args.encoder_layers),
dropout=args.dropout,
max_positions=args.max_source_positions,
normalization_constant=args.normalization_constant,
)
decoder = FConvDecoder(
dictionary=task.target_dictionary,
embed_dim=args.decoder_embed_dim,
embed_dict=decoder_embed_dict,
convolutions=eval(args.decoder_layers),
out_embed_dim=args.decoder_out_embed_dim,
attention=eval(args.decoder_attention),
dropout=args.dropout,
max_positions=args.max_target_positions,
share_embed=args.share_input_output_embed,
normalization_constant=args.normalization_constant,
)
return FConvModel(encoder, decoder)
def copy_prev_embedding(embed_path, dictionary, embed_dim, prev_embedded_tokens_path, prev_dict): num_embeddings = len(dictionary) padding_idx = dictionary.pad() embed_tokens = nn.Embedding(num_embeddings, embed_dim, padding_idx) prev_embedded_tokens = load_random_embedding(prev_embedded_tokens_path) for i in range(5, num_embeddings): if prev_dict.index(dictionary.symbols[i])!= prev_dict.unk() and i!=dictionary.unk(): embed_tokens.weight.data[i] = prev_embedded_tokens[prev_dict.index(dictionary.symbols[i])] #embed_tokens.weight = nn.Parameter(prev_embedded_tokens) embed_dict = utils.parse_embedding(embed_path) utils.print_embed_overlap(embed_dict, dictionary) return utils.load_embedding(embed_dict, dictionary, embed_tokens)
def build_embedding(dictionary, embed_dim, path=None):
# construct and return an embedding layer;
# load pretrained embedding path if specified.
num_embeddings = len(dictionary)
padding_idx = dictionary.pad()
emb = Embedding(num_embeddings, embed_dim, padding_idx)
# if provided, load from preloaded dictionaries
if path:
embed_dict = utils.parse_embedding(path)
utils.load_embedding(embed_dict, dictionary, emb)
logging.info(
'Loaded pretrained embeddings from {}'.format(path))
return emb
def build_embedding(cls, args, dictionary, embed_dim, path=None):
num_embeddings = len(dictionary)
padding_idx = dictionary.pad()
emb = Embedding(num_embeddings, embed_dim, padding_idx)
# if provided, load from preloaded dictionaries
if path:
embed_dict = utils.parse_embedding(path)
utils.load_embedding(embed_dict, dictionary, emb)
'''for i in range(0,10):
print(dictionary[i])
print("********")'''
return emb
def build_model(cls, args, task):
"""Build a new model instance."""
# make sure that all args are properly defaulted (in case there are any new ones)
base_architecture(args)
attention_layer = {
'dot': DotAttentionLayer,
'general': GeneralAttentionLayer,
'multi-head': MultiheadAttention,
'mlp': MLPAttentionLayer,
}
decoder_embed_dict = None
if args.decoder_embed_path:
decoder_embed_dict = utils.parse_embedding(args.decoder_embed_path)
utils.print_embed_overlap(decoder_embed_dict,
task.target_dictionary)
# replaced source_dict with target_dict
encoder = ASTS2SEncoder(
args=args,
linear_dim=args.encoder_embed_dim,
convolutions=eval(args.encoder_convolutions),
layers=args.encoder_layers,
dropout=args.dropout,
max_positions=args.max_source_positions,
normalization_constant=args.normalization_constant,
last_state=args.encoder_state,
weight_norm=args.weight_norm,
learn_initial=args.learn_initial_state,
conv_1d=args.conv_1d,
audio_features=task.audio_features if not args.wav2vec else 512,
)
decoder = CLSTMDecoder(
dictionary=task.target_dictionary,
embed_dim=args.decoder_embed_dim,
out_embed_dim=args.decoder_out_embed_dim,
attention=eval(args.decoder_attention),
dropout_in=args.dropout,
dropout_out=args.dropout,
num_layers=args.decoder_layers,
attention_layer=attention_layer[args.attention_type],
initial_state=args.decoder_initial_state,
weight_norm=args.weight_norm,
attention_function=args.attention_function,
max_positions=args.max_target_positions,
scale_norm=args.scale_norm,
scale=args.scale,
)
return ASTS2SModel(encoder, decoder)
def build_embedding(dictionary, embed_dim, path=None, feat=False):
if feat:
emb = Embeddings([Embedding(len(vocab), embed_dim, vocab.pad())
for _, vocab in dictionary.items()])
else:
padding_idx = dictionary.pad()
num_embeddings = len(dictionary)
emb = Embedding(num_embeddings, embed_dim, padding_idx)
# if provided, load from preloaded dictionaries
if path:
embed_dict = utils.parse_embedding(path)
utils.load_embedding(embed_dict, dictionary, emb)
return emb
def build_model(cls, args, src_dict, dst_dict):
"""Build a new model instance."""
if not hasattr(args, 'encoder_embed_path'):
args.encoder_embed_path = None
if not hasattr(args, 'decoder_embed_path'):
args.decoder_embed_path = None
encoder_embed_dict = None
if args.encoder_embed_path:
encoder_embed_dict = utils.parse_embedding(args.encoder_embed_path)
utils.print_embed_overlap(encoder_embed_dict, src_dict)
decoder_embed_dict = None
if args.decoder_embed_path:
decoder_embed_dict = utils.parse_embedding(args.decoder_embed_path)
utils.print_embed_overlap(decoder_embed_dict, dst_dict)
encoder = LSTMEncoder(
src_dict,
embed_dim=args.encoder_embed_dim,
embed_dict=encoder_embed_dict,
num_layers=args.encoder_layers,
dropout_in=args.encoder_dropout_in,
dropout_out=args.encoder_dropout_out,
)
decoder = LSTMDecoder(
dst_dict,
encoder_embed_dim=args.encoder_embed_dim,
embed_dim=args.decoder_embed_dim,
embed_dict=decoder_embed_dict,
out_embed_dim=args.decoder_out_embed_dim,
num_layers=args.decoder_layers,
attention=bool(eval(args.decoder_attention)),
dropout_in=args.decoder_dropout_in,
dropout_out=args.decoder_dropout_out,
)
return cls(encoder, decoder)
def build_embedding(dictionary, embed_dim, path=None):
num_embeddings = len(dictionary)
padding_idx = dictionary.pad()
emb = Embedding(num_embeddings, embed_dim, padding_idx)
# if provided, load from preloaded dictionaries
if path:
embed_dict = utils.parse_embedding(path)
utils.load_embedding(embed_dict, dictionary, emb)
# if not path and args.disable_training_embeddings:
# raise ValueError('Do not set --disable_training_embeddings when pretrained embeddings are not provided.')
# if args.disable_training_embeddings:
# emb.weight.requires_grad = False
return emb
def build_embedding(dictionary, embed_dim, is_encoder, path=None):
if path is not None:
if path.startswith('elmo:'):
lm_path = path[5:]
task = LanguageModelingTask(args, dictionary, dictionary)
models, _ = utils.load_ensemble_for_inference(
[lm_path], task, {'remove_head': True})
assert len(
models
) == 1, 'ensembles are currently not supported for elmo embeddings'
embedder = ElmoTokenEmbedder(
models[0],
dictionary.eos(),
dictionary.pad(),
add_bos=is_encoder,
remove_bos=is_encoder,
combine_tower_states=is_encoder,
projection_dim=embed_dim,
add_final_predictive=is_encoder,
add_final_context=is_encoder)
return embedder, 1
elif path.startswith('bilm:'):
lm_path = path[5:]
task = LanguageModelingTask(args, dictionary, dictionary)
models, _ = utils.load_ensemble_for_inference(
[lm_path], task, {
'remove_head': True,
'dropout': args.bilm_model_dropout,
'attention_dropout': args.bilm_attention_dropout,
'relu_dropout': args.bilm_relu_dropout,
})
assert len(
models
) == 1, 'ensembles are currently not supported for elmo embeddings'
return BILMEmbedder(models[0], args, args.encoder_embed_dim) if is_encoder \
else LMEmbedder(models[0], args.decoder_embed_dim)
num_embeddings = len(dictionary)
padding_idx = dictionary.pad()
emb = nn.Embedding(num_embeddings, embed_dim, padding_idx)
# if provided, load from preloaded dictionaries
if path:
embed_dict = utils.parse_embedding(path)
utils.load_embedding(embed_dict, dictionary, emb)
return emb
def build_embedding(dictionary, embed_dim, path=None):
"""
Copied from fairseq.models.transformer
:param dictionary:
:param embed_dim:
:param path:
:return:
"""
num_embeddings = len(dictionary)
padding_idx = dictionary.pad()
emb = Embedding(num_embeddings, embed_dim, padding_idx)
# if provided, load from preloaded dictionaries
if path:
embed_dict = utils.parse_embedding(path)
utils.load_embedding(embed_dict, dictionary, emb)
return emb
def build_model(cls, args, task):
"""Build a new model instance."""
# make sure that all args are properly defaulted (in case there are any new ones)
fconv.base_architecture(args)
encoder_embed_dict = None
assert args.encoder_embed_path == None
# if args.encoder_embed_path:
# encoder_embed_dict = utils.parse_embedding(args.encoder_embed_path)
# utils.print_embed_overlap(encoder_embed_dict, task.source_dictionary)
decoder_embed_dict = None
if args.decoder_embed_path:
decoder_embed_dict = utils.parse_embedding(args.decoder_embed_path)
utils.print_embed_overlap(decoder_embed_dict,
task.target_dictionary)
encoder1 = FConvEncoder(
dictionary=task.source_dictionary1,
embed_dim=args.encoder_embed_dim,
embed_dict=encoder_embed_dict,
convolutions=eval(args.encoder_layers),
dropout=args.dropout,
max_positions=args.max_source_positions,
)
encoder2 = FConvEncoder(
dictionary=task.source_dictionary2,
embed_dim=args.encoder_embed_dim,
embed_dict=encoder_embed_dict,
convolutions=eval(args.encoder_layers),
dropout=args.dropout,
max_positions=args.max_source_positions,
)
decoder = fconv_dual.FConvDecoderWithDualAttention(
dictionary=task.target_dictionary,
embed_dim=args.decoder_embed_dim,
embed_dict=decoder_embed_dict,
convolutions=eval(args.decoder_layers),
out_embed_dim=args.decoder_out_embed_dim,
attention=eval(args.decoder_attention),
dropout=args.dropout,
max_positions=args.max_target_positions,
share_embed=args.share_input_output_embed,
encoder_output_dropout=args.encoder_output_dropout,
)
return FConvModelDualEncoderSingleDecoder(encoder1, encoder2, decoder)
def build_embedding(dictionary, embed_dim, path=None):
# The dictionary may include additional items that can be used in
# place of the normal OOV token and that all map to the same
# embedding. Using a different token for each input position allows
# one to restore the word identities from the original source text.
num_embeddings = len(dictionary) - args.source_position_markers
padding_idx = dictionary.pad()
unk_idx = dictionary.unk()
logger.info(
"dictionary indices from {0} to {1} will be mapped to {2}".
format(num_embeddings,
len(dictionary) - 1, unk_idx))
emb = Embedding(num_embeddings, embed_dim, padding_idx, unk_idx)
# if provided, load from preloaded dictionaries
if path:
embed_dict = utils.parse_embedding(path)
utils.load_embedding(embed_dict, dictionary, emb)
return emb
def build_embedding(cls, args, dictionary, embed_dim, path=None):
num_embeddings = len(dictionary)
padding_idx = dictionary.pad()
if args.no_embed:
one_hot_matrix = F.one_hot(torch.arange(num_embeddings)).float()
one_hot_embed = torch.cat((one_hot_matrix, torch.zeros((num_embeddings, embed_dim - num_embeddings))),
dim=1)
one_hot_embed[padding_idx] = torch.zeros(embed_dim).unsqueeze(0)
emb = nn.Embedding(num_embeddings, embed_dim, padding_idx=padding_idx)
emb.weight = torch.nn.parameter.Parameter(one_hot_embed, requires_grad=False)
else:
emb = Embedding(num_embeddings, embed_dim, padding_idx)
# if provided, load from preloaded dictionaries
if path:
embed_dict = utils.parse_embedding(path)
utils.load_embedding(embed_dict, dictionary, emb)
return emb
def load_embedding(embedding, dictionary, pretrained_embed):
"""Loads pretrained embeddings.
Loads pretrained embeddings into a nn.Embedding layer. pretrained_embed
can either be a nn.Embedding layer, in which case the embedding is set
to the pretrained_embed argument, or a path to an embedding file.
Arguments:
embedding (nn.Embedding): Embedding layer whose weights are to be set.
dictionary (fairseq.data.dictionary.Dictionary): dictionary with the
same vocabulary size as the embedding argument.
pretrained_embed (Union(string, nn.Embedding)): source of the
weights to be loaded.
"""
if pretrained_embed is None:
pass
elif isinstance(pretrained_embed, torch.nn.Embedding):
embedding.weight = pretrained_embed.weight
else:
embed_dict = utils.parse_embedding(pretrained_embed)
utils.load_embedding(embed_dict, dictionary, embedding)
def build_embedding(dictionary, embed_dim, path=None, num_embed_chunks=1):
assert embed_dim % num_embed_chunks == 0, (
f"Number of embedding chunks = {num_embed_chunks} should be "
+ f"divisible by the embedding dimension = {embed_dim}"
)
assert path is None or num_embed_chunks == 1, (
"Loading embedding from a path with number of embedding chunks > 1"
+ " is not yet supported"
)
num_embeddings = len(dictionary)
padding_idx = dictionary.pad()
# if provided, load from preloaded dictionaries
if path:
emb = Embedding(num_embeddings, embed_dim, padding_idx)
embed_dict = utils.parse_embedding(path)
utils.load_embedding(embed_dict, dictionary, emb)
else:
embed_chunk_dim = embed_dim // num_embed_chunks
emb = nn.ModuleList()
for i in range(num_embed_chunks):
emb.append(Embedding(num_embeddings, embed_chunk_dim, padding_idx))
return emb
def build_embed(self, num_embeddings, embedding_dim, padding_idx,
dictionary, pretrain_path, tune_epoch):
print(f'| Load embedding at {pretrain_path}')
emb = transformer.Embedding(num_embeddings, embedding_dim, padding_idx)
embed_dict = utils.parse_embedding(pretrain_path)
self.mask_factor = []
for idx in range(len(dictionary)):
token = dictionary[idx]
if token in embed_dict:
emb.weight.data[idx] = embed_dict[token]
self.mask_factor.append(0)
else:
self.mask_factor.append(1)
self.mask_factor = torch.tensor(
self.mask_factor,
dtype=torch.float32).unsqueeze_(-1).expand(len(self.mask_factor),
embedding_dim)
emb.weight.register_hook(self.hook)
return emb
def FreePretrainedEmbedding(num_embeddings, embedding_dim, padding_idx,
dictionary, pretrain_path):
print(f'| Load FreePretained embedding at {pretrain_path}')
emb = transformer.Embedding(num_embeddings, embedding_dim, padding_idx)
embed_dict = utils.parse_embedding(pretrain_path)
# mask_factor = []
for idx in range(len(dictionary)):
token = dictionary[idx]
if token in embed_dict:
emb.weight.data[idx] = embed_dict[token]
# mask_factor.append(0)
# else:
# mask_factor.append(1)
special_toks = ['-LRB-', '-RRB-']
bind_toks = ['(', ')']
for tok, btok in zip(special_toks, bind_toks):
idx = dictionary.index(tok)
if btok in embed_dict and idx != dictionary.unk_index:
print(f'Replace embed for {tok} {btok} - {idx}')
emb.weight.data[idx] = embed_dict[btok]
return emb
def build_embed(self, num_embeddings, embedding_dim, padding_idx,
dictionary, pretrain_path, tune_epoch):
print(f'| Load embedding dict at {pretrain_path}')
pretrained_embed_dict = utils.parse_embedding(pretrain_path)
remap_pretrained_embed = []
remap_pretrained_indices = []
remap_new_indices = []
self.pretrained_pad_idx = None
self.new_pad_idx = None
self.pretraied_unk_idx = None
self.new_unk_idx = None
unk_idx = dictionary.unk_index
phrase_label_idx = []
nonphrase_label_idx = []
for idx in range(len(dictionary)):
token = dictionary[idx]
is_padding = idx == padding_idx
is_unk = idx == unk_idx
if token in pretrained_embed_dict:
remap_pretrained_embed.append(pretrained_embed_dict[token])
remap_pretrained_indices.append(idx)
if is_padding:
print(
f'Padding {idx} in pretained embeddings, map to {len(remap_pretrained_indices) - 1}'
)
self.pretrained_pad_idx = len(remap_pretrained_indices) - 1
if is_unk:
print(
f'UNK {idx} in pretained embeddings, map to {len(remap_pretrained_indices) - 1}'
)
self.pretraied_unk_idx = len(remap_pretrained_indices) - 1
else:
remap_new_indices.append(idx)
if "_label" in token:
phrase_label_idx.append(idx)
else:
nonphrase_label_idx.append(idx)
if is_padding:
print(
f'Padding {idx} in new embeddings, map to {len(remap_new_indices) - 1}'
)
self.new_pad_idx = len(remap_new_indices) - 1
if is_unk:
print(
f'UNK {idx} in new embeddings, map to {len(remap_new_indices) - 1}'
)
self.new_unk_idx = len(remap_new_indices) - 1
assert self.pretrained_pad_idx is None or self.new_pad_idx is None
assert self.pretraied_unk_idx is None or self.new_unk_idx is None
if self.pretraied_unk_idx is not None:
self.final_unk_idx = self.pretraied_unk_idx
else:
self.final_unk_idx = len(remap_pretrained_embed) + self.new_unk_idx
final_indices = remap_pretrained_indices + remap_new_indices
reordering = sorted(range(len(final_indices)),
key=lambda k: final_indices[k])
if self.enforce_unk:
print(
f'Reordering indices for UNK, non-phrase={len(nonphrase_label_idx)}'
)
if len(phrase_label_idx) == 0:
raise ValueError(
f'enforce unk, phrase-label empty, enforce will cause all phrase-lavel to UNK'
)
redirect_reordering = []
for idx in reordering:
if idx in nonphrase_label_idx:
redirect_reordering.append(self.final_unk_idx)
else:
redirect_reordering.append(idx)
reordering = redirect_reordering
reordering = nn.Parameter(torch.tensor(reordering),
requires_grad=False)
pretrained_embed = nn.Embedding.from_pretrained(
torch.stack(remap_pretrained_embed),
freeze=self.freeze,
padding_idx=self.pretrained_pad_idx)
# new_embed = transformer.Embedding(len(remap_new_indices), embedding_dim, self.new_pad_idx)
new_embed = self.create_new_embed(len(remap_new_indices),
embedding_dim,
self.new_pad_idx,
pretrained_emb=pretrained_embed)
# [0, 1, 2, 3, 4]
# [2, 3, 1, 0, 4]
# [3, 2, 0, 1, 4]
# 3 -> 1
print(f'Phrase-label : {len(phrase_label_idx)}')
if len(phrase_label_idx) == 0:
print(f'WARNING !!! Phrase-label set empty!')
return reordering, pretrained_embed, new_embed
def __init__(
self,
src_dict,
dst_dict,
vocab_reduction_params=None,
out_embed_dim=512,
project_output=True,
pretrained_embed=None,
out_embed_norm=None,
att_weighted_src_embeds=False,
src_embed_dim=512,
att_weighted_activation_type="tanh",
predictor=None,
fp16: bool = False,
):
super().__init__(dst_dict)
self.project_output = project_output
if project_output:
self.num_embeddings = len(dst_dict)
self.out_embed_dim = out_embed_dim
self.out_embed_norm = out_embed_norm
self.att_weighted_src_embeds = att_weighted_src_embeds
self.src_embed_dim = src_embed_dim
self.vocab_reduction_module = None
if vocab_reduction_params or predictor is not None:
self.vocab_reduction_module = vocab_reduction.VocabReduction(
src_dict=src_dict,
dst_dict=dst_dict,
vocab_reduction_params=vocab_reduction_params,
predictor=predictor,
fp16=fp16,
)
projection_weights = torch.FloatTensor(
self.num_embeddings, self.out_embed_dim
).uniform_(-0.1, 0.1)
if isinstance(pretrained_embed, nn.Embedding):
projection_weights.data = pretrained_embed.weights.data
elif pretrained_embed is not None:
embed_dict = utils.parse_embedding(pretrained_embed)
# equivalent to utils.load_embedding but for nn.Parameter
for idx in range(len(dst_dict)):
token = dst_dict[idx]
if token in embed_dict:
projection_weights[idx] = embed_dict[token]
self.output_projection_w = nn.Parameter(projection_weights)
self.output_projection_b = nn.Parameter(
torch.FloatTensor(self.num_embeddings).zero_()
)
if att_weighted_activation_type == "tanh":
activation_fn = nn.Tanh
self.att_weighted_activation_fn = torch.tanh
elif att_weighted_activation_type == "relu":
activation_fn = nn.ReLU
self.att_weighted_activation_fn = torch.relu
else:
raise Exception(
"att_weighted_activation_type '%s' not implemented"
% att_weighted_activation_type
)
if att_weighted_src_embeds:
print(att_weighted_activation_type)
self.lexical_layer = NonlinearLayer(
self.src_embed_dim,
self.out_embed_dim,
bias=False,
activation_fn=activation_fn,
)
self.output_projection_w_lex = nn.Parameter(
torch.FloatTensor(self.num_embeddings, self.out_embed_dim).uniform_(
-0.1, 0.1
)
)
self.output_projection_b_lex = nn.Parameter(
torch.FloatTensor(self.num_embeddings).zero_()
)
def build_model(cls, args, task):
"""Build a new model instance."""
# make sure all arguments are present in older models
base_architecture(args)
attention_layer = {
'dot': DotAttentionLayer,
'general': GeneralAttentionLayer,
'multi-head': MultiheadAttention,
'mlp': MLPAttentionLayer,
}
if args.decoder_embed_path:
decoder_embed_dict = utils.parse_embedding(args.decoder_embed_path)
utils.print_embed_overlap(decoder_embed_dict,
task.target_dictionary)
if not hasattr(args, 'max_source_positions'):
args.max_source_positions = 1024
if not hasattr(args, 'max_target_positions'):
args.max_target_positions = 1024
src_dict, tgt_dict = task.source_dictionary, task.target_dictionary
def build_embedding(dictionary, embed_dim, path=None):
num_embeddings = len(dictionary)
padding_idx = dictionary.pad()
emb = Embedding(num_embeddings, embed_dim, padding_idx)
# if provided, load from preloaded dictionaries
if path:
embed_dict = utils.parse_embedding(path)
utils.load_embedding(embed_dict, dictionary, emb)
return emb
def build_audio_embedding(embed_dim, dropout):
m = nn.Linear(task.audio_features, embed_dim)
nn.init.normal_(m.weight,
mean=0,
std=math.sqrt((1 - dropout) / task.audio_features))
nn.init.constant_(m.bias, 0)
return m
encoder_embed_tokens = build_audio_embedding(
2 * args.encoder_embed_dim, args.dropout)
encoder = ProxyEncoder(
args,
tgt_dict,
encoder_embed_tokens,
audio_features=task.audio_features,
)
decoder = CLSTMDecoder(
dictionary=task.target_dictionary,
embed_dim=args.decoder_embed_dim,
out_embed_dim=args.decoder_out_embed_dim,
attention=eval(args.decoder_attention),
hidden_size=args.hidden_size,
dropout_in=args.dropout,
dropout_out=args.dropout,
num_layers=args.decoder_layers,
attention_layer=attention_layer[args.attention_type],
initial_state=args.decoder_initial_state,
weight_norm=args.weight_norm,
attention_function=args.attention_function,
scale_norm=args.scale_norm,
)
return Transf2BerModel(encoder, decoder)
def build_model(cls, args, task):
"""Build a new model instance."""
# make sure that all args are properly defaulted (in case there are any new ones)
base_architecture(args)
decoder_embed_dict = None
if args.decoder_embed_path:
decoder_embed_dict = utils.parse_embedding(args.decoder_embed_path)
utils.print_embed_overlap(decoder_embed_dict,
task.target_dictionary)
def eval_str_nested_list_or_tuple(x, type=int):
if x is None:
return None
if isinstance(x, str):
x = eval(x)
if isinstance(x, list):
return list(
map(lambda s: eval_str_nested_list_or_tuple(s, type), x))
elif isinstance(x, tuple):
return tuple(
map(lambda s: eval_str_nested_list_or_tuple(s, type), x))
else:
try:
return type(x)
except TypeError:
raise TypeError
out_channels = eval_str_nested_list_or_tuple(
args.encoder_conv_channels, type=int)
kernel_sizes = eval_str_nested_list_or_tuple(
args.encoder_conv_kernel_sizes, type=int)
strides = eval_str_nested_list_or_tuple(args.encoder_conv_strides,
type=int)
print('| input feature dimension: {}, channels: {}'.format(
task.feat_dim, task.feat_in_channels))
assert task.feat_dim % task.feat_in_channels == 0
conv_layers = ConvBNReLU(
out_channels,
kernel_sizes,
strides,
in_channels=task.feat_in_channels,
) if out_channels is not None else None
fconv_encoder_input_size = task.feat_dim // task.feat_in_channels
if conv_layers is not None:
for stride in strides:
if isinstance(stride, (list, tuple)):
assert len(stride) > 0
s = stride[1] if len(stride) > 1 else stride[0]
else:
assert isinstance(stride, int)
s = stride
fconv_encoder_input_size = (fconv_encoder_input_size + s -
1) // s
fconv_encoder_input_size *= out_channels[-1]
encoder = SpeechFConvEncoder(
conv_layers_before=conv_layers,
input_size=fconv_encoder_input_size,
embed_dim=args.encoder_embed_dim,
convolutions=eval(args.encoder_layers),
dropout=args.dropout,
)
decoder = SpeechFConvDecoder(
dictionary=task.target_dictionary,
embed_dim=args.decoder_embed_dim,
embed_dict=decoder_embed_dict,
convolutions=eval(args.decoder_layers),
out_embed_dim=args.decoder_out_embed_dim,
attention=eval(args.decoder_attention),
dropout=args.dropout,
max_positions=args.max_target_positions,
share_embed=args.share_input_output_embed,
positional_embeddings=args.decoder_positional_embed,
)
return SpeechFConvModel(encoder, decoder)
def build_model(cls, args, task):
"""Build a new model instance."""
# make sure that all args are properly defaulted (in case there are any new ones)
base_architecture(args)
encoder_embed_dict = None
if args.encoder_embed_path:
encoder_embed_dict = utils.parse_embedding(args.encoder_embed_path)
utils.print_embed_overlap(encoder_embed_dict,
task.source_dictionary)
decoder_embed_dict = None
if args.decoder_embed_path:
decoder_embed_dict = utils.parse_embedding(args.decoder_embed_path)
utils.print_embed_overlap(decoder_embed_dict,
task.target_dictionary)
kw_embed_dict = None
if args.keywords_embed_path:
kw_embed_dict = utils.parse_embedding(args.keywords_embed_path)
print(">> Keywords embeddings loaded!")
docEncoder = ExtendedEncoder(
args,
FConvEncoder(
dictionary=task.source_dictionary,
embed_dim=args.encoder_embed_dim,
embed_dict=encoder_embed_dict,
convolutions=eval(args.encoder_layers),
dropout=args.dropout,
max_positions=args.max_source_positions,
normalization_constant=args.normalization_constant,
))
decoder = CondFConvDecoder(
dictionary=task.target_dictionary,
embed_dim=args.decoder_embed_dim,
embed_dict=decoder_embed_dict,
convolutions=eval(args.decoder_layers),
out_embed_dim=args.decoder_out_embed_dim,
attention=eval(args.decoder_attention),
dropout=args.dropout,
max_positions=args.max_tgt_sentence_length,
share_embed=args.share_input_output_embed,
normalization_constant=args.normalization_constant,
)
docDecoder = LDocumentDecoder(
args,
decoder,
embed_dim=args.decoder_embed_dim,
hidden_size=args.decoder_embed_dim,
out_embed_dim=args.decoder_out_embed_dim,
encoder_embed_dim=args.encoder_embed_dim,
encoder_output_units=args.encoder_embed_dim,
num_layers=1,
attention=eval(args.decoder_attention),
dropout_in=0.1,
dropout_out=0.1,
pretrained_embed=None,
num_topics=args.num_topics,
)
return FEncFairseqModel(docEncoder, docDecoder, args.lambda_keyloss)
def __init__(self,
# new approach
params: Params = None,
vocab: Dictionary = None,
# old approach
args = None,
task = None
):
if params is not None and args is not None:
raise ConfigurationError("you cannot define both, params and args, you have to device which one to use (just one way is allowed)")
if params is not None:
encoder = FConvEncoder(
dictionary=vocab
)
decoder = FConvDecoder(
dictionary=vocab
)
else:
if args is not None and task is not None:
# Create and initialize encoder and decoder here
encoder_embed_dict = None
if args.encoder_embed_path:
encoder_embed_dict = utils.parse_embedding(args.encoder_embed_path)
utils.print_embed_overlap(encoder_embed_dict, task.source_dictionary)
decoder_embed_dict = None
if args.decoder_embed_path:
decoder_embed_dict = utils.parse_embedding(args.decoder_embed_path)
utils.print_embed_overlap(decoder_embed_dict, task.target_dictionary)
encoder = FConvEncoder(
dictionary=task.source_dictionary,
embed_dim=args.encoder_embed_dim,
embed_dict=encoder_embed_dict,
convolutions=eval(args.encoder_layers),
dropout=args.dropout,
max_positions=args.max_source_positions,
batch_norm=args.batch_norm,
use_linear_se=True#args.use_linear_se
)
decoder = FConvDecoder(
dictionary=task.target_dictionary,
embed_dim=args.decoder_embed_dim,
embed_dict=decoder_embed_dict,
convolutions=eval(args.decoder_layers),
out_embed_dim=args.decoder_out_embed_dim,
attention=eval(args.decoder_attention),
dropout=args.dropout,
max_positions=args.max_target_positions,
share_embed=args.share_input_output_embed,
use_linear_se=False#args.use_linear_se
)
else:
# We have a problem!
raise ConfigurationError("params and (args, task) are all None, something is wrong here.")
# Call the super class
super(FConvModel, self).__init__(encoder, decoder)
# Correctly set the number of attention layers
self.encoder.num_attention_layers = sum(layer is not None for layer in decoder.attention)
