def _get_dynamic_fields(opts):
# NOTE: not support tgt feats yet
tgt_feats = None
with_align = hasattr(opts, 'lambda_align') and opts.lambda_align > 0.0
fields = get_fields('text',
opts.src_feats_vocab,
tgt_feats,
dynamic_dict=opts.copy_attn,
src_truncate=opts.src_seq_length_trunc,
tgt_truncate=opts.tgt_seq_length_trunc,
with_align=with_align,
data_task=opts.data_task)
return fields
def _get_dynamic_fields(opts):
# NOTE: not support nfeats > 0 yet
src_nfeats = 0
tgt_nfeats = 0
with_align = hasattr(opts, 'lambda_align') and opts.lambda_align > 0.0
fields = get_fields('text',
src_nfeats,
tgt_nfeats,
dynamic_dict=opts.copy_attn,
src_truncate=opts.src_seq_length_trunc,
tgt_truncate=opts.tgt_seq_length_trunc,
with_align=with_align)
return fields
def build_fields(ds: Dataset):
# initialize the frequency counter
counters = defaultdict(Counter)
_src_vocab, _src_vocab_size = inputter._load_vocab(ds.vocab.source, 'src',
counters)
_tgt_vocab, _tgt_vocab_size = inputter._load_vocab(ds.vocab.target, 'tgt',
counters)
# initialize fields
src_nfeats, tgt_nfeats = 0, 0 # do not support word features for now
fields = inputter.get_fields(defaults.vocabulary["data_type"], src_nfeats,
tgt_nfeats)
return inputter._build_fields_vocab(fields, counters,
**defaults.vocabulary)