def _get_dynamic_fields(opts): # NOTE: not support tgt feats yet tgt_feats = None with_align = hasattr(opts, 'lambda_align') and opts.lambda_align > 0.0 fields = get_fields('text', opts.src_feats_vocab, tgt_feats, dynamic_dict=opts.copy_attn, src_truncate=opts.src_seq_length_trunc, tgt_truncate=opts.tgt_seq_length_trunc, with_align=with_align, data_task=opts.data_task) return fields
def _get_dynamic_fields(opts): # NOTE: not support nfeats > 0 yet src_nfeats = 0 tgt_nfeats = 0 with_align = hasattr(opts, 'lambda_align') and opts.lambda_align > 0.0 fields = get_fields('text', src_nfeats, tgt_nfeats, dynamic_dict=opts.copy_attn, src_truncate=opts.src_seq_length_trunc, tgt_truncate=opts.tgt_seq_length_trunc, with_align=with_align) return fields
def build_fields(ds: Dataset): # initialize the frequency counter counters = defaultdict(Counter) _src_vocab, _src_vocab_size = inputter._load_vocab(ds.vocab.source, 'src', counters) _tgt_vocab, _tgt_vocab_size = inputter._load_vocab(ds.vocab.target, 'tgt', counters) # initialize fields src_nfeats, tgt_nfeats = 0, 0 # do not support word features for now fields = inputter.get_fields(defaults.vocabulary["data_type"], src_nfeats, tgt_nfeats) return inputter._build_fields_vocab(fields, counters, **defaults.vocabulary)