def get_fields(data_type): """ Args: n_src_features: the number of source features to create `torchtext.data.Field` for. n_tgt_features: the number of target features to create `torchtext.data.Field` for. data_type: concat / query / hier Returns: A dictionary whose keys are strings and whose values are the corresponding Field objects. """ return TextDataset.get_fields(data_type)
def get_fields(data_type, n_src_features, n_tgt_features): """ Args: data_type: type of the source input. Options are [text|img|audio]. n_src_features: the number of source features to create `torchtext.data.Field` for. n_tgt_features: the number of target features to create `torchtext.data.Field` for. Returns: A dictionary whose keys are strings and whose values are the corresponding Field objects. """ return TextDataset.get_fields(n_src_features, n_tgt_features)
def get_fields(data_type, n_src_features, n_tgt_features, n_ans_features): """ Args: data_type: type of the source input. Options are [text|img|audio]. n_src_features: the number of source features to create `torchtext.data.Field` for. n_tgt_features: the number of target features to create `torchtext.data.Field` for. Returns: A dictionary whose keys are strings and whose values are the corresponding Field objects. """ if data_type == 'text': return TextDataset.get_fields(n_src_features, n_tgt_features, n_ans_features) else: raise ValueError("Data type not implemented")
def load_fields_from_vocab(vocab, data_type="text"): """ Load Field objects from `vocab.pt` file. """ vocab = dict(vocab) n_src_features = len(collect_features(vocab, 'src')) n_tgt_features = len(collect_features(vocab, 'tgt')) n_ans_features = len(collect_features(vocab, 'ans')) #logger.info("n_src_features " + str(n_src_features)) #logger.info("n_tgt_features" + str(n_tgt_features)) #logger.info("n_ans_features " + str(n_ans_features)) fields = TextDataset.get_fields(n_src_features, n_tgt_features, n_ans_features) #fields = get_fields(data_type, n_src_features, n_tgt_features) for k, v in vocab.items(): # Hack. Can't pickle defaultdict :( v.stoi = defaultdict(lambda: 0, v.stoi) fields[k].vocab = v return fields