def load(self, paths: Union[str, Dict[str, str]] = None,ratio_tr_d_te:tuple =ratio_tr_d_te) -> DataBundle: ''' :param paths: 为str时,读入训练集合的所有训练样本,并在训练集的基础上按比例8:1:1划分 为Dict[str,str]时,通过test,val,train这种键值来pick训练、测试、验证集,(train必须要有) 如果没有val、test,从train中划分一定比例充当验证/测试集 :return: ''' paths =self.check_loader_paths(paths) #此时的paths是字典{'train':XXX,..} datasets ={_:self._load(path=path) for _,path in paths.items()} # 对所有数据做shuffle 处理 for name,ds in datasets.items(): shuffled_ds =DataSet() indices =[_ for _ in range(len(ds))] random.shuffle(indices) for _ in indices: shuffled_ds.append(ds[_]) datasets[name] =shuffled_ds # shuffle 处理结束 if len(datasets) ==1: print('检测到只load train中的dataset,默认8:1:1拆分为train/test/val 三份集合') ds =datasets['train'] train_count =int(len(ds)*(ratio_tr_d_te[0]/sum(ratio_tr_d_te))) test_count = int(len(ds)*(ratio_tr_d_te[2]/sum(ratio_tr_d_te))) return DataBundle(datasets={'train':ds[:train_count],'val':ds[train_count:-test_count],'test':ds[-test_count:]}) elif len(datasets) ==3: print('检测到train,test,val,不需要从train划分') return DataBundle(datasets=datasets) elif 'val' not in datasets: print('检测到train,test,从train划分出val') ds = datasets['train'] val_count = int(len(ds) * (ratio_tr_d_te[1] / sum(ratio_tr_d_te))) return DataBundle(datasets= {'train': ds[:-val_count], 'val': ds[-val_count:], 'test': datasets['test']}) elif 'test' not in datasets: print('检测到train,val,从train划分出test') ds = datasets['train'] test_count = int(len(ds) * (ratio_tr_d_te[2] / sum(ratio_tr_d_te))) return DataBundle(datasets={'train': ds[:-test_count], 'val': ds[-test_count:], 'test': datasets['test']})
def load(self, folder): data_bundle = DataBundle() for name in ['desc.json', 'dev.json', 'seen.json', 'train.json', 'unseen.json']: path = os.path.join(folder, name) dataset = DataSet() with open(path, 'r', encoding='utf-8') as f: data = json.load(f) for d in data: word = d['word'].lower() definition = d['definitions'].lower() ins = Instance(word=word, definition=definition) dataset.append(ins) data_bundle.set_dataset(dataset, name=name.split('.')[0]) words = [] with open(os.path.join(folder, 'target_words.txt'), 'r', encoding='utf-8') as f: for line in f: line = line.strip() if line: words.append(line) setattr(data_bundle, 'target_words', words) return data_bundle
def process(self, paths): """ :param paths: :return: Dataset包含以下的field chars: bigrams: trigrams: pre_chars: pre_bigrams: pre_trigrams: seg_targets: seg_masks: seq_lens: char_labels: char_heads: gold_word_pairs: seg_targets: seg_masks: char_labels: char_heads: pun_masks: gold_label_word_pairs: """ paths = check_dataloader_paths(paths) data = DataBundle() for name, path in paths.items(): print(name,path) dataset = self.load(path) data.datasets[name] = dataset char_labels_vocab = Vocabulary(padding=None, unknown=None) def process(dataset, char_label_vocab): dataset.apply(add_word_lst, new_field_name='word_lst') dataset.apply(lambda x: list(chain(*x['word_lst'])), new_field_name='chars') dataset.apply(add_bigram, field_name='chars', new_field_name='bigrams') dataset.apply(add_trigram, field_name='chars', new_field_name='trigrams') dataset.apply(add_char_heads, new_field_name='char_heads') dataset.apply(add_char_labels, new_field_name='char_labels') dataset.apply(add_segs, new_field_name='seg_targets') dataset.apply(add_mask, new_field_name='seg_masks') dataset.add_seq_len('chars', new_field_name='seq_lens') dataset.apply(add_pun_masks, new_field_name='pun_masks') if len(char_label_vocab.word_count)==0: char_label_vocab.from_dataset(dataset, field_name='char_labels') char_label_vocab.index_dataset(dataset, field_name='char_labels') new_dataset = add_root(dataset) new_dataset.apply(add_word_pairs, new_field_name='gold_word_pairs', ignore_type=True) global add_label_word_pairs add_label_word_pairs = partial(add_label_word_pairs, label_vocab=char_label_vocab) new_dataset.apply(add_label_word_pairs, new_field_name='gold_label_word_pairs', ignore_type=True) new_dataset.set_pad_val('char_labels', -1) new_dataset.set_pad_val('char_heads', -1) return new_dataset for name in list(paths.keys()): dataset = data.datasets[name] dataset = process(dataset, char_labels_vocab) data.datasets[name] = dataset data.vocabs['char_labels'] = char_labels_vocab char_vocab = Vocabulary(min_freq=2).from_dataset(data.datasets['train'], field_name='chars') bigram_vocab = Vocabulary(min_freq=5).from_dataset(data.datasets['train'], field_name='bigrams') trigram_vocab = Vocabulary(min_freq=5).from_dataset(data.datasets['train'], field_name='trigrams') for name in ['chars', 'bigrams', 'trigrams']: vocab = Vocabulary().from_dataset(field_name=name, no_create_entry_dataset=list(data.datasets.values())) vocab.index_dataset(*data.datasets.values(), field_name=name, new_field_name='pre_' + name) data.vocabs['pre_{}'.format(name)] = vocab for name, vocab in zip(['chars', 'bigrams', 'trigrams'], [char_vocab, bigram_vocab, trigram_vocab]): vocab.index_dataset(*data.datasets.values(), field_name=name, new_field_name=name) data.vocabs[name] = vocab for name, dataset in data.datasets.items(): dataset.set_input('chars', 'bigrams', 'trigrams', 'seq_lens', 'char_labels', 'char_heads', 'pre_chars', 'pre_bigrams', 'pre_trigrams') dataset.set_target('gold_word_pairs', 'seq_lens', 'seg_targets', 'seg_masks', 'char_labels', 'char_heads', 'pun_masks', 'gold_label_word_pairs') return data