def test_dataset_split_parameter(): """ Dataset is split in the desired ratios """ split_parameter = [0.7, 0.1, 0.2] st = SentenceTokenizer(vocab, 30) result, result_dicts, _ = st.split_train_val_test(sentences, dicts, split_parameter, extend_with=0) train = result[0] val = result[1] test = result[2] train_dicts = result_dicts[0] val_dicts = result_dicts[1] test_dicts = result_dicts[2] assert len(train) == len(sentences) * split_parameter[0] assert len(val) == len(sentences) * split_parameter[1] assert len(test) == len(sentences) * split_parameter[2] assert len(train_dicts) == len(dicts) * split_parameter[0] assert len(val_dicts) == len(dicts) * split_parameter[1] assert len(test_dicts) == len(dicts) * split_parameter[2]
def load_benchmark(path, vocab, extend_with=0): """ Loads the given benchmark dataset. Tokenizes the texts using the provided vocabulary, extending it with words from the training dataset if extend_with > 0. Splits them into three lists: training, validation and testing (in that order). Also calculates the maximum length of the texts and the suggested batch_size. # Arguments: path: Path to the dataset to be loaded. vocab: Vocabulary to be used for tokenizing texts. extend_with: If > 0, the vocabulary will be extended with up to extend_with tokens from the training set before tokenizing. # Returns: A dictionary with the following fields: texts: List of three lists, containing tokenized inputs for training, validation and testing (in that order). labels: List of three lists, containing labels for training, validation and testing (in that order). added: Number of tokens added to the vocabulary. batch_size: Batch size. maxlen: Maximum length of an input. """ # Pre-processing dataset with open(path, 'rb') as dataset: if IS_PYTHON2: data = pickle.load(dataset) else: data = pickle.load(dataset, fix_imports=True) # Decode data try: texts = [unicode_(x) for x in data['texts']] except UnicodeDecodeError: texts = [x.decode('utf-8') for x in data['texts']] # Extract labels labels = [x['label'] for x in data['info']] batch_size, maxlen = calculate_batchsize_maxlen(texts) st = SentenceTokenizer(vocab, maxlen) # Split up dataset. Extend the existing vocabulary with up to extend_with # tokens from the training dataset. texts, labels, added = st.split_train_val_test(texts, labels, [data['train_ind'], data['val_ind'], data['test_ind']], extend_with=extend_with) return {'texts': texts, 'labels': labels, 'added': added, 'batch_size': batch_size, 'maxlen': maxlen}
def load_benchmark(path, vocab, extend_with=0): """ Loads the given benchmark dataset. Tokenizes the texts using the provided vocabulary, extending it with words from the training dataset if extend_with > 0. Splits them into three lists: training, validation and testing (in that order). Also calculates the maximum length of the texts and the suggested batch_size. # Arguments: path: Path to the dataset to be loaded. vocab: Vocabulary to be used for tokenizing texts. extend_with: If > 0, the vocabulary will be extended with up to extend_with tokens from the training set before tokenizing. # Returns: A dictionary with the following fields: texts: List of three lists, containing tokenized inputs for training, validation and testing (in that order). labels: List of three lists, containing labels for training, validation and testing (in that order). added: Number of tokens added to the vocabulary. batch_size: Batch size. maxlen: Maximum length of an input. """ # Pre-processing dataset with open(path, 'rb') as dataset: if IS_PYTHON2: data = pickle.load(dataset) else: data = pickle.load(dataset, fix_imports=True) # Decode data try: texts = [unicode_(x) for x in data['texts']] except UnicodeDecodeError: texts = [x.decode('utf-8') for x in data['texts']] # Extract labels labels = [x['label'] for x in data['info']] batch_size, maxlen = calculate_batchsize_maxlen(texts) st = SentenceTokenizer(vocab, maxlen) # Split up dataset. Extend the existing vocabulary with up to extend_with # tokens from the training dataset. texts, labels, added = st.split_train_val_test(texts, labels, [data['train_ind'], data['val_ind'], data['test_ind']], extend_with=extend_with) return {'texts': texts, 'labels': labels, 'added': added, 'batch_size': batch_size, 'maxlen': maxlen}
def convert_dataset(filepath, extend_with, vocab): print('-- Generating {} '.format(filepath)) sys.stdout.flush() st = SentenceTokenizer(vocab, maxlen) tokenized, dicts, _ = st.split_train_val_test( texts, labels, [data['train_ind'], data['val_ind'], data['test_ind']], extend_with=extend_with) pick = format_pickle(dset, tokenized[0], tokenized[1], tokenized[2], dicts[0], dicts[1], dicts[2]) with open(filepath, 'w') as f: pickle.dump(pick, f) cover = coverage(tokenized[2]) print(' done. Coverage: {}'.format(cover))
def test_dataset_split_explicit(): """ Dataset is split according to given indices """ split_parameter = [train_ind, val_ind, test_ind] st = SentenceTokenizer(vocab, 30) tokenized, _, _ = st.tokenize_sentences(sentences) result, result_dicts, added = st.split_train_val_test(sentences, dicts, split_parameter, extend_with=0) train = result[0] val = result[1] test = result[2] train_dicts = result_dicts[0] val_dicts = result_dicts[1] test_dicts = result_dicts[2] tokenized = tokenized for i, sentence in enumerate(sentences): if i in train_ind: assert tokenized[i] in train assert dicts[i] in train_dicts elif i in val_ind: assert tokenized[i] in val assert dicts[i] in val_dicts elif i in test_ind: assert tokenized[i] in test assert dicts[i] in test_dicts assert len(train) == len(train_ind) assert len(val) == len(val_ind) assert len(test) == len(test_ind) assert len(train_dicts) == len(train_ind) assert len(val_dicts) == len(val_ind) assert len(test_dicts) == len(test_ind)
if IS_PYTHON2: s = pickle.load(f) else: s = pickle.load(f, fix_imports=True) # Decode data try: s['texts'] = [unicode(x) for x in s['texts']] except UnicodeDecodeError: s['texts'] = [x.decode('utf-8') for x in s['texts']] # Own st = SentenceTokenizer({}, 30) tests, dicts, _ = st.split_train_val_test(s['texts'], s['info'], [s['train_ind'], s['val_ind'], s['test_ind']], extend_with=10000) coverage_result.append(coverage(tests[2])) # Last st = SentenceTokenizer(vocab, 30) tests, dicts, _ = st.split_train_val_test(s['texts'], s['info'], [s['train_ind'], s['val_ind'], s['test_ind']], extend_with=0) coverage_result.append(coverage(tests[2])) # Full st = SentenceTokenizer(vocab, 30)
}, { 'label': 'sentence 5' }, { 'label': 'sentence 6' }, { 'label': 'sentence 7' }, { 'label': 'sentence 8' }, { 'label': 'sentence 9' }, ] with open('../model/vocabulary.json', 'r') as f: vocab = json.load(f) st = SentenceTokenizer(vocab, 30) # Split using the default split ratio print(st.split_train_val_test(DATASET, INFO_DICTS)) # Split explicitly print( st.split_train_val_test(DATASET, INFO_DICTS, [[0, 1, 2, 4, 9], [5, 6], [7, 8, 3]], extend_with=1))
'I am sentence 7', 'I am sentence 8', 'I am sentence 9 newword', ] INFO_DICTS = [ {'label': 'sentence 0'}, {'label': 'sentence 1'}, {'label': 'sentence 2'}, {'label': 'sentence 3'}, {'label': 'sentence 4'}, {'label': 'sentence 5'}, {'label': 'sentence 6'}, {'label': 'sentence 7'}, {'label': 'sentence 8'}, {'label': 'sentence 9'}, ] with open('../model/vocabulary.json', 'r') as f: vocab = json.load(f) st = SentenceTokenizer(vocab, 30) # Split using the default split ratio print(st.split_train_val_test(DATASET, INFO_DICTS)) # Split explicitly print(st.split_train_val_test(DATASET, INFO_DICTS, [[0, 1, 2, 4, 9], [5, 6], [7, 8, 3]], extend_with=1))