def get_words(self, sentence): """ Tokenizes a sentence into individual words. Converts Unicode punctuation into ASCII if that option is set. Ignores sentences with Unicode if that option is set. Returns an empty list of words if the sentence has Unicode and that is not allowed. """ if not isinstance(sentence, str): raise ValueError("All sentences should be Unicode-encoded!") sentence = sentence.strip().lower() if self.break_replacement: sentence = convert_linebreaks(sentence) if self.remove_variation_selectors: sentence = remove_variation_selectors(sentence) # Split into words using simple whitespace splitting and convert # Unicode. This is done to prevent word splitting issues with # tokenize and Unicode words = sentence.split() converted_words = [] for w in words: accept_sentence, c_w = self.convert_unicode_word(w) # Unicode word detected and not allowed if not accept_sentence: return [] else: converted_words.append(c_w) sentence = ' '.join(converted_words) words = tokenize(sentence) words = [process_word(w) for w in words] return words
def test_base(tests): """ Base function for running tests. """ for (test, expected) in tests: actual = tokenize(test) assert actual == expected, \ "Tokenization of \'{}\' failed, expected: {}, actual: {}"\ .format(test, expected, actual)
def test_base(tests): """ Base function for running tests. """ for (test, expected) in tests: actual = tokenize(test) assert actual == expected, \ "Tokenization of \'{}\' failed, expected: {}, actual: {}"\ .format(test, expected, actual)
def calculate_batchsize_maxlen(texts): """ Calculates the maximum length in the provided texts and a suitable batch size. Rounds up maxlen to the nearest multiple of ten. # Arguments: texts: List of inputs. # Returns: Batch size, max length """ def roundup(x): return int(math.ceil(x / 10.0)) * 10 # Calculate max length of sequences considered # Adjust batch_size accordingly to prevent GPU overflow lengths = [len(tokenize(t)) for t in texts] maxlen = roundup(np.percentile(lengths, 80.0)) batch_size = 250 if maxlen <= 100 else 50 return batch_size, maxlen
PATH_RAW = '{}/{}/{}'.format(DIR, dset, FILENAME_RAW) PATH_OWN = '{}/{}/{}'.format(DIR, dset, FILENAME_OWN) PATH_OUR = '{}/{}/{}'.format(DIR, dset, FILENAME_OUR) PATH_COMBINED = '{}/{}/{}'.format(DIR, dset, FILENAME_COMBINED) with open(PATH_RAW) as dataset: data = pickle.load(dataset) # Decode data try: texts = [unicode(x) for x in data['texts']] except UnicodeDecodeError: texts = [x.decode('utf-8') for x in data['texts']] wg = WordGenerator(texts) vb = VocabBuilder(wg) vb.count_all_words() # Calculate max length of sequences considered # Adjust batch_size accordingly to prevent GPU overflow lengths = [len(tokenize(t)) for t in texts] maxlen = roundup(np.percentile(lengths, 80.0)) # Extract labels labels = [x['label'] for x in data['info']] convert_dataset(PATH_OWN, 50000, {}) convert_dataset(PATH_OUR, 0, vocab) convert_dataset(PATH_COMBINED, 10000, vocab)
PATH_RAW = '{}/{}/{}'.format(DIR, dset, FILENAME_RAW) PATH_OWN = '{}/{}/{}'.format(DIR, dset, FILENAME_OWN) PATH_OUR = '{}/{}/{}'.format(DIR, dset, FILENAME_OUR) PATH_COMBINED = '{}/{}/{}'.format(DIR, dset, FILENAME_COMBINED) with open(PATH_RAW) as dataset: data = pickle.load(dataset) # # # Decode data # try: # texts = [unicode(x) for x in data['texts']] # except UnicodeDecodeError: # texts = [x.decode('utf-8') for x in data['texts']] wg = WordGenerator(texts) vb = VocabBuilder(wg) vb.count_all_words() # Calculate max length of sequences considered # Adjust batch_size accordingly to prevent GPU overflow lengths = [len(tokenize(t)) for t in texts] maxlen = roundup(np.percentile(lengths, 80.0)) # Extract labels labels = [x['label'] for x in data['info']] convert_dataset(PATH_OWN, 50000, {}) convert_dataset(PATH_OUR, 0, vocab) convert_dataset(PATH_COMBINED, 10000, vocab)
def test_base(tests): """ Base function for running tests. """ for (test, expected) in tests: actual = tokenize(test)