def get_words(self, sentence):
""" Tokenizes a sentence into individual words.
Converts Unicode punctuation into ASCII if that option is set.
Ignores sentences with Unicode if that option is set.
Returns an empty list of words if the sentence has Unicode and
that is not allowed.
"""
if not isinstance(sentence, unicode):
raise ValueError("All sentences should be Unicode-encoded!")
sentence = sentence.strip().lower()
if self.break_replacement:
sentence = convert_linebreaks(sentence)
if self.remove_variation_selectors:
sentence = remove_variation_selectors(sentence)
# Split into words using simple whitespace splitting and convert
# Unicode. This is done to prevent word splitting issues with
# twokenize and Unicode
words = sentence.split()
converted_words = []
for w in words:
accept_sentence, c_w = self.convert_unicode_word(w)
# Unicode word detected and not allowed
if not accept_sentence:
return []
else:
converted_words.append(c_w)
sentence = ' '.join(converted_words)
words = tokenize(sentence)
words = [process_word(w) for w in words]
return words
def test_base(tests):
""" Base function for running tests.
"""
for (test, expected) in tests:
actual = tokenize(test)
assert actual == expected, \
"Tokenization of \'{}\' failed, expected: {}, actual: {}"\
.format(test, expected, actual)
def calculate_batchsize_maxlen(texts):
""" Calculates the maximum length in the provided texts and a suitable
batch size. Rounds up maxlen to the nearest multiple of ten.
# Arguments:
texts: List of inputs.
# Returns:
Batch size,
max length
"""
def roundup(x):
return int(math.ceil(x / 10.0)) * 10
# Calculate max length of sequences considered
# Adjust batch_size accordingly to prevent GPU overflow
lengths = [len(tokenize(t)) for t in texts]
maxlen = roundup(np.percentile(lengths, 80.0))
batch_size = 250 if maxlen <= 100 else 50
return batch_size, maxlen
def calculate_batchsize_maxlen(texts):
""" Calculates the maximum length in the provided texts and a suitable
batch size. Rounds up maxlen to the nearest multiple of ten.
# Arguments:
texts: List of inputs.
# Returns:
Batch size,
max length
"""
def roundup(x):
return int(math.ceil(x / 10.0)) * 10
# Calculate max length of sequences considered
# Adjust batch_size accordingly to prevent GPU overflow
lengths = [len(tokenize(t)) for t in texts]
maxlen = roundup(np.percentile(lengths, 80.0))
batch_size = 250 if maxlen <= 100 else 50
return batch_size, maxlen
PATH_OUR = '{}/{}/{}'.format(DIR, dset, FILENAME_OUR)
PATH_COMBINED = '{}/{}/{}'.format(DIR, dset, FILENAME_COMBINED)
with open(PATH_RAW, 'rb') as dataset:
if IS_PYTHON2:
data = pickle.load(dataset)
else:
data = pickle.load(dataset, fix_imports=True)
# Decode data
try:
texts = [unicode(x) for x in data['texts']]
except UnicodeDecodeError:
texts = [x.decode('utf-8') for x in data['texts']]
wg = WordGenerator(texts)
vb = VocabBuilder(wg)
vb.count_all_words()
# Calculate max length of sequences considered
# Adjust batch_size accordingly to prevent GPU overflow
lengths = [len(tokenize(t)) for t in texts]
maxlen = roundup(np.percentile(lengths, 80.0))
# Extract labels
labels = [x['label'] for x in data['info']]
convert_dataset(PATH_OWN, 50000, {})
convert_dataset(PATH_OUR, 0, vocab)
convert_dataset(PATH_COMBINED, 10000, vocab)