def test_remove_all_whitespace():
# set up
test_str = 'a b c d'
expected_results = 'abcd'
# execution
actual_results = string_utils.remove_all_whitespace(test_str)
# asserts
nt.eq_(expected_results, actual_results)
def test_remove_all_whitespace_str_param_none_return_none():
# set up
test_str = None
expected_results = None
# execution
actual_results = string_utils.remove_all_whitespace(test_str)
# asserts
nt.eq_(expected_results, actual_results)
def test_remove_all_whitespace_str_param_none_return_none():
# set up
test_str = None
expected_results = None
# execution
actual_results = string_utils.remove_all_whitespace(test_str)
# asserts
nt.eq_(expected_results, actual_results)
def test_remove_all_whitespace():
# set up
test_str = 'a b c d'
expected_results = 'abcd'
# execution
actual_results = string_utils.remove_all_whitespace(test_str)
# asserts
nt.eq_(expected_results, actual_results)
def _k_shingles_generator(doc_generator, size=DEFAULT_SHINGLE_SIZE):
"""
Generator that yields set of k-shingles
:param doc_generator: generator that returns documents as strings
:param size: size of k-shingles
:return: yields list of shingles (list of strings) and original document shingles produced from, yields
None if it can't generate list of shingles
"""
for doc in doc_generator:
if doc:
#step 1: remove all white space from the string
cleaned_doc = strings_utils.remove_all_whitespace(doc)
#step 2: remove punctuation and make all lower case
cleaned_doc = strings_utils.normalize(cleaned_doc)
#step 3: get shingles list
yield _get_list_of_shingles(cleaned_doc, size), doc
else:
yield None
