def __test__ (ut) :
from nlplib.general.unittest import mock
urls = {'a' : mock(geturl=lambda : 'a', read=lambda : b'aaa'),
'b' : mock(geturl=lambda : 'b', read=lambda : b'bbb'),
'c' : mock(geturl=lambda : 'cc', read=lambda : b'ccc'),
'd' : mock(geturl=lambda : 'd', read=lambda : b'ddd'),
b'\xc3\x9c'.decode() : mock(geturl=lambda : b'\xc3\x9c'.decode(),
read=lambda : b'\xc3\x9c\xc3\x9c\xc3\x9c')}
def mocked (cls, urls=urls) :
# A mockup is made, so that this test doesn't depend on external resources (the internet).
def open_ (url) :
def enter (*args, **kw) :
try :
return urls[url]
except KeyError :
raise URLError('Something went horribly horribly wrong!!!')
return mock(__enter__=enter, __exit__=lambda *args, **kw: None)
class Mocked (cls) :
def _make_opener (self) :
return mock(open=open_)
return Mocked
all_ = {Response('a', 'aaa', 'a'), Response('b', 'bbb', 'b'), Response('cc', 'ccc', 'c'),
Response('d', 'ddd', 'd')}
for kw in [{'serial' : False}, {'serial' : True}] :
scraped = mocked(Scraped)(['a', 'b', 'c'], **kw)
ut.assert_equal(set(scraped),
{Response('a', 'aaa', 'a'), Response('b', 'bbb', 'b'), Response('cc', 'ccc', 'c')})
scraped.urls.append('d')
ut.assert_equal(set(scraped), all_)
# Test with generated URLs
gen_urls = (url for url in 'abcd')
gen_scraped = mocked(Scraped)(gen_urls, **kw)
ut.assert_equal(set(gen_scraped), all_)
ut.assert_equal(set(gen_scraped), set())
_test_with_memory_errors(ut, mock, mocked, **kw)
_test_with_unicode(ut, mocked, **kw)
_test_revisit(ut, mocked, **kw)
_test_when_unavailable(ut, mocked, **kw)
_test_with_infinite_generator(mocked, **kw)
def open_ (url) :
def enter (*args, **kw) :
try :
return urls[url]
except KeyError :
raise URLError('Something went horribly horribly wrong!!!')
return mock(__enter__=enter, __exit__=lambda *args, **kw: None)
def _test_with_memory_errors (ut, mock, mocked, **kw) :
def memory_error_while_reading () :
raise MemoryError('This is huge!')
urls = {'a' : mock(geturl=lambda : 'a', read=memory_error_while_reading)}
ut.assert_raises(lambda : list(mocked(Scraped, urls=urls)(['a'], **kw)), CouldNotOpenURL)
ut.assert_doesnt_raise(lambda : list(mocked(Scraped, urls=urls)(['a'], silent=True, **kw)), CouldNotOpenURL)
def __test__ (ut) :
from nlplib.general.unittest import mock
ut.assert_equal(count('foo'), 1)
ut.assert_equal(count('bar', default=-3), -3)
ut.assert_equal(count(mock(count=35)), 35)
ut.assert_equal(levenshtein_distance('hello', 'hello'), 0)
ut.assert_equal(levenshtein_distance('hello', 'helo'), 1)
ut.assert_equal(levenshtein_distance('hello', 'helo'), 1)
ut.assert_equal(levenshtein_distance([0, 1, 2], [0, 1, 2]), 0)
ut.assert_equal(levenshtein_distance([0, 1, 2], [0, 2]), 1)
def _make_opener (self) :
return mock(open=open_)
