def __test__ (ut) : from nlplib.general.unittest import mock urls = {'a' : mock(geturl=lambda : 'a', read=lambda : b'aaa'), 'b' : mock(geturl=lambda : 'b', read=lambda : b'bbb'), 'c' : mock(geturl=lambda : 'cc', read=lambda : b'ccc'), 'd' : mock(geturl=lambda : 'd', read=lambda : b'ddd'), b'\xc3\x9c'.decode() : mock(geturl=lambda : b'\xc3\x9c'.decode(), read=lambda : b'\xc3\x9c\xc3\x9c\xc3\x9c')} def mocked (cls, urls=urls) : # A mockup is made, so that this test doesn't depend on external resources (the internet). def open_ (url) : def enter (*args, **kw) : try : return urls[url] except KeyError : raise URLError('Something went horribly horribly wrong!!!') return mock(__enter__=enter, __exit__=lambda *args, **kw: None) class Mocked (cls) : def _make_opener (self) : return mock(open=open_) return Mocked all_ = {Response('a', 'aaa', 'a'), Response('b', 'bbb', 'b'), Response('cc', 'ccc', 'c'), Response('d', 'ddd', 'd')} for kw in [{'serial' : False}, {'serial' : True}] : scraped = mocked(Scraped)(['a', 'b', 'c'], **kw) ut.assert_equal(set(scraped), {Response('a', 'aaa', 'a'), Response('b', 'bbb', 'b'), Response('cc', 'ccc', 'c')}) scraped.urls.append('d') ut.assert_equal(set(scraped), all_) # Test with generated URLs gen_urls = (url for url in 'abcd') gen_scraped = mocked(Scraped)(gen_urls, **kw) ut.assert_equal(set(gen_scraped), all_) ut.assert_equal(set(gen_scraped), set()) _test_with_memory_errors(ut, mock, mocked, **kw) _test_with_unicode(ut, mocked, **kw) _test_revisit(ut, mocked, **kw) _test_when_unavailable(ut, mocked, **kw) _test_with_infinite_generator(mocked, **kw)
def open_ (url) : def enter (*args, **kw) : try : return urls[url] except KeyError : raise URLError('Something went horribly horribly wrong!!!') return mock(__enter__=enter, __exit__=lambda *args, **kw: None)
def _test_with_memory_errors (ut, mock, mocked, **kw) : def memory_error_while_reading () : raise MemoryError('This is huge!') urls = {'a' : mock(geturl=lambda : 'a', read=memory_error_while_reading)} ut.assert_raises(lambda : list(mocked(Scraped, urls=urls)(['a'], **kw)), CouldNotOpenURL) ut.assert_doesnt_raise(lambda : list(mocked(Scraped, urls=urls)(['a'], silent=True, **kw)), CouldNotOpenURL)
def __test__ (ut) : from nlplib.general.unittest import mock ut.assert_equal(count('foo'), 1) ut.assert_equal(count('bar', default=-3), -3) ut.assert_equal(count(mock(count=35)), 35) ut.assert_equal(levenshtein_distance('hello', 'hello'), 0) ut.assert_equal(levenshtein_distance('hello', 'helo'), 1) ut.assert_equal(levenshtein_distance('hello', 'helo'), 1) ut.assert_equal(levenshtein_distance([0, 1, 2], [0, 1, 2]), 0) ut.assert_equal(levenshtein_distance([0, 1, 2], [0, 2]), 1)
def _make_opener (self) : return mock(open=open_)