def test_caching(getmtime_mock): URL = 'http://www.google.com' DATA = b'This is google.com!' clock = ThreadedClock() getmtime_mock.return_value = 0 opener = MockOpener(DATA) web = CachedWeb(clock=clock, opener=opener) assert not os.path.exists(web.translate_url(URL)) with contextlib.closing(web.open(URL)) as fp: assert fp.read() == DATA assert os.path.exists(web.translate_url(URL)) assert opener.opened.is_set() opener.clear() assert web.expired(URL, ttl=0.5) is False clock.tick(1) assert web.expired(URL, ttl=0.5) with contextlib.closing(web.open(URL)) as fp: assert fp.read() == DATA assert not opener.opened.is_set() with contextlib.closing(web.open(URL, ttl=0.5)) as fp: assert fp.read() == DATA assert opener.opened.is_set(), 'expect expired url to cause http get'
def test_caching(): URL = 'http://www.google.com' DATA = b'This is google.com!' clock = ThreadedClock() m = mox.Mox() m.StubOutWithMock(os.path, 'getmtime') os.path.getmtime(mox.IgnoreArg()).MultipleTimes().AndReturn(0) m.ReplayAll() opener = MockOpener(DATA) web = CachedWeb(clock=clock, opener=opener) assert not os.path.exists(web.translate_url(URL)) with contextlib.closing(web.open(URL)) as fp: assert fp.read() == DATA assert os.path.exists(web.translate_url(URL)) assert opener.opened.is_set() opener.clear() assert web.expired(URL, ttl=0.5) is False clock.tick(1) assert web.expired(URL, ttl=0.5) with contextlib.closing(web.open(URL)) as fp: assert fp.read() == DATA assert not opener.opened.is_set() with contextlib.closing(web.open(URL, ttl=0.5)) as fp: assert fp.read() == DATA assert opener.opened.is_set(), 'expect expired url to cause http get' m.UnsetStubs() m.VerifyAll()
def test_connect_timeout_using_open(): URL = 'http://www.google.com' DATA = b'This is google.com!' clock = ThreadedClock() opener = MockOpener(DATA) web = CachedWeb(clock=clock, opener=opener) assert not os.path.exists(web.translate_url(URL)) with pytest.raises(FetchError): with contextlib.closing(web.open(URL, conn_timeout=0)): pass with contextlib.closing(web.open(URL, conn_timeout=0.01)) as fp: assert fp.read() == DATA
def test_local_cached_open(): cache = safe_mkdtemp() web = CachedWeb(cache=cache) source_dir = safe_mkdtemp() source = os.path.join(source_dir, 'filename') with open(source, 'wb') as fp: fp.write(b'data') with contextlib.closing(web.open(source)) as cached_fp1: assert b'data' == cached_fp1.read() with contextlib.closing(web.open(source)) as cached_fp2: assert b'data' == cached_fp2.read()
def test_local_cached_open(): cache = safe_mkdtemp() web = CachedWeb(cache=cache) source_dir = safe_mkdtemp() source = os.path.join(source_dir, "filename") with open(source, "wb") as fp: fp.write(b"data") with contextlib.closing(web.open(source)) as cached_fp1: assert b"data" == cached_fp1.read() with contextlib.closing(web.open(source)) as cached_fp2: assert b"data" == cached_fp2.read()