def test_caching(getmtime_mock):
URL = 'http://www.google.com'
DATA = b'This is google.com!'
clock = ThreadedClock()
getmtime_mock.return_value = 0
opener = MockOpener(DATA)
web = CachedWeb(clock=clock, opener=opener)
assert not os.path.exists(web.translate_url(URL))
with contextlib.closing(web.open(URL)) as fp:
assert fp.read() == DATA
assert os.path.exists(web.translate_url(URL))
assert opener.opened.is_set()
opener.clear()
assert web.expired(URL, ttl=0.5) is False
clock.tick(1)
assert web.expired(URL, ttl=0.5)
with contextlib.closing(web.open(URL)) as fp:
assert fp.read() == DATA
assert not opener.opened.is_set()
with contextlib.closing(web.open(URL, ttl=0.5)) as fp:
assert fp.read() == DATA
assert opener.opened.is_set(), 'expect expired url to cause http get'
def test_caching():
URL = 'http://www.google.com'
DATA = b'This is google.com!'
clock = ThreadedClock()
m = mox.Mox()
m.StubOutWithMock(os.path, 'getmtime')
os.path.getmtime(mox.IgnoreArg()).MultipleTimes().AndReturn(0)
m.ReplayAll()
opener = MockOpener(DATA)
web = CachedWeb(clock=clock, opener=opener)
assert not os.path.exists(web.translate_url(URL))
with contextlib.closing(web.open(URL)) as fp:
assert fp.read() == DATA
assert os.path.exists(web.translate_url(URL))
assert opener.opened.is_set()
opener.clear()
assert web.expired(URL, ttl=0.5) is False
clock.tick(1)
assert web.expired(URL, ttl=0.5)
with contextlib.closing(web.open(URL)) as fp:
assert fp.read() == DATA
assert not opener.opened.is_set()
with contextlib.closing(web.open(URL, ttl=0.5)) as fp:
assert fp.read() == DATA
assert opener.opened.is_set(), 'expect expired url to cause http get'
m.UnsetStubs()
m.VerifyAll()
def test_connect_timeout_using_open():
URL = 'http://www.google.com'
DATA = b'This is google.com!'
clock = ThreadedClock()
opener = MockOpener(DATA)
web = CachedWeb(clock=clock, opener=opener)
assert not os.path.exists(web.translate_url(URL))
with pytest.raises(FetchError):
with contextlib.closing(web.open(URL, conn_timeout=0)):
pass
with contextlib.closing(web.open(URL, conn_timeout=0.01)) as fp:
assert fp.read() == DATA
def test_local_cached_open():
cache = safe_mkdtemp()
web = CachedWeb(cache=cache)
source_dir = safe_mkdtemp()
source = os.path.join(source_dir, 'filename')
with open(source, 'wb') as fp:
fp.write(b'data')
with contextlib.closing(web.open(source)) as cached_fp1:
assert b'data' == cached_fp1.read()
with contextlib.closing(web.open(source)) as cached_fp2:
assert b'data' == cached_fp2.read()
def test_connect_timeout_using_open():
URL = 'http://www.google.com'
DATA = b'This is google.com!'
clock = ThreadedClock()
opener = MockOpener(DATA)
web = CachedWeb(clock=clock, opener=opener)
assert not os.path.exists(web.translate_url(URL))
with pytest.raises(FetchError):
with contextlib.closing(web.open(URL, conn_timeout=0)):
pass
with contextlib.closing(web.open(URL, conn_timeout=0.01)) as fp:
assert fp.read() == DATA
def test_local_cached_open():
cache = safe_mkdtemp()
web = CachedWeb(cache=cache)
source_dir = safe_mkdtemp()
source = os.path.join(source_dir, "filename")
with open(source, "wb") as fp:
fp.write(b"data")
with contextlib.closing(web.open(source)) as cached_fp1:
assert b"data" == cached_fp1.read()
with contextlib.closing(web.open(source)) as cached_fp2:
assert b"data" == cached_fp2.read()
def test_caching(getmtime_mock):
URL = 'http://www.google.com'
DATA = b'This is google.com!'
clock = ThreadedClock()
getmtime_mock.return_value = 0
opener = MockOpener(DATA)
web = CachedWeb(clock=clock, opener=opener)
assert not os.path.exists(web.translate_url(URL))
with contextlib.closing(web.open(URL)) as fp:
assert fp.read() == DATA
assert os.path.exists(web.translate_url(URL))
assert opener.opened.is_set()
opener.clear()
assert web.expired(URL, ttl=0.5) is False
clock.tick(1)
assert web.expired(URL, ttl=0.5)
with contextlib.closing(web.open(URL)) as fp:
assert fp.read() == DATA
assert not opener.opened.is_set()
with contextlib.closing(web.open(URL, ttl=0.5)) as fp:
assert fp.read() == DATA
assert opener.opened.is_set(), 'expect expired url to cause http get'