class TestStorageFileCache(object):
@pytest.fixture()
def sess(self, server):
self.url = server.application_url
self.cache = FileCache(STORAGE_FOLDER)
sess = CacheControl(requests.Session(), cache=self.cache)
return sess
def test_filecache_from_cache(self, sess):
response = sess.get(self.url)
assert not response.from_cache
response = sess.get(self.url)
assert response.from_cache
def test_key_length(self, sess):
"""
Hash table keys:
Most file systems have a 255 characters path limitation.
* Make sure hash method does not produce too long keys
* Ideally hash method generate fixed length keys
"""
url0 = url1 = 'http://example.org/res?a=1'
while len(url0) < 255:
url0 += randomdata()
url1 += randomdata()
assert len(self.cache.encode(url0)) < 200
assert len(self.cache.encode(url0)) == len(self.cache.encode(url1))
class TestStorageFileCache(object):
@pytest.fixture()
def sess(self, url, tmpdir):
self.url = url
self.cache = FileCache(str(tmpdir))
sess = CacheControl(requests.Session(), cache=self.cache)
yield sess
# closing session object
sess.close()
def test_filecache_from_cache(self, sess):
response = sess.get(self.url)
assert not response.from_cache
response = sess.get(self.url)
assert response.from_cache
def test_filecache_directory_not_exists(self, tmpdir, sess):
url = self.url + ''.join(sample(string.ascii_lowercase, randint(2, 4)))
# Make sure our cache dir doesn't exist
tmp_cache = tmpdir.join('missing', 'folder', 'name').strpath
assert not os.path.exists(tmp_cache)
self.cache.directory = tmp_cache
# trigger a cache save
sess.get(url)
# Now our cache dir does exist
assert os.path.exists(tmp_cache)
def test_filecache_directory_already_exists(self, tmpdir, sess):
"""
Assert no errors are raised when using a cache directory
that already exists on the filesystem.
"""
url = self.url + ''.join(sample(string.ascii_lowercase, randint(2, 4)))
# Make sure our cache dir DOES exist
tmp_cache = tmpdir.join('missing', 'folder', 'name').strpath
os.makedirs(tmp_cache, self.cache.dirmode)
assert os.path.exists(tmp_cache)
self.cache.directory = tmp_cache
# trigger a cache save
sess.get(url)
assert True # b/c no exceptions were raised
def test_key_length(self, sess):
"""
Hash table keys:
Most file systems have a 255 characters path limitation.
* Make sure hash method does not produce too long keys
* Ideally hash method generate fixed length keys
"""
url0 = url1 = 'http://example.org/res?a=1'
while len(url0) < 255:
url0 += randomdata()
url1 += randomdata()
assert len(self.cache.encode(url0)) < 200
assert len(self.cache.encode(url0)) == len(self.cache.encode(url1))
def test_cant_use_dir_and_lock_class(self, tmpdir):
with pytest.raises(ValueError):
FileCache(str(tmpdir), use_dir_lock=True, lock_class=object())
@pytest.mark.parametrize(
("value", "expected"),
[
(None, LockFile),
(True, MkdirLockFile),
(False, LockFile),
],
)
def test_simple_lockfile_arg(self, tmpdir, value, expected):
if value is not None:
cache = FileCache(str(tmpdir), use_dir_lock=value)
else:
cache = FileCache(str(tmpdir))
assert issubclass(cache.lock_class, expected)
cache.close()
def test_lock_class(self, tmpdir):
lock_class = object()
cache = FileCache(str(tmpdir), lock_class=lock_class)
assert cache.lock_class is lock_class
cache.close()
def test_filecache_with_delete_request(self, tmpdir, sess):
# verifies issue #155
url = self.url + ''.join(sample(string.ascii_lowercase, randint(2, 4)))
sess.delete(url)
assert True # test verifies no exceptions were raised
def test_filecache_with_put_request(self, tmpdir, sess):
# verifies issue #155
url = self.url + ''.join(sample(string.ascii_lowercase, randint(2, 4)))
sess.put(url)
assert True # test verifies no exceptions were raised
class TestStorageFileCache(object):
@pytest.fixture()
def sess(self, server, tmpdir):
self.url = server.application_url
self.cache = FileCache(str(tmpdir))
sess = CacheControl(requests.Session(), cache=self.cache)
return sess
def test_filecache_from_cache(self, sess):
response = sess.get(self.url)
assert not response.from_cache
response = sess.get(self.url)
assert response.from_cache
def test_filecache_directory_not_exists(self, tmpdir, sess):
url = self.url + ''.join(sample(string.ascii_lowercase, randint(2, 4)))
# Make sure our cache dir doesn't exist
tmp_cache = tmpdir.join('missing', 'folder', 'name').strpath
assert not os.path.exists(tmp_cache)
self.cache.directory = tmp_cache
# trigger a cache save
sess.get(url)
# Now our cache dir does exist
assert os.path.exists(tmp_cache)
def test_filecache_directory_already_exists(self, tmpdir, sess):
"""
Assert no errors are raised when using a cache directory
that already exists on the filesystem.
"""
url = self.url + ''.join(sample(string.ascii_lowercase, randint(2, 4)))
# Make sure our cache dir DOES exist
tmp_cache = tmpdir.join('missing', 'folder', 'name').strpath
os.makedirs(tmp_cache, self.cache.dirmode)
assert os.path.exists(tmp_cache)
self.cache.directory = tmp_cache
# trigger a cache save
sess.get(url)
assert True # b/c no exceptions were raised
def test_key_length(self, sess):
"""
Hash table keys:
Most file systems have a 255 characters path limitation.
* Make sure hash method does not produce too long keys
* Ideally hash method generate fixed length keys
"""
url0 = url1 = 'http://example.org/res?a=1'
while len(url0) < 255:
url0 += randomdata()
url1 += randomdata()
assert len(self.cache.encode(url0)) < 200
assert len(self.cache.encode(url0)) == len(self.cache.encode(url1))
class TestStorageFileCache(object):
@pytest.fixture()
def sess(self, server, tmpdir):
self.url = server.application_url
self.cache = FileCache(str(tmpdir))
sess = CacheControl(requests.Session(), cache=self.cache)
return sess
def test_filecache_from_cache(self, sess):
response = sess.get(self.url)
assert not response.from_cache
response = sess.get(self.url)
assert response.from_cache
def test_filecache_directory_not_exists(self, tmpdir, sess):
url = self.url + ''.join(sample(string.ascii_lowercase, randint(2, 4)))
# Make sure our cache dir doesn't exist
tmp_cache = tmpdir.join('missing', 'folder', 'name').strpath
assert not os.path.exists(tmp_cache)
self.cache.directory = tmp_cache
# trigger a cache save
sess.get(url)
# Now our cache dir does exist
assert os.path.exists(tmp_cache)
def test_filecache_directory_already_exists(self, tmpdir, sess):
"""
Assert no errors are raised when using a cache directory
that already exists on the filesystem.
"""
url = self.url + ''.join(sample(string.ascii_lowercase, randint(2, 4)))
# Make sure our cache dir DOES exist
tmp_cache = tmpdir.join('missing', 'folder', 'name').strpath
os.makedirs(tmp_cache, self.cache.dirmode)
assert os.path.exists(tmp_cache)
self.cache.directory = tmp_cache
# trigger a cache save
sess.get(url)
assert True # b/c no exceptions were raised
def test_key_length(self, sess):
"""
Hash table keys:
Most file systems have a 255 characters path limitation.
* Make sure hash method does not produce too long keys
* Ideally hash method generate fixed length keys
"""
url0 = url1 = 'http://example.org/res?a=1'
while len(url0) < 255:
url0 += randomdata()
url1 += randomdata()
assert len(self.cache.encode(url0)) < 200
assert len(self.cache.encode(url0)) == len(self.cache.encode(url1))
def test_max_bytes(self, tmpdir, sess):
"""
Test that caches the first url but not the second because
the maximum bytes have been reached for the cache.
"""
# use a cache with max_bytes set
max_bytes = 1400
self.cache = FileCache(str(tmpdir), max_bytes=max_bytes)
sess = CacheControl(requests.Session(), cache=self.cache)
url1 = self.url + ''.join(sample(string.ascii_lowercase, randint(2, 4)))
url2 = self.url + ''.join(sample(string.ascii_lowercase, randint(2, 4)))
assert url1 != url2
# fill up the cache with url1
response = sess.get(url1)
assert not response.from_cache
# make sure it got into the cache
response = sess.get(url1)
assert response.from_cache
# do url2 now
response = sess.get(url2)
assert not response.from_cache
# make sure url2 was NOT cached
response = sess.get(url2)
assert not response.from_cache
# clear the cache
response = sess.delete(url1)
assert not response.from_cache
# re-add to cache since bytes should be back to 0
response = sess.get(url1)
assert not response.from_cache
# verify from cache again
response = sess.get(url1)
assert response.from_cache
class TestStorageFileCache(object):
@pytest.fixture()
def sess(self, server, tmpdir):
self.url = server.application_url
self.cache = FileCache(str(tmpdir))
sess = CacheControl(requests.Session(), cache=self.cache)
return sess
def test_filecache_from_cache(self, sess):
response = sess.get(self.url)
assert not response.from_cache
response = sess.get(self.url)
assert response.from_cache
def test_filecache_directory_not_exists(self, tmpdir, sess):
url = self.url + "".join(sample(string.ascii_lowercase, randint(2, 4)))
# Make sure our cache dir doesn't exist
tmp_cache = tmpdir.join("missing", "folder", "name").strpath
assert not os.path.exists(tmp_cache)
self.cache.directory = tmp_cache
# trigger a cache save
sess.get(url)
# Now our cache dir does exist
assert os.path.exists(tmp_cache)
def test_filecache_directory_already_exists(self, tmpdir, sess):
"""
Assert no errors are raised when using a cache directory
that already exists on the filesystem.
"""
url = self.url + "".join(sample(string.ascii_lowercase, randint(2, 4)))
# Make sure our cache dir DOES exist
tmp_cache = tmpdir.join("missing", "folder", "name").strpath
os.makedirs(tmp_cache, self.cache.dirmode)
assert os.path.exists(tmp_cache)
self.cache.directory = tmp_cache
# trigger a cache save
sess.get(url)
assert True # b/c no exceptions were raised
def test_key_length(self, sess):
"""
Hash table keys:
Most file systems have a 255 characters path limitation.
* Make sure hash method does not produce too long keys
* Ideally hash method generate fixed length keys
"""
url0 = url1 = "http://example.org/res?a=1"
while len(url0) < 255:
url0 += randomdata()
url1 += randomdata()
assert len(self.cache.encode(url0)) < 200
assert len(self.cache.encode(url0)) == len(self.cache.encode(url1))
class TestStorageFileCache(object):
@pytest.fixture()
def sess(self, url, tmpdir):
self.url = url
self.cache = FileCache(str(tmpdir))
sess = CacheControl(requests.Session(), cache=self.cache)
return sess
def test_filecache_from_cache(self, sess):
response = sess.get(self.url)
assert not response.from_cache
response = sess.get(self.url)
assert response.from_cache
def test_filecache_directory_not_exists(self, tmpdir, sess):
url = self.url + ''.join(sample(string.ascii_lowercase, randint(2, 4)))
# Make sure our cache dir doesn't exist
tmp_cache = tmpdir.join('missing', 'folder', 'name').strpath
assert not os.path.exists(tmp_cache)
self.cache.directory = tmp_cache
# trigger a cache save
sess.get(url)
# Now our cache dir does exist
assert os.path.exists(tmp_cache)
def test_filecache_directory_already_exists(self, tmpdir, sess):
"""
Assert no errors are raised when using a cache directory
that already exists on the filesystem.
"""
url = self.url + ''.join(sample(string.ascii_lowercase, randint(2, 4)))
# Make sure our cache dir DOES exist
tmp_cache = tmpdir.join('missing', 'folder', 'name').strpath
os.makedirs(tmp_cache, self.cache.dirmode)
assert os.path.exists(tmp_cache)
self.cache.directory = tmp_cache
# trigger a cache save
sess.get(url)
assert True # b/c no exceptions were raised
def test_key_length(self, sess):
"""
Hash table keys:
Most file systems have a 255 characters path limitation.
* Make sure hash method does not produce too long keys
* Ideally hash method generate fixed length keys
"""
url0 = url1 = 'http://example.org/res?a=1'
while len(url0) < 255:
url0 += randomdata()
url1 += randomdata()
assert len(self.cache.encode(url0)) < 200
assert len(self.cache.encode(url0)) == len(self.cache.encode(url1))
def test_cant_use_dir_and_lock_class(self, tmpdir):
with pytest.raises(ValueError):
FileCache(str(tmpdir), use_dir_lock=True, lock_class=object())
@pytest.mark.parametrize(
("value", "expected"),
[
(None, LockFile),
(True, MkdirLockFile),
(False, LockFile),
],
)
def test_simple_lockfile_arg(self, tmpdir, value, expected):
if value is not None:
cache = FileCache(str(tmpdir), use_dir_lock=value)
else:
cache = FileCache(str(tmpdir))
assert issubclass(cache.lock_class, expected)
def test_lock_class(self, tmpdir):
lock_class = object()
cache = FileCache(str(tmpdir), lock_class=lock_class)
assert cache.lock_class is lock_class
def test_url_to_file_path(self, tmpdir):
cache = FileCache(str(tmpdir))
# We'll add a long sorted suffix so that unsorted queries have a low
# collision probability (about 3.8e-23 for each sorted/unsorted
# comparison).
letter_n_numbers = list(enumerate(string.ascii_lowercase[3:], start=4))
suff = '&' + '&'.join('%s=%s' % (k, v) for v, k in letter_n_numbers)
def get_param(url):
"""Mock losing order when processing params"""
shuffle(letter_n_numbers)
params = {k: v for v, k in letter_n_numbers}
url = url.replace(suff, '')
query = '&' + '&'.join('%s=%s' % item for item in params.items())
return url + query
urls = {
'no_query':
'http://example.com',
'unsorted_query':
'http://example.com?b=2&c=3&a=1' + suff,
'sorted_query':
'http://example.com?a=1&b=2&c=3' + suff,
'unsorted_empty_value':
'http://example.com?b=&c=&a=1' + suff,
'sorted_empty_value':
'http://example.com?a=1&b=&c=3' + suff,
'unsorted_repeated_key':
'http://example.com?b=2&c=3&b=0'
'&c=5&a=1' + suff,
'sorted_repeated_key':
'http://example.com?a=1&b=0&b=2&c=3&'
'c=5' + suff
}
unoquery = url_to_file_path(urls['no_query'], cache, sort_query=False)
snoquery = url_to_file_path(urls['no_query'], cache, sort_query=True)
assert unoquery == snoquery
urls.pop('no_query')
sortedpaths = {
urlname: url_to_file_path(urlvalue, cache, True)
for urlname, urlvalue in urls.items()
}
for key, value in urls.items():
if key.startswith('sorted'):
assert url_to_file_path(value, cache, True) == sortedpaths[key]
unsortedpaths = {
urlname: url_to_file_path(urlvalue, cache, False)
for urlname, urlvalue in urls.items()
}
for key, url in urls.items():
if key.startswith('unsorted'):
assert sortedpaths[key] != unsortedpaths[key]
else:
assert sortedpaths[key] == unsortedpaths[key]
# Randomize and sort params
sort_param = url_to_file_path(get_param(url), cache, True)
assert sort_param == sortedpaths[key]
# Randomize but don't sort params
unsort_param = url_to_file_path(get_param(url), cache, False)
assert unsort_param != unsortedpaths[key]
