def test_bulk_get(self):
tiles = self.tile_mgr.creator().create_tiles([Tile((0, 0, 2))])
eq_(len(tiles), 2 * 2)
eq_(self.file_cache.stored_tiles,
set([(0, 0, 2), (1, 0, 2), (0, 1, 2), (1, 1, 2)]))
for requested in [
self.source_base.requested, self.source_overlay.requested
]:
eq_(
set(requested),
set([
((-180.0, 0.0, -90.0, 90.0), (256, 256), SRS(4326)),
((-90.0, 0.0, 0.0, 90.0), (256, 256), SRS(4326)),
((-180.0, -90.0, -90.0, 0.0), (256, 256), SRS(4326)),
((-90.0, -90.0, 0.0, 0.0), (256, 256), SRS(4326)),
]))
def test_template_values(self):
template = CouchDBMDTemplate({'row': '{{y}}', 'tile_column': '{{x}}',
'zoom': '{{level}}', 'time': '{{timestamp}}', 'coord': '{{wgs_tile_centroid}}',
'datetime': '{{utc_iso}}', 'coord_webmerc': '{{tile_centroid}}'})
doc = template.doc(Tile((1, 0, 2)), tile_grid(3857))
assert_almost_equal(doc['time'], time.time(), 2)
assert 'timestamp' not in doc
eq_(doc['row'], 0)
eq_(doc['tile_column'], 1)
eq_(doc['zoom'], 2)
assert_almost_equal(doc['coord'][0], -45.0)
assert_almost_equal(doc['coord'][1], -79.17133464081945)
assert_almost_equal(doc['coord_webmerc'][0], -5009377.085697311)
assert_almost_equal(doc['coord_webmerc'][1], -15028131.257091932)
assert re.match('20\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ', doc['datetime']), doc['datetime']
def test_concurrent(self):
def do_it():
self.tile_mgr.creator().create_tiles(
[Tile((0, 0, 1)), Tile((1, 0, 1))])
threads = [threading.Thread(target=do_it) for _ in range(3)]
[t.start() for t in threads]
[t.join() for t in threads]
eq_(self.file_cache.stored_tiles, set([(0, 0, 1), (1, 0, 1)]))
eq_(self.file_cache.loaded_tiles,
counting_set([(0, 0, 1), (1, 0, 1), (0, 0, 1), (1, 0, 1)]))
eq_(self.source.requested,
[((-180.0, -90.0, 180.0, 90.0), (512, 256), SRS(4326))])
assert os.path.exists(self.file_cache.tile_location(Tile((0, 0, 1))))
def test_defragmentation_empty_bundle(self):
cache = self.cache_class(self.cache_dir)
t = Tile((5000, 1000, 12),
ImageSource(BytesIO(b'a' * 60 * 1024),
image_opts=ImageOptions(format='image/png')))
cache.store_tile(t)
cache.remove_tile(t)
fname = os.path.join(self.cache_dir, 'L12', 'R0380C1380.bundle')
assert os.path.exists(fname)
logger = mockProgressLog()
defrag_compact_cache(cache, min_bytes=50000, log_progress=logger)
assert not os.path.exists(fname)
def test_scaled_tiles(self, name, file_cache, tile_locker, rescale_tiles, tiles, store, expected_load, output):
res = [
1.40625, # 0
0.703125, # 1
0.3515625, # 2
0.17578125, # 3
0.087890625, # 4
0.0439453125, # 5
0.02197265625, # 6
0.010986328125, # 7
0.007, # 8 additional resolution to test unregular grids
0.0054931640625, # 9
0.00274658203125, # 10
]
grid = TileGrid(SRS(4326), origin='sw', bbox=[-180, -90, 180, 90], res=res)
image_opts = ImageOptions(format='image/png', resampling='nearest')
tm = TileManager(
grid, file_cache, [], 'png',
locker=tile_locker,
image_opts=image_opts,
rescale_tiles=rescale_tiles,
)
if store:
colors = set()
if output == "partial":
colors.add((255, 255, 255))
for i, t in enumerate(store):
color = (150+i*35, 5+i*35, 5+i*35)
colors.add(color)
tile = Tile(t, ImageSource(create_tmp_image_buf((256, 256), color=color)))
file_cache.store_tile(tile)
loaded_tiles = tm.load_tile_coords(tiles)
assert not is_blank(loaded_tiles)
assert len(loaded_tiles) == len(tiles)
got_colors = set()
for t in loaded_tiles:
got_colors.update([c for _, c in t.source.as_image().getcolors()])
assert got_colors == colors
else:
loaded_tiles = tm.load_tile_coords(tiles)
assert is_blank(loaded_tiles) == (output == "blank")
assert len(loaded_tiles.tiles) == len(tiles)
assert file_cache.stored_tiles == set(store)
assert file_cache.loaded_tiles == counting_set(expected_load)
def defrag_compact_cache(cache,
min_percent=0.1,
min_bytes=1024 * 1024,
log_progress=None,
dry_run=False):
bundles = glob.glob(
os.path.join(cache.cache_dir, 'L??', 'R????C????.bundle'))
for i, bundle_file in enumerate(bundles):
offset = bundle_offset(bundle_file)
b = cache.bundle_class(bundle_file.rstrip('.bundle'), offset)
size, file_size = b.size()
defrag = 1 - float(size) / file_size
defrag_bytes = file_size - size
skip = False
if defrag < min_percent or defrag_bytes < min_bytes:
skip = True
if log_progress:
log_progress.log(
fname=bundle_file,
fragmentation=defrag * 100,
fragmentation_bytes=defrag_bytes,
num=i + 1,
total=len(bundles),
defrag=not skip,
)
if skip or dry_run:
continue
tmp_bundle = os.path.join(cache.cache_dir, 'tmp_defrag')
defb = cache.bundle_class(tmp_bundle, offset)
for y in range(128):
tiles = [Tile((x, y, 0)) for x in range(128)]
b.load_tiles(tiles)
tiles = [t for t in tiles if t.source]
if tiles:
defb.store_tiles(tiles)
os.rename(tmp_bundle + '.bundle', bundle_file)
if os.path.exists(tmp_bundle + '.bundlx'):
os.rename(tmp_bundle + '.bundlx', bundle_file[:-1] + 'x')
os.unlink(tmp_bundle + '.lck')
def test_concurrent(self, tile_mgr, mock_file_cache, slow_source):
def do_it():
tile_mgr.creator().create_tiles([Tile((0, 0, 1)), Tile((1, 0, 1))])
threads = [threading.Thread(target=do_it) for _ in range(3)]
[t.start() for t in threads]
[t.join() for t in threads]
assert mock_file_cache.stored_tiles == set([(0, 0, 1), (1, 0, 1)])
assert mock_file_cache.loaded_tiles == counting_set([(0, 0, 1),
(1, 0, 1),
(0, 0, 1),
(1, 0, 1)])
assert slow_source.requested == \
[((-180.0, -90.0, 180.0, 90.0), (512, 256), SRS(4326))]
assert os.path.exists(mock_file_cache.tile_location(Tile((0, 0, 1))))
def test_template_values(self):
template = CouchDBMDTemplate({
'row': '{{y}}',
'tile_column': '{{x}}',
'zoom': '{{level}}',
'time': '{{timestamp}}',
'coord': '{{wgs_tile_centroid}}',
'datetime': '{{utc_iso}}',
'coord_webmerc': '{{tile_centroid}}'
})
doc = template.doc(Tile((1, 0, 2)), tile_grid(3857))
assert doc['time'] == pytest.approx(time.time(), 0.1)
assert 'timestamp' not in doc
assert doc['row'] == 0
assert doc['tile_column'] == 1
assert doc['zoom'] == 2
assert doc['coord'][0] == pytest.approx(-45.0)
assert doc['coord'][1] == pytest.approx(-79.17133464081945)
assert doc['coord_webmerc'][0] == pytest.approx(-5009377.085697311)
assert doc['coord_webmerc'][1] == pytest.approx(-15028131.25709193)
assert re.match(r'20\d\d-\d\d-\d\dT\d\d:\d\d:\d\dZ',
doc['datetime']), doc['datetime']
def test_defragmentation_min_bytes(self):
cache = self.cache_class(self.cache_dir)
for _ in range(2):
t = Tile((5000, 1000, 12),
ImageSource(BytesIO(b'a' * 60*1024), image_opts=ImageOptions(format='image/png')))
cache.store_tile(t)
logger = mockProgressLog()
fname = os.path.join(self.cache_dir, 'L12', 'R0380C1380.bundle')
before = os.path.getsize(fname)
defrag_compact_cache(cache, log_progress=logger)
assert len(logger.logs) == 1
eq_(logger.logs[0]['defrag'], False)
after = os.path.getsize(fname)
assert before == after
logger = mockProgressLog()
before = os.path.getsize(fname)
defrag_compact_cache(cache, min_bytes=50000, log_progress=logger)
assert len(logger.logs) == 1
eq_(logger.logs[0]['defrag'], True)
after = os.path.getsize(fname)
assert after < before
def test_load_tile_not_cached(self):
tile = Tile((0, 0, 4))
assert not self.cache.load_tile(tile)
assert tile.source is None
assert tile.is_missing()
def test_is_cached_none(self):
assert self.cache.is_cached(Tile(None))
def test_is_cached_miss(self):
assert not self.cache.is_cached(Tile((0, 0, 4)))
def test_load_1001_tiles(self):
assert_raises(CacheBackendError, self.cache.load_tiles, [Tile((19, 1, 1))] * 1001)
def test_load_metadata_missing_tile(self):
tile = Tile((0, 0, 0))
self.cache.load_tile_metadata(tile)
assert tile.timestamp == 0
assert tile.size == 0
def do_it():
tile_mgr.creator().create_tiles([Tile((0, 0, 1)), Tile((1, 0, 1))])
def test_create_tile_multiple_fragmented(self, tile_mgr, mock_file_cache, mock_wms_client):
tile_mgr.creator().create_tiles([Tile((4, 0, 3)), Tile((5, 2, 3))])
assert mock_file_cache.stored_tiles == \
set([(4, 0, 3), (4, 1, 3), (4, 2, 3), (5, 0, 3), (5, 1, 3), (5, 2, 3)])
assert sorted(mock_wms_client.requested) == \
[((-1.7578125, -90, 91.7578125, 46.7578125), (532, 778), SRS(4326))]
def test_get_single(self, tile_mgr, file_cache, slow_source):
tile_mgr.creator().create_tiles([Tile((0, 0, 1)), Tile((1, 0, 1))])
assert file_cache.stored_tiles == set([(0, 0, 1), (1, 0, 1)])
assert slow_source.requested == \
[((-180.0, -90.0, 180.0, 90.0), (512, 256), SRS(4326))]
def test_create_tiles(self, tile_mgr, mock_file_cache, mock_tile_client):
tile_mgr.creator().create_tiles([Tile((1, 0, 1))])
assert mock_file_cache.stored_tiles == set([(1, 0, 1)])
assert mock_tile_client.requested_tiles == [(0, 0, 0)]
def test_bulk_get_error(self, tile_mgr, source_base):
tile_mgr.sources = [source_base, ErrorSource()]
try:
tile_mgr.creator().create_tiles([Tile((0, 0, 2))])
except Exception as ex:
assert ex.args[0] == "source error"
def test_create_tiles_out_of_bounds(self, tile_mgr):
with pytest.raises(InvalidSourceQuery):
tile_mgr.creator().create_tiles([Tile((0, 0, 0))])
def test_load_empty_tileset(self):
assert self.cache.load_tiles([Tile(None)]) == True
assert self.cache.load_tiles([Tile(None), Tile(None), Tile(None)]) == True
def test_create_tile(self, tile_mgr, mock_file_cache, mock_source):
tile_mgr.creator().create_tiles([Tile((0, 0, 1)), Tile((1, 0, 1))])
assert mock_file_cache.stored_tiles == set([(0, 0, 1), (1, 0, 1)])
assert sorted(mock_source.requested) == \
[((-180.0, -90.0, 0.0, 90.0), (256, 256), SRS(4326)),
((0.0, -90.0, 180.0, 90.0), (256, 256), SRS(4326))]
def create_another_tile(self, coord=(0, 0, 4)):
return Tile(coord,
ImageSource(tile_image2,
image_opts=ImageOptions(format='image/png')))
def test_same_lock_for_meta_tile(self, tile_mgr):
assert tile_mgr.lock(Tile((0, 0, 1))).lock_file == \
tile_mgr.lock(Tile((1, 0, 1))).lock_file
def test_is_cached_hit(self):
tile = self.create_tile()
self.create_cached_tile(tile)
assert self.cache.is_cached(Tile((0, 0, 4)))
def test_locks_for_meta_tiles(self, tile_mgr):
assert tile_mgr.lock(Tile((0, 0, 2))).lock_file != \
tile_mgr.lock(Tile((2, 0, 2))).lock_file
def test_load_tile_none(self):
assert self.cache.load_tile(Tile(None))
def test_create_tile_first_level(self, tile_mgr, mock_file_cache, mock_wms_client):
tile_mgr.creator().create_tiles([Tile((0, 0, 1)), Tile((1, 0, 1))])
assert mock_file_cache.stored_tiles == set([(0, 0, 1), (1, 0, 1)])
assert mock_wms_client.requested == \
[((-180.0, -90.0, 180.0, 90.0), (512, 256), SRS(4326))]
def test_load_tile_cached(self):
tile = self.create_tile()
self.create_cached_tile(tile)
tile = Tile((0, 0, 4))
assert self.cache.load_tile(tile) == True
assert not tile.is_missing()
def test_create_tile(self, tile_mgr, mock_file_cache, mock_wms_client):
tile_mgr.creator().create_tiles([Tile((0, 0, 2))])
assert mock_file_cache.stored_tiles == \
set([(0, 0, 2), (1, 0, 2), (0, 1, 2), (1, 1, 2)])
assert sorted(mock_wms_client.requested) == \
[((-180.0, -90.0, 0.0, 90.0), (512, 512), SRS(4326))]
