def test_metatile_file_timing(self):
from time import gmtime, time
from tilequeue.metatile import metatiles_are_equal
# tilequeue's "GET before PUT" optimisation relies on being able to
# fetch a tile from S3 and compare it to the one that was just
# generated. to do this, we should try to make the tiles as similar
# as possible across multiple runs.
json = "{\"json\":true}"
tiles = [
dict(tile=json,
coord=Coordinate(0, 0, 0),
format=json_format,
layer='all')
]
when_will_then_be_now = 10
t = time()
now = gmtime(t)[0:6]
then = gmtime(t - when_will_then_be_now)[0:6]
metatile_1 = make_metatiles(1, tiles, then)
metatile_2 = make_metatiles(1, tiles, now)
self.assertTrue(
metatiles_are_equal(metatile_1[0]['tile'], metatile_2[0]['tile']))
def test_make_metatiles_multiple_coordinates(self):
# checks that we can make a single metatile which contains multiple
# coordinates. this is used for "512px" tiles as well as cutting out
# z17+ tiles and storing them in the z16 (z15 if "512px") metatile.
json = "{\"json\":true}"
tiles = [
# NOTE: coordinates are (y, x, z), possibly the most confusing
# possible permutation.
dict(tile=json, coord=Coordinate(123, 456, 17),
format=json_format, layer='all'),
dict(tile=json, coord=Coordinate(123, 457, 17),
format=json_format, layer='all'),
]
metatiles = make_metatiles(1, tiles)
self.assertEqual(1, len(metatiles))
meta = metatiles[0]
# NOTE: Coordinate(y, x, z)
coord = Coordinate(61, 228, 16)
self.assertEqual(meta['coord'], coord)
self.assertEqual('all', meta['layer'])
self.assertEqual(zip_format, meta['format'])
buf = StringIO.StringIO(meta['tile'])
with zipfile.ZipFile(buf, mode='r') as z:
self.assertEqual(json, z.open('1/0/1.json').read())
self.assertEqual(json, z.open('1/1/1.json').read())
# test extracting as well
self.assertEqual(json, extract_metatile(
buf, json_format, offset=Coordinate(zoom=1, column=0, row=1)))
self.assertEqual(json, extract_metatile(
buf, json_format, offset=Coordinate(zoom=1, column=1, row=1)))
def save_tiles(self, tiles):
async_jobs = []
if self.metatile_size:
metatiles = make_metatiles(self.metatile_size, tiles)
# allow the metatile to be stored, or both the metatile
# and originals, which allows for a smooth cutover.
if self.store_metatile_and_originals:
tiles.extend(metatiles)
else:
tiles = metatiles
for tile in tiles:
async_result = self.io_pool.apply_async(
write_tile_if_changed, (
self.store,
tile['tile'],
# important to use the coord from the
# formatted tile here, because we could have
# cut children tiles that have separate zooms
# too
tile['coord'],
tile['format'],
tile['layer']))
async_jobs.append(async_result)
return async_jobs
def test_make_metatiles_multiple_coordinates(self):
# we need to be able to handle this so that we can do "cut out"
# overzoomed tiles at z>16.
json = "{\"json\":true}"
tiles = [
dict(tile=json, coord=Coordinate(17, 123, 456),
format=json_format, layer='all'),
dict(tile=json, coord=Coordinate(17, 123, 457),
format=json_format, layer='all'),
]
metatiles = make_metatiles(1, tiles)
self.assertEqual(2, len(metatiles))
coords = set([Coordinate(17, 123, 456), Coordinate(17, 123, 457)])
for meta in metatiles:
self.assertTrue(meta['coord'] in coords)
coords.remove(meta['coord'])
self.assertEqual('all', meta['layer'])
self.assertEqual(zip_format, meta['format'])
buf = StringIO.StringIO(meta['tile'])
with zipfile.ZipFile(buf, mode='r') as z:
self.assertEqual(json, z.open('0/0/0.json').read())
# check all coords were consumed
self.assertEqual(0, len(coords))
def test_extract_metatiles_single(self):
json = "{\"json\":true}"
tile = dict(tile=json, coord=Coordinate(0, 0, 0),
format=json_format, layer='all')
metatiles = make_metatiles(1, [tile])
self.assertEqual(1, len(metatiles))
buf = StringIO.StringIO(metatiles[0]['tile'])
extracted = extract_metatile(buf, json_format)
self.assertEqual(json, extracted)
def test_extract_metatiles_single(self):
json = "{\"json\":true}"
tile = dict(tile=json, coord=Coordinate(0, 0, 0),
format=json_format, layer='all')
metatiles = make_metatiles(1, [tile])
self.assertEqual(1, len(metatiles))
buf = StringIO.StringIO(metatiles[0]['tile'])
extracted = extract_metatile(1, buf, tile)
self.assertEqual(json, extracted)
def test_make_metatiles_single(self):
json = "{\"json\":true}"
tiles = [dict(tile=json, coord=Coordinate(0, 0, 0),
format=json_format, layer='all')]
metatiles = make_metatiles(1, tiles)
self.assertEqual(1, len(metatiles))
self.assertEqual(Coordinate(0, 0, 0), metatiles[0]['coord'])
self.assertEqual('all', metatiles[0]['layer'])
self.assertEqual(zip_format, metatiles[0]['format'])
buf = StringIO.StringIO(metatiles[0]['tile'])
with zipfile.ZipFile(buf, mode='r') as z:
self.assertEqual(json, z.open('0/0/0.json').read())
def test_make_metatiles_single(self):
json = "{\"json\":true}"
tiles = [dict(tile=json, coord=Coordinate(0, 0, 0),
format=json_format, layer='all')]
metatiles = make_metatiles(1, tiles)
self.assertEqual(1, len(metatiles))
self.assertEqual(Coordinate(0, 0, 0), metatiles[0]['coord'])
self.assertEqual('all', metatiles[0]['layer'])
self.assertEqual(zip_format, metatiles[0]['format'])
buf = StringIO.StringIO(metatiles[0]['tile'])
with zipfile.ZipFile(buf, mode='r') as z:
self.assertEqual(json, z.open('0/0/0.json').read())
def test_metatile_file_timing(self):
from time import gmtime, time
from tilequeue.metatile import metatiles_are_equal
# tilequeue's "GET before PUT" optimisation relies on being able to
# fetch a tile from S3 and compare it to the one that was just
# generated. to do this, we should try to make the tiles as similar
# as possible across multiple runs.
json = "{\"json\":true}"
tiles = [dict(tile=json, coord=Coordinate(0, 0, 0),
format=json_format, layer='all')]
when_will_then_be_now = 10
t = time()
now = gmtime(t)[0:6]
then = gmtime(t - when_will_then_be_now)[0:6]
metatile_1 = make_metatiles(1, tiles, then)
metatile_2 = make_metatiles(1, tiles, now)
self.assertTrue(metatiles_are_equal(
metatile_1[0]['tile'], metatile_2[0]['tile']))
def test_make_metatiles_multiple_coordinates(self):
# checks that we can make a single metatile which contains multiple
# coordinates. this is used for "512px" tiles as well as cutting out
# z17+ tiles and storing them in the z16 (z15 if "512px") metatile.
json = "{\"json\":true}"
tiles = [
# NOTE: coordinates are (y, x, z), possibly the most confusing
# possible permutation.
dict(tile=json,
coord=Coordinate(123, 456, 17),
format=json_format,
layer='all'),
dict(tile=json,
coord=Coordinate(123, 457, 17),
format=json_format,
layer='all'),
]
metatiles = make_metatiles(1, tiles)
self.assertEqual(1, len(metatiles))
meta = metatiles[0]
# NOTE: Coordinate(y, x, z)
coord = Coordinate(61, 228, 16)
self.assertEqual(meta['coord'], coord)
self.assertEqual('all', meta['layer'])
self.assertEqual(zip_format, meta['format'])
buf = StringIO.StringIO(meta['tile'])
with zipfile.ZipFile(buf, mode='r') as z:
self.assertEqual(json, z.open('1/0/1.json').read())
self.assertEqual(json, z.open('1/1/1.json').read())
# test extracting as well
self.assertEqual(
json,
extract_metatile(buf,
json_format,
offset=Coordinate(zoom=1, column=0, row=1)))
self.assertEqual(
json,
extract_metatile(buf,
json_format,
offset=Coordinate(zoom=1, column=1, row=1)))
def save_tiles(self, tiles):
async_jobs = []
if self.metatile_size:
tiles = make_metatiles(self.metatile_size, tiles)
for tile in tiles:
async_result = self.io_pool.apply_async(
write_tile_if_changed, (
self.store,
tile['tile'],
# important to use the coord from the
# formatted tile here, because we could have
# cut children tiles that have separate zooms
# too
tile['coord'],
tile['format']))
async_jobs.append(async_result)
return async_jobs
