def test_find_unique_tiles(expectedSuper, uniqueExpected):
tiler = tile_utils.TileUtils()
uniqueTiles = tiler.get_unique_tiles(expectedSuper)
assert np.array_equal(uniqueTiles, uniqueExpected)
print("# OK - %s " % (inspect.stack()[0][3]))
def test_get_xys_invalid_zoom(expectedTileList):
tiler = tile_utils.TileUtils()
with pytest.raises(ValueError):
tiles, minX, minY, maxX, maxY = tiler.select_tiles(
expectedTileList, 20)
print("# OK - %s " % (inspect.stack()[0][3]))
def test_create_supertiles_fails(tilesShort):
tiler = tile_utils.TileUtils()
with pytest.raises(ValueError):
superTiles = tiler.get_super_tiles(tilesShort, 20)
print("# OK - %s " % (inspect.stack()[0][3]))
def test_find_zoom_tiles_fail(expectedTileList):
tiler = tile_utils.TileUtils()
superTiles = tiler.get_super_tiles(expectedTileList, 13)[:-10]
with pytest.raises(ValueError):
maxZ, maxZcoverage = tiler.get_zoom_tiles(expectedTileList, superTiles,
superTiles[0])
print("# OK - %s " % (inspect.stack()[0][3]))
def test_create_supertiles(tilesShort, expectedSuper):
tiler = tile_utils.TileUtils()
superTiles = tiler.get_super_tiles(tilesShort, 14)
assert tilesShort.shape == superTiles.shape
assert np.array_equal(superTiles, expectedSuper)
print("# OK - %s " % (inspect.stack()[0][3]))
def test_find_zoom_tiles_floor_fail(expectedTileList):
### a subset of tiles that don't have any tiles less than z 17
tiles = expectedTileList[:1000]
tiler = tile_utils.TileUtils()
superTiles = tiler.get_super_tiles(tiles, 12)
with pytest.raises(ValueError):
tiler.get_zoom_tiles(tiles, superTiles, superTiles[0], 17)
print("# OK - %s " % (inspect.stack()[0][3]))
def test_get_xys(expectedTileList, expectedTiles19):
tiler = tile_utils.TileUtils()
tiles, minX, minY, maxX, maxY = tiler.select_tiles(expectedTileList, 19)
assert np.array_equal(tiles, np.array(expectedTiles19['tiles']))
assert minX == expectedTiles19['minX']
assert maxX == expectedTiles19['maxX']
assert minY == expectedTiles19['minY']
assert maxY == expectedTiles19['maxY']
print("# OK - %s " % (inspect.stack()[0][3]))
def stream_dir(inputDir,
outputDir,
compositezoom,
maxzoom,
logdir,
read_template,
scene_template,
workers,
creation_opts,
no_fill,
tile_resolution=256):
tiler = tile_utils.TileUtils()
allFiles = tiler.search_dir(inputDir)
template, readTemplate, separator = tile_utils.parse_template(
"%s/%s" % (inputDir, read_template))
allTiles = np.array(
[i for i in tiler.get_tiles(allFiles, template, separator)])
if allTiles.shape[0] == 0 or allTiles.shape[1] != 3:
raise ValueError("No tiles were found for that template")
if maxzoom:
allTiles = tiler.filter_tiles(allTiles, maxzoom)
if allTiles.shape[0] == 0:
raise ValueError("No tiles were found below that maxzoom")
_, sceneTemplate, _ = tile_utils.parse_template(
"%s/%s" % (outputDir, scene_template))
pool = Pool(workers, global_setup, (inputDir, {
'maxzoom': maxzoom,
'readTemplate': readTemplate,
'outputDir': outputDir,
'tileResolution': tile_resolution,
'compositezoom': compositezoom,
'fileTemplate': '%s/%s_%s_%s_%s.tif',
'sceneTemplate': sceneTemplate,
'logdir': logdir,
'creation_opts': creation_opts,
'no_fill': no_fill
}))
superTiles = tiler.get_super_tiles(allTiles, compositezoom)
for p in pool.imap_unordered(streaming_tile_worker,
tiler.get_sub_tiles(allTiles, superTiles)):
click.echo(p)
pool.close()
pool.join()
def test_find_zoom_tiles(expectedTileList, expectedZooms):
tiler = tile_utils.TileUtils()
superTiles = tiler.get_super_tiles(expectedTileList, 13)
for t, e in zip(tiler.get_unique_tiles(superTiles), expectedZooms):
maxZ, maxZcoverage = tiler.get_zoom_tiles(expectedTileList, superTiles,
t)
assert np.array_equal(maxZ, e['maxZ'])
assert np.array_equal(maxZcoverage, e['maxZcoverage'])
print("# OK - %s " % (inspect.stack()[0][3]))
def test_get_xys_invalid_tiles():
tiler = tile_utils.TileUtils()
badtiles = np.array([0])
with pytest.raises(ValueError):
tiles, minX, minY, maxX, maxY = tiler.select_tiles(badtiles, 19)
badtiles = np.array([[1, 2], [1, 2]])
with pytest.raises(ValueError):
tiles, minX, minY, maxX, maxY = tiler.select_tiles(badtiles, 19)
print("# OK - %s " % (inspect.stack()[0][3]))
def test_find_zoom_tiles_floor(expectedTileList):
### a subset of tiles that have tiles less than z 17
tiles = expectedTileList[1000:]
tiler = tile_utils.TileUtils()
superTiles = tiler.get_super_tiles(tiles, 13)
zMaxtiles, zFloortiles = tiler.get_zoom_tiles(tiles, superTiles,
superTiles[-1], 17)
assert zMaxtiles.shape == (848, 3)
assert zFloortiles.shape == (68, 3)
assert zFloortiles[:, 0].min() == 17 and zFloortiles[:, 0].max() == 17
print("# OK - %s " % (inspect.stack()[0][3]))
def inspect_dir(inputDir, zoom, read_template):
tiler = tile_utils.TileUtils()
allFiles = tiler.search_dir(inputDir)
template, readTemplate, separator = tile_utils.parse_template(
"%s/%s" % (inputDir, read_template))
allTiles = np.array(
[i for i in tiler.get_tiles(allFiles, template, separator)])
allTiles, _, _, _, _ = tiler.select_tiles(allTiles, zoom)
for t in allTiles:
z, x, y = t
click.echo([x, y, z])
def test_parse_tiles(inputTilenames, expectedTileList):
matchTemplate = '3857_9_83_202_20130517_242834/jpg/\d+/\d+/\d+.jpg'
tiler = tile_utils.TileUtils()
output_tiles = np.array(
[t for t in tiler.get_tiles(inputTilenames, matchTemplate, '/')])
assert np.array_equal(output_tiles, expectedTileList)
tweakedTilenames = [f.replace('/', '?') for f in inputTilenames]
output_tiles = np.array(
[t for t in tiler.get_tiles(tweakedTilenames, matchTemplate, '/')])
assert len(output_tiles) == 0
assert np.array_equal(output_tiles, expectedTileList) == False
print("# OK - %s " % (inspect.stack()[0][3]))
def streaming_tile_worker(data):
size = 2**(data['zMax'] -
globalArgs['compositezoom']) * globalArgs['tileResolution']
out_meta = make_src_meta(merc.bounds(data['x'], data['y'], data['z']),
size, globalArgs['creation_opts'])
z, x, y = [int(i) for i in (data['z'], data['x'], data['y'])]
filename = globalArgs['sceneTemplate'] % (z, x, y)
subtiler = tile_utils.TileUtils()
log = 'FILE: %s\n' % filename
try:
with rasterio.open(filename, 'w', **out_meta) as dst:
if data['zMaxCov']:
superTiles = subtiler.get_super_tiles(data['zMaxTiles'],
data['zMaxCov'])
fillbaseX, fillbaseY = subtiler.get_sub_base_zoom(
data['x'], data['y'], data['z'], data['zMaxCov'])
## fill thresh == the number of sub tiles that would need to occur in a fill tile to not fill (eg completely covered)
fThresh = 4**(data['zMax'] - data['zMaxCov'])
fDiff = 2**(data['zMax'] - data['zMaxCov'])
toFaux, frFaux = affaux(fDiff)
if not globalArgs['no_fill']:
print('filling')
## Read and write the fill tiles first
for t in subtiler.get_fill_super_tiles(
superTiles, data['maxCovTiles'], fThresh):
z, x, y = [int(i) for i in t]
path = globalArgs['readTemplate'] % (z, x, y)
log += '%s %s %s\n' % (z, x, y)
with rasterio.open(path) as src:
imdata = src.read()
imdata = make_image_array(imdata,
globalArgs['tileResolution'])
imdata = upsample(imdata, fDiff, frFaux, toFaux)
window = make_window(
x, y, fillbaseX, fillbaseY,
globalArgs['tileResolution'] * fDiff)
dst.write(imdata, window=window)
baseX, baseY = subtiler.get_sub_base_zoom(data['x'], data['y'],
data['z'], data['zMax'])
for t in data['zMaxTiles']:
z, x, y = [int(i) for i in t]
path = globalArgs['readTemplate'] % (z, x, y)
log += '%s %s %s\n' % (z, x, y)
with rasterio.open(path) as src:
imdata = src.read()
imdata = make_image_array(imdata, globalArgs['tileResolution'])
window = make_window(x, y, baseX, baseY,
globalArgs['tileResolution'])
dst.write(imdata, window=window)
if globalArgs['logdir']:
with open(
os.path.join(globalArgs['logdir'],
'%s.log' % os.path.basename(filename)),
'w') as logger:
logwriter(logger, log)
return filename
except Exception as e:
click.echo("%s errored" % (path), err=True)
raise e