def test_find_unique_tiles(expectedSuper, uniqueExpected): tiler = tile_utils.TileUtils() uniqueTiles = tiler.get_unique_tiles(expectedSuper) assert np.array_equal(uniqueTiles, uniqueExpected) print("# OK - %s " % (inspect.stack()[0][3]))
def test_get_xys_invalid_zoom(expectedTileList): tiler = tile_utils.TileUtils() with pytest.raises(ValueError): tiles, minX, minY, maxX, maxY = tiler.select_tiles( expectedTileList, 20) print("# OK - %s " % (inspect.stack()[0][3]))
def test_create_supertiles_fails(tilesShort): tiler = tile_utils.TileUtils() with pytest.raises(ValueError): superTiles = tiler.get_super_tiles(tilesShort, 20) print("# OK - %s " % (inspect.stack()[0][3]))
def test_find_zoom_tiles_fail(expectedTileList): tiler = tile_utils.TileUtils() superTiles = tiler.get_super_tiles(expectedTileList, 13)[:-10] with pytest.raises(ValueError): maxZ, maxZcoverage = tiler.get_zoom_tiles(expectedTileList, superTiles, superTiles[0]) print("# OK - %s " % (inspect.stack()[0][3]))
def test_create_supertiles(tilesShort, expectedSuper): tiler = tile_utils.TileUtils() superTiles = tiler.get_super_tiles(tilesShort, 14) assert tilesShort.shape == superTiles.shape assert np.array_equal(superTiles, expectedSuper) print("# OK - %s " % (inspect.stack()[0][3]))
def test_find_zoom_tiles_floor_fail(expectedTileList): ### a subset of tiles that don't have any tiles less than z 17 tiles = expectedTileList[:1000] tiler = tile_utils.TileUtils() superTiles = tiler.get_super_tiles(tiles, 12) with pytest.raises(ValueError): tiler.get_zoom_tiles(tiles, superTiles, superTiles[0], 17) print("# OK - %s " % (inspect.stack()[0][3]))
def test_get_xys(expectedTileList, expectedTiles19): tiler = tile_utils.TileUtils() tiles, minX, minY, maxX, maxY = tiler.select_tiles(expectedTileList, 19) assert np.array_equal(tiles, np.array(expectedTiles19['tiles'])) assert minX == expectedTiles19['minX'] assert maxX == expectedTiles19['maxX'] assert minY == expectedTiles19['minY'] assert maxY == expectedTiles19['maxY'] print("# OK - %s " % (inspect.stack()[0][3]))
def stream_dir(inputDir, outputDir, compositezoom, maxzoom, logdir, read_template, scene_template, workers, creation_opts, no_fill, tile_resolution=256): tiler = tile_utils.TileUtils() allFiles = tiler.search_dir(inputDir) template, readTemplate, separator = tile_utils.parse_template( "%s/%s" % (inputDir, read_template)) allTiles = np.array( [i for i in tiler.get_tiles(allFiles, template, separator)]) if allTiles.shape[0] == 0 or allTiles.shape[1] != 3: raise ValueError("No tiles were found for that template") if maxzoom: allTiles = tiler.filter_tiles(allTiles, maxzoom) if allTiles.shape[0] == 0: raise ValueError("No tiles were found below that maxzoom") _, sceneTemplate, _ = tile_utils.parse_template( "%s/%s" % (outputDir, scene_template)) pool = Pool(workers, global_setup, (inputDir, { 'maxzoom': maxzoom, 'readTemplate': readTemplate, 'outputDir': outputDir, 'tileResolution': tile_resolution, 'compositezoom': compositezoom, 'fileTemplate': '%s/%s_%s_%s_%s.tif', 'sceneTemplate': sceneTemplate, 'logdir': logdir, 'creation_opts': creation_opts, 'no_fill': no_fill })) superTiles = tiler.get_super_tiles(allTiles, compositezoom) for p in pool.imap_unordered(streaming_tile_worker, tiler.get_sub_tiles(allTiles, superTiles)): click.echo(p) pool.close() pool.join()
def test_find_zoom_tiles(expectedTileList, expectedZooms): tiler = tile_utils.TileUtils() superTiles = tiler.get_super_tiles(expectedTileList, 13) for t, e in zip(tiler.get_unique_tiles(superTiles), expectedZooms): maxZ, maxZcoverage = tiler.get_zoom_tiles(expectedTileList, superTiles, t) assert np.array_equal(maxZ, e['maxZ']) assert np.array_equal(maxZcoverage, e['maxZcoverage']) print("# OK - %s " % (inspect.stack()[0][3]))
def test_get_xys_invalid_tiles(): tiler = tile_utils.TileUtils() badtiles = np.array([0]) with pytest.raises(ValueError): tiles, minX, minY, maxX, maxY = tiler.select_tiles(badtiles, 19) badtiles = np.array([[1, 2], [1, 2]]) with pytest.raises(ValueError): tiles, minX, minY, maxX, maxY = tiler.select_tiles(badtiles, 19) print("# OK - %s " % (inspect.stack()[0][3]))
def test_find_zoom_tiles_floor(expectedTileList): ### a subset of tiles that have tiles less than z 17 tiles = expectedTileList[1000:] tiler = tile_utils.TileUtils() superTiles = tiler.get_super_tiles(tiles, 13) zMaxtiles, zFloortiles = tiler.get_zoom_tiles(tiles, superTiles, superTiles[-1], 17) assert zMaxtiles.shape == (848, 3) assert zFloortiles.shape == (68, 3) assert zFloortiles[:, 0].min() == 17 and zFloortiles[:, 0].max() == 17 print("# OK - %s " % (inspect.stack()[0][3]))
def inspect_dir(inputDir, zoom, read_template): tiler = tile_utils.TileUtils() allFiles = tiler.search_dir(inputDir) template, readTemplate, separator = tile_utils.parse_template( "%s/%s" % (inputDir, read_template)) allTiles = np.array( [i for i in tiler.get_tiles(allFiles, template, separator)]) allTiles, _, _, _, _ = tiler.select_tiles(allTiles, zoom) for t in allTiles: z, x, y = t click.echo([x, y, z])
def test_parse_tiles(inputTilenames, expectedTileList): matchTemplate = '3857_9_83_202_20130517_242834/jpg/\d+/\d+/\d+.jpg' tiler = tile_utils.TileUtils() output_tiles = np.array( [t for t in tiler.get_tiles(inputTilenames, matchTemplate, '/')]) assert np.array_equal(output_tiles, expectedTileList) tweakedTilenames = [f.replace('/', '?') for f in inputTilenames] output_tiles = np.array( [t for t in tiler.get_tiles(tweakedTilenames, matchTemplate, '/')]) assert len(output_tiles) == 0 assert np.array_equal(output_tiles, expectedTileList) == False print("# OK - %s " % (inspect.stack()[0][3]))
def streaming_tile_worker(data): size = 2**(data['zMax'] - globalArgs['compositezoom']) * globalArgs['tileResolution'] out_meta = make_src_meta(merc.bounds(data['x'], data['y'], data['z']), size, globalArgs['creation_opts']) z, x, y = [int(i) for i in (data['z'], data['x'], data['y'])] filename = globalArgs['sceneTemplate'] % (z, x, y) subtiler = tile_utils.TileUtils() log = 'FILE: %s\n' % filename try: with rasterio.open(filename, 'w', **out_meta) as dst: if data['zMaxCov']: superTiles = subtiler.get_super_tiles(data['zMaxTiles'], data['zMaxCov']) fillbaseX, fillbaseY = subtiler.get_sub_base_zoom( data['x'], data['y'], data['z'], data['zMaxCov']) ## fill thresh == the number of sub tiles that would need to occur in a fill tile to not fill (eg completely covered) fThresh = 4**(data['zMax'] - data['zMaxCov']) fDiff = 2**(data['zMax'] - data['zMaxCov']) toFaux, frFaux = affaux(fDiff) if not globalArgs['no_fill']: print('filling') ## Read and write the fill tiles first for t in subtiler.get_fill_super_tiles( superTiles, data['maxCovTiles'], fThresh): z, x, y = [int(i) for i in t] path = globalArgs['readTemplate'] % (z, x, y) log += '%s %s %s\n' % (z, x, y) with rasterio.open(path) as src: imdata = src.read() imdata = make_image_array(imdata, globalArgs['tileResolution']) imdata = upsample(imdata, fDiff, frFaux, toFaux) window = make_window( x, y, fillbaseX, fillbaseY, globalArgs['tileResolution'] * fDiff) dst.write(imdata, window=window) baseX, baseY = subtiler.get_sub_base_zoom(data['x'], data['y'], data['z'], data['zMax']) for t in data['zMaxTiles']: z, x, y = [int(i) for i in t] path = globalArgs['readTemplate'] % (z, x, y) log += '%s %s %s\n' % (z, x, y) with rasterio.open(path) as src: imdata = src.read() imdata = make_image_array(imdata, globalArgs['tileResolution']) window = make_window(x, y, baseX, baseY, globalArgs['tileResolution']) dst.write(imdata, window=window) if globalArgs['logdir']: with open( os.path.join(globalArgs['logdir'], '%s.log' % os.path.basename(filename)), 'w') as logger: logwriter(logger, log) return filename except Exception as e: click.echo("%s errored" % (path), err=True) raise e