def benchmark_database(big_raster_file_nodata, big_raster_file_mask, tmpdir_factory): from terracotta import get_driver, update_settings keys = ['type', 'band'] update_settings(RASTER_CACHE_SIZE=0) dbpath = tmpdir_factory.mktemp('db').join('db-readonly.sqlite') driver = get_driver(dbpath, provider='sqlite') driver.create(keys) mtd = driver.compute_metadata(str(big_raster_file_nodata)) with driver.connect(): driver.insert(['nodata', '1'], str(big_raster_file_nodata), metadata=mtd) driver.insert(['nodata', '2'], str(big_raster_file_nodata), metadata=mtd) driver.insert(['nodata', '3'], str(big_raster_file_nodata), metadata=mtd) driver.insert(['mask', '1'], str(big_raster_file_mask), metadata=mtd) return dbpath
def cli(ctx: click.Context, config: Mapping[str, Any] = None, loglevel: str = None) -> None: """The command line interface for the Terracotta tile server. All flags must be passed before specifying a subcommand. Example: $ terracotta -c config.toml connect localhost:5000 """ if ctx.invoked_subcommand is None: click.echo(ctx.get_help()) # update settings from config file if config is not None: update_settings(**config) # setup logging settings = get_settings() if loglevel is None: loglevel = settings.LOGLEVEL logs.set_logger(loglevel, catch_warnings=True)
def test_update_config(): from terracotta import get_settings, update_settings update_settings(DRIVER_PATH='test') new_settings = get_settings() assert new_settings.DRIVER_PATH == 'test' update_settings(DEFAULT_TILE_SIZE=[50, 50]) new_settings = get_settings() assert new_settings.DRIVER_PATH == 'test' and new_settings.DEFAULT_TILE_SIZE == (50, 50)
def test_no_multiprocessing(): import concurrent.futures from terracotta import update_settings from terracotta.drivers.geotiff_raster_store import create_executor update_settings(USE_MULTIPROCESSING=False) executor = create_executor() assert isinstance(executor, concurrent.futures.ThreadPoolExecutor)
def test_key_handler_env_config(testdb, monkeypatch): import terracotta from terracotta.handlers import keys monkeypatch.setenv('TC_DRIVER_PATH', str(testdb)) terracotta.update_settings() driver = terracotta.get_driver(str(testdb)) handler_response = keys.keys() assert handler_response assert tuple(row['key'] for row in handler_response) == driver.key_names
def get_client(metadata_origins=None, tile_origins=None): from terracotta.server import create_app import terracotta if metadata_origins is not None: terracotta.update_settings(ALLOWED_ORIGINS_METADATA=metadata_origins) if tile_origins is not None: terracotta.update_settings(ALLOWED_ORIGINS_TILES=tile_origins) flask_app = create_app() return flask_app.test_client()
def test_singleband_handler(use_testdb, raster_file_xyz, upsampling_method): import terracotta terracotta.update_settings(UPSAMPLING_METHOD=upsampling_method) from terracotta.handlers import datasets, singleband settings = terracotta.get_settings() ds = datasets.datasets() for keys in ds: raw_img = singleband.singleband(keys, raster_file_xyz) img_data = np.asarray(Image.open(raw_img)) assert img_data.shape == settings.DEFAULT_TILE_SIZE
def test_bench_rgb_out_of_bounds(benchmark, big_raster_file_nodata, benchmark_database): from terracotta.server import create_app from terracotta import update_settings update_settings(DRIVER_PATH=str(benchmark_database)) x, y, z = 0, 0, 20 flask_app = create_app() with flask_app.test_client() as client: rv = benchmark(client.get, f'/rgb/nodata/{z}/{x}/{y}.png?r=1&g=2&b=3') assert rv.status_code == 200
def use_non_writable_testdb(testdb, monkeypatch, raster_file): import terracotta terracotta.update_settings(DRIVER_PATH=str(testdb)) driver = terracotta.get_driver(testdb) with driver.connect(): driver.insert(('first', 'second', 'third'), str(raster_file), skip_metadata=True) with monkeypatch.context() as m: m.setattr(driver.meta_store, "_WRITABLE", False) yield driver.delete(('first', 'second', 'third'))
def test_bench_singleband_out_of_bounds(benchmark, benchmark_database): from terracotta.server import create_app from terracotta import update_settings update_settings(DRIVER_PATH=str(benchmark_database)) x, y, z = 0, 0, 20 flask_app = create_app() with flask_app.test_client() as client: rv = benchmark(client.get, f'/singleband/nodata/1/{z}/{x}/{y}.png') assert rv.status_code == 200
def run_test(): from terracotta import update_settings import terracotta.profile update_settings(XRAY_PROFILE=True) with XRaySegment(): with pytest.raises(NotImplementedError): with terracotta.profile.trace('dummy') as subsegment: raise NotImplementedError('foo') assert len(subsegment.cause['exceptions']) == 1 assert subsegment.cause['exceptions'][0].message == 'foo'
def test_raster_retrieval(driver_path, provider, raster_file, resampling_method): import terracotta terracotta.update_settings(RESAMPLING_METHOD=resampling_method) from terracotta import drivers db = drivers.get_driver(driver_path, provider=provider) keys = ('some', 'keynames') db.create(keys) db.insert(['some', 'value'], str(raster_file)) db.insert(['some', 'other_value'], str(raster_file)) data1 = db.get_raster_tile(['some', 'value'], tile_size=(256, 256)) assert data1.shape == (256, 256) data2 = db.get_raster_tile(['some', 'other_value'], tile_size=(256, 256)) assert data2.shape == (256, 256) np.testing.assert_array_equal(data1, data2)
def test_raster_cache_fail(driver_path, provider, raster_file, asynchronous): """Retrieve a tile that is larger than the total cache size""" from terracotta import drivers, update_settings update_settings(RASTER_CACHE_SIZE=1) db = drivers.get_driver(driver_path, provider=provider) keys = ('some', 'keynames') db.create(keys) db.insert(['some', 'value'], str(raster_file)) assert len(db._raster_cache) == 0 data1 = db.get_raster_tile(['some', 'value'], tile_size=(256, 256), asynchronous=asynchronous) if asynchronous: data1 = data1.result() time.sleep(1) # allow callback to finish assert len(db._raster_cache) == 0
def test_bench_rgb(benchmark, zoom, resampling, big_raster_file_nodata, benchmark_database): from terracotta.server import create_app from terracotta import update_settings update_settings(DRIVER_PATH=str(benchmark_database), RESAMPLING_METHOD=resampling, REPROJECTION_METHOD=resampling) zoom_level = ZOOM_XYZ[zoom] flask_app = create_app() with flask_app.test_client() as client: if zoom_level is not None: x, y, z = get_xyz(big_raster_file_nodata, zoom_level) rv = benchmark(client.get, f'/rgb/nodata/{z}/{x}/{y}.png?r=1&g=2&b=3') else: rv = benchmark(client.get, '/rgb/nodata/preview.png?r=1&g=2&b=3') assert rv.status_code == 200
def test_bench_singleband(benchmark, zoom, resampling, big_raster_file_nodata, benchmark_database): from terracotta.server import create_app from terracotta import update_settings, get_driver update_settings(DRIVER_PATH=str(benchmark_database), RESAMPLING_METHOD=resampling, REPROJECTION_METHOD=resampling) zoom_level = ZOOM_XYZ[zoom] flask_app = create_app() with flask_app.test_client() as client: if zoom_level is not None: x, y, z = get_xyz(big_raster_file_nodata, zoom_level) rv = benchmark(client.get, f'/singleband/nodata/1/{z}/{x}/{y}.png') else: rv = benchmark(client.get, f'/singleband/nodata/1/preview.png') assert rv.status_code == 200 assert not len(get_driver(str(benchmark_database))._raster_cache)
def test_xray_tracing(caplog): from terracotta import update_settings import terracotta.profile update_settings(XRAY_PROFILE=True) @terracotta.profile.trace('dummy') def func_to_trace(): time.sleep(0.1) with XRaySegment(): func_to_trace() with XRaySegment(): with terracotta.profile.trace('dummy2'): time.sleep(0.1) for record in caplog.records: assert record.levelname != 'ERROR' # sanity check, recording without starting a segment should fail func_to_trace() assert any('cannot find the current segment' in rec.message for rec in caplog.records)
def run_test_server(driver_path, port): from terracotta import update_settings update_settings(DRIVER_PATH=driver_path) from terracotta.server.flask_api import create_app create_app().run(port=port)
driver = tc.get_driver(DRIVER_PATH) # Create an empty database if it doesn't exist os.makedirs(os.path.dirname(DRIVER_PATH), exist_ok=True) if nuke and os.path.isfile(DRIVER_PATH): os.remove(DRIVER_PATH) if not os.path.isfile(DRIVER_PATH): driver.create(keys=keys, key_descriptions=key_descriptions) # Insert the parameters. with driver.connect(): for entry in [ entry for entry in os.listdir(GEOTIFF_DIR) if entry[-5:] == ".tiff" ]: tiff_path = os.path.join(GEOTIFF_DIR, entry) tiff_dict = { keys[i]: value for i, value in enumerate(entry[:-5].split(".")) } driver.insert(tiff_dict, tiff_path) init_db(keys=["gfs", "parameter"], nuke=True) # Create terracotta server. update_settings(DRIVER_PATH=DRIVER_PATH, REPROJECTION_METHOD="nearest") server = create_app() # Bind custom stuff. point.register(server) if __name__ == '__main__': server.run(port=TC_PORT, host=TC_HOST, threaded=False)
def serve(database: str = None, raster_pattern: RasterPatternType = None, debug: bool = False, profile: bool = False, database_provider: str = None, allow_all_ips: bool = False, port: int = None, rgb_key: str = None) -> None: """Serve rasters through a local Flask development server. Either -d/--database or -r/--raster-pattern must be given. Example: $ terracotta serve -r /path/to/rasters/{name}/{date}_{band}_{}.tif The empty group {} is replaced by a wildcard matching anything (similar to * in glob patterns). This command is a data exploration tool and not meant for production use. Deploy Terracotta as a WSGI or serverless app instead. """ from terracotta import get_driver, update_settings from terracotta.server import create_app if (database is None) == (raster_pattern is None): raise click.UsageError('Either --database or --raster-pattern must be given') if raster_pattern is not None: dbfile = tempfile.NamedTemporaryFile(suffix='.sqlite', delete=False) dbfile.close() keys, raster_files = raster_pattern if rgb_key is not None: if rgb_key not in keys: raise click.BadParameter('RGB key not found in raster pattern') # re-order keys rgb_idx = keys.index(rgb_key) def push_to_last(seq: Sequence[Any], index: int) -> Tuple[Any, ...]: return (*seq[:index], *seq[index + 1:], seq[index]) keys = list(push_to_last(keys, rgb_idx)) raster_files = {push_to_last(k, rgb_idx): v for k, v in raster_files.items()} driver = get_driver(dbfile.name, provider='sqlite') driver.create(keys) with driver.connect(): click.echo('') for key, filepath in tqdm.tqdm(raster_files.items(), desc="Ingesting raster files"): driver.insert(key, filepath, skip_metadata=True) click.echo('') database = dbfile.name update_settings(DRIVER_PATH=database, DRIVER_PROVIDER=database_provider, DEBUG=debug, FLASK_PROFILE=profile) # find suitable port port_range = [port] if port is not None else range(5000, 5100) port = find_open_port(port_range) if port is None: click.echo(f'Could not find open port to bind to (ports tried: {port_range})', err=True) raise click.Abort() host = '0.0.0.0' if allow_all_ips else 'localhost' server_app = create_app(debug=debug, profile=profile) if os.environ.get('TC_TESTING'): return server_app.run(port=port, host=host, threaded=False) # pragma: no cover
def use_testdb(testdb, monkeypatch): import terracotta terracotta.update_settings(DRIVER_PATH=str(testdb))
def test_app(): from terracotta import update_settings update_settings(DEBUG=True) from terracotta.server.app import app assert app.debug