def benchmark_database(big_raster_file_nodata, big_raster_file_mask,
tmpdir_factory):
from terracotta import get_driver, update_settings
keys = ['type', 'band']
update_settings(RASTER_CACHE_SIZE=0)
dbpath = tmpdir_factory.mktemp('db').join('db-readonly.sqlite')
driver = get_driver(dbpath, provider='sqlite')
driver.create(keys)
mtd = driver.compute_metadata(str(big_raster_file_nodata))
with driver.connect():
driver.insert(['nodata', '1'],
str(big_raster_file_nodata),
metadata=mtd)
driver.insert(['nodata', '2'],
str(big_raster_file_nodata),
metadata=mtd)
driver.insert(['nodata', '3'],
str(big_raster_file_nodata),
metadata=mtd)
driver.insert(['mask', '1'], str(big_raster_file_mask), metadata=mtd)
return dbpath
def cli(ctx: click.Context,
config: Mapping[str, Any] = None,
loglevel: str = None) -> None:
"""The command line interface for the Terracotta tile server.
All flags must be passed before specifying a subcommand.
Example:
$ terracotta -c config.toml connect localhost:5000
"""
if ctx.invoked_subcommand is None:
click.echo(ctx.get_help())
# update settings from config file
if config is not None:
update_settings(**config)
# setup logging
settings = get_settings()
if loglevel is None:
loglevel = settings.LOGLEVEL
logs.set_logger(loglevel, catch_warnings=True)
def test_update_config():
from terracotta import get_settings, update_settings
update_settings(DRIVER_PATH='test')
new_settings = get_settings()
assert new_settings.DRIVER_PATH == 'test'
update_settings(DEFAULT_TILE_SIZE=[50, 50])
new_settings = get_settings()
assert new_settings.DRIVER_PATH == 'test' and new_settings.DEFAULT_TILE_SIZE == (50, 50)
def test_no_multiprocessing():
import concurrent.futures
from terracotta import update_settings
from terracotta.drivers.geotiff_raster_store import create_executor
update_settings(USE_MULTIPROCESSING=False)
executor = create_executor()
assert isinstance(executor, concurrent.futures.ThreadPoolExecutor)
def test_key_handler_env_config(testdb, monkeypatch):
import terracotta
from terracotta.handlers import keys
monkeypatch.setenv('TC_DRIVER_PATH', str(testdb))
terracotta.update_settings()
driver = terracotta.get_driver(str(testdb))
handler_response = keys.keys()
assert handler_response
assert tuple(row['key'] for row in handler_response) == driver.key_names
def get_client(metadata_origins=None, tile_origins=None):
from terracotta.server import create_app
import terracotta
if metadata_origins is not None:
terracotta.update_settings(ALLOWED_ORIGINS_METADATA=metadata_origins)
if tile_origins is not None:
terracotta.update_settings(ALLOWED_ORIGINS_TILES=tile_origins)
flask_app = create_app()
return flask_app.test_client()
def test_singleband_handler(use_testdb, raster_file_xyz, upsampling_method):
import terracotta
terracotta.update_settings(UPSAMPLING_METHOD=upsampling_method)
from terracotta.handlers import datasets, singleband
settings = terracotta.get_settings()
ds = datasets.datasets()
for keys in ds:
raw_img = singleband.singleband(keys, raster_file_xyz)
img_data = np.asarray(Image.open(raw_img))
assert img_data.shape == settings.DEFAULT_TILE_SIZE
def test_bench_rgb_out_of_bounds(benchmark, big_raster_file_nodata, benchmark_database):
from terracotta.server import create_app
from terracotta import update_settings
update_settings(DRIVER_PATH=str(benchmark_database))
x, y, z = 0, 0, 20
flask_app = create_app()
with flask_app.test_client() as client:
rv = benchmark(client.get, f'/rgb/nodata/{z}/{x}/{y}.png?r=1&g=2&b=3')
assert rv.status_code == 200
def use_non_writable_testdb(testdb, monkeypatch, raster_file):
import terracotta
terracotta.update_settings(DRIVER_PATH=str(testdb))
driver = terracotta.get_driver(testdb)
with driver.connect():
driver.insert(('first', 'second', 'third'), str(raster_file), skip_metadata=True)
with monkeypatch.context() as m:
m.setattr(driver.meta_store, "_WRITABLE", False)
yield
driver.delete(('first', 'second', 'third'))
def test_bench_singleband_out_of_bounds(benchmark, benchmark_database):
from terracotta.server import create_app
from terracotta import update_settings
update_settings(DRIVER_PATH=str(benchmark_database))
x, y, z = 0, 0, 20
flask_app = create_app()
with flask_app.test_client() as client:
rv = benchmark(client.get, f'/singleband/nodata/1/{z}/{x}/{y}.png')
assert rv.status_code == 200
def run_test():
from terracotta import update_settings
import terracotta.profile
update_settings(XRAY_PROFILE=True)
with XRaySegment():
with pytest.raises(NotImplementedError):
with terracotta.profile.trace('dummy') as subsegment:
raise NotImplementedError('foo')
assert len(subsegment.cause['exceptions']) == 1
assert subsegment.cause['exceptions'][0].message == 'foo'
def test_raster_retrieval(driver_path, provider, raster_file, resampling_method):
import terracotta
terracotta.update_settings(RESAMPLING_METHOD=resampling_method)
from terracotta import drivers
db = drivers.get_driver(driver_path, provider=provider)
keys = ('some', 'keynames')
db.create(keys)
db.insert(['some', 'value'], str(raster_file))
db.insert(['some', 'other_value'], str(raster_file))
data1 = db.get_raster_tile(['some', 'value'], tile_size=(256, 256))
assert data1.shape == (256, 256)
data2 = db.get_raster_tile(['some', 'other_value'], tile_size=(256, 256))
assert data2.shape == (256, 256)
np.testing.assert_array_equal(data1, data2)
def test_raster_cache_fail(driver_path, provider, raster_file, asynchronous):
"""Retrieve a tile that is larger than the total cache size"""
from terracotta import drivers, update_settings
update_settings(RASTER_CACHE_SIZE=1)
db = drivers.get_driver(driver_path, provider=provider)
keys = ('some', 'keynames')
db.create(keys)
db.insert(['some', 'value'], str(raster_file))
assert len(db._raster_cache) == 0
data1 = db.get_raster_tile(['some', 'value'], tile_size=(256, 256), asynchronous=asynchronous)
if asynchronous:
data1 = data1.result()
time.sleep(1) # allow callback to finish
assert len(db._raster_cache) == 0
def test_bench_rgb(benchmark, zoom, resampling, big_raster_file_nodata,
benchmark_database):
from terracotta.server import create_app
from terracotta import update_settings
update_settings(DRIVER_PATH=str(benchmark_database),
RESAMPLING_METHOD=resampling,
REPROJECTION_METHOD=resampling)
zoom_level = ZOOM_XYZ[zoom]
flask_app = create_app()
with flask_app.test_client() as client:
if zoom_level is not None:
x, y, z = get_xyz(big_raster_file_nodata, zoom_level)
rv = benchmark(client.get,
f'/rgb/nodata/{z}/{x}/{y}.png?r=1&g=2&b=3')
else:
rv = benchmark(client.get, '/rgb/nodata/preview.png?r=1&g=2&b=3')
assert rv.status_code == 200
def test_bench_singleband(benchmark, zoom, resampling, big_raster_file_nodata,
benchmark_database):
from terracotta.server import create_app
from terracotta import update_settings, get_driver
update_settings(DRIVER_PATH=str(benchmark_database),
RESAMPLING_METHOD=resampling,
REPROJECTION_METHOD=resampling)
zoom_level = ZOOM_XYZ[zoom]
flask_app = create_app()
with flask_app.test_client() as client:
if zoom_level is not None:
x, y, z = get_xyz(big_raster_file_nodata, zoom_level)
rv = benchmark(client.get, f'/singleband/nodata/1/{z}/{x}/{y}.png')
else:
rv = benchmark(client.get, f'/singleband/nodata/1/preview.png')
assert rv.status_code == 200
assert not len(get_driver(str(benchmark_database))._raster_cache)
def test_xray_tracing(caplog):
from terracotta import update_settings
import terracotta.profile
update_settings(XRAY_PROFILE=True)
@terracotta.profile.trace('dummy')
def func_to_trace():
time.sleep(0.1)
with XRaySegment():
func_to_trace()
with XRaySegment():
with terracotta.profile.trace('dummy2'):
time.sleep(0.1)
for record in caplog.records:
assert record.levelname != 'ERROR'
# sanity check, recording without starting a segment should fail
func_to_trace()
assert any('cannot find the current segment' in rec.message for rec in caplog.records)
def run_test_server(driver_path, port):
from terracotta import update_settings
update_settings(DRIVER_PATH=driver_path)
from terracotta.server.flask_api import create_app
create_app().run(port=port)
driver = tc.get_driver(DRIVER_PATH)
# Create an empty database if it doesn't exist
os.makedirs(os.path.dirname(DRIVER_PATH), exist_ok=True)
if nuke and os.path.isfile(DRIVER_PATH):
os.remove(DRIVER_PATH)
if not os.path.isfile(DRIVER_PATH):
driver.create(keys=keys, key_descriptions=key_descriptions)
# Insert the parameters.
with driver.connect():
for entry in [
entry for entry in os.listdir(GEOTIFF_DIR)
if entry[-5:] == ".tiff"
]:
tiff_path = os.path.join(GEOTIFF_DIR, entry)
tiff_dict = {
keys[i]: value
for i, value in enumerate(entry[:-5].split("."))
}
driver.insert(tiff_dict, tiff_path)
init_db(keys=["gfs", "parameter"], nuke=True)
# Create terracotta server.
update_settings(DRIVER_PATH=DRIVER_PATH, REPROJECTION_METHOD="nearest")
server = create_app()
# Bind custom stuff.
point.register(server)
if __name__ == '__main__':
server.run(port=TC_PORT, host=TC_HOST, threaded=False)
def serve(database: str = None,
raster_pattern: RasterPatternType = None,
debug: bool = False,
profile: bool = False,
database_provider: str = None,
allow_all_ips: bool = False,
port: int = None,
rgb_key: str = None) -> None:
"""Serve rasters through a local Flask development server.
Either -d/--database or -r/--raster-pattern must be given.
Example:
$ terracotta serve -r /path/to/rasters/{name}/{date}_{band}_{}.tif
The empty group {} is replaced by a wildcard matching anything (similar to * in glob patterns).
This command is a data exploration tool and not meant for production use. Deploy Terracotta as
a WSGI or serverless app instead.
"""
from terracotta import get_driver, update_settings
from terracotta.server import create_app
if (database is None) == (raster_pattern is None):
raise click.UsageError('Either --database or --raster-pattern must be given')
if raster_pattern is not None:
dbfile = tempfile.NamedTemporaryFile(suffix='.sqlite', delete=False)
dbfile.close()
keys, raster_files = raster_pattern
if rgb_key is not None:
if rgb_key not in keys:
raise click.BadParameter('RGB key not found in raster pattern')
# re-order keys
rgb_idx = keys.index(rgb_key)
def push_to_last(seq: Sequence[Any], index: int) -> Tuple[Any, ...]:
return (*seq[:index], *seq[index + 1:], seq[index])
keys = list(push_to_last(keys, rgb_idx))
raster_files = {push_to_last(k, rgb_idx): v for k, v in raster_files.items()}
driver = get_driver(dbfile.name, provider='sqlite')
driver.create(keys)
with driver.connect():
click.echo('')
for key, filepath in tqdm.tqdm(raster_files.items(), desc="Ingesting raster files"):
driver.insert(key, filepath, skip_metadata=True)
click.echo('')
database = dbfile.name
update_settings(DRIVER_PATH=database, DRIVER_PROVIDER=database_provider,
DEBUG=debug, FLASK_PROFILE=profile)
# find suitable port
port_range = [port] if port is not None else range(5000, 5100)
port = find_open_port(port_range)
if port is None:
click.echo(f'Could not find open port to bind to (ports tried: {port_range})', err=True)
raise click.Abort()
host = '0.0.0.0' if allow_all_ips else 'localhost'
server_app = create_app(debug=debug, profile=profile)
if os.environ.get('TC_TESTING'):
return
server_app.run(port=port, host=host, threaded=False) # pragma: no cover
def use_testdb(testdb, monkeypatch):
import terracotta
terracotta.update_settings(DRIVER_PATH=str(testdb))
def test_app():
from terracotta import update_settings
update_settings(DEBUG=True)
from terracotta.server.app import app
assert app.debug
