def test_update_config():
from terracotta import get_settings, update_settings
update_settings(DRIVER_PATH='test')
new_settings = get_settings()
assert new_settings.DRIVER_PATH == 'test'
update_settings(DEFAULT_TILE_SIZE=[50, 50])
new_settings = get_settings()
assert new_settings.DRIVER_PATH == 'test' and new_settings.DEFAULT_TILE_SIZE == (50, 50)
def test_get_compute(client, use_testdb, raster_file_xyz):
import terracotta
settings = terracotta.get_settings()
# default tile size
x, y, z = raster_file_xyz
rv = client.get(
f'/compute/val21/x/{z}/{x}/{y}.png'
'?expression=v1*v2&v1=val22&v2=val23'
'&stretch_range=[0,10000]'
)
assert rv.status_code == 200
img = Image.open(BytesIO(rv.data))
assert np.asarray(img).shape == settings.DEFAULT_TILE_SIZE
# custom tile size
rv = client.get(
f'/compute/val21/x/{z}/{x}/{y}.png'
'?expression=v1*v2&v1=val22&v2=val23'
'&stretch_range=[0,10000]'
'&tile_size=[128,128]'
)
assert rv.status_code == 200
img = Image.open(BytesIO(rv.data))
assert np.asarray(img).shape == (128, 128)
def _update_db(self, remote_path: str, local_path: str) -> None:
settings = get_settings()
if self._last_updated < time.time() - settings.REMOTE_DB_CACHE_TTL:
logger.debug('Remote database cache expired, re-downloading')
_update_from_s3(remote_path, local_path)
self._last_updated = time.time()
def cli(ctx: click.Context,
config: Mapping[str, Any] = None,
loglevel: str = None) -> None:
"""The command line interface for the Terracotta tile server.
All flags must be passed before specifying a subcommand.
Example:
$ terracotta -c config.toml connect localhost:5000
"""
if ctx.invoked_subcommand is None:
click.echo(ctx.get_help())
# update settings from config file
if config is not None:
update_settings(**config)
# setup logging
settings = get_settings()
if loglevel is None:
loglevel = settings.LOGLEVEL
logs.set_logger(loglevel, catch_warnings=True)
def test_compute_consistency(use_testdb, testdb, raster_file_xyz):
import terracotta
from terracotta.xyz import get_tile_data
from terracotta.handlers import compute
from terracotta.image import to_uint8
settings = terracotta.get_settings()
raw_img = compute.compute(
'v1 + v2',
['val21', 'x'],
{'v1': 'val22', 'v2': 'val23'},
stretch_range=(0, 10000),
tile_xyz=raster_file_xyz
)
img_data = np.asarray(Image.open(raw_img))
assert img_data.shape == settings.DEFAULT_TILE_SIZE
driver = terracotta.get_driver(testdb)
with driver.connect():
v1 = get_tile_data(driver, ['val21', 'x', 'val22'], raster_file_xyz)
v2 = get_tile_data(driver, ['val21', 'x', 'val23'], raster_file_xyz)
np.testing.assert_array_equal(
img_data,
to_uint8(v1 + v2, 0, 10000)
)
def __init__(self, *args: Any, **kwargs: Any) -> None:
settings = get_settings()
self._raster_cache = CompressedLFUCache(
settings.RASTER_CACHE_SIZE,
compression_level=settings.RASTER_CACHE_COMPRESS_LEVEL)
self._cache_lock = threading.RLock()
super().__init__(*args, **kwargs)
def test_rgb_handler(use_testdb, raster_file, raster_file_xyz):
import terracotta
from terracotta.handlers import rgb
raw_img = rgb.rgb(['val21', 'x'], ['val22', 'val23', 'val24'],
raster_file_xyz)
img_data = np.asarray(Image.open(raw_img))
assert img_data.shape == (*terracotta.get_settings().DEFAULT_TILE_SIZE, 3)
def __init__(self, *args: Any, **kwargs: Any) -> None:
settings = get_settings()
self._raster_cache = CompressedLFUCache(
settings.RASTER_CACHE_SIZE,
compression_level=settings.RASTER_CACHE_COMPRESS_LEVEL)
self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=3)
super().__init__(*args, **kwargs)
def __init__(self, remote_path: str) -> None:
"""Initialize the RemoteSQLiteDriver.
This should not be called directly, use :func:`~terracotta.get_driver` instead.
Arguments:
remote_path: S3 URL in the form ``s3://bucket/key`` to remote SQLite database
(has to exist).
"""
settings = get_settings()
self.__rm = os.remove # keep reference to use in __del__
os.makedirs(settings.REMOTE_DB_CACHE_DIR, exist_ok=True)
local_db_file = tempfile.NamedTemporaryFile(
dir=settings.REMOTE_DB_CACHE_DIR,
prefix='tc_s3_db_',
suffix='.sqlite',
delete=False)
local_db_file.close()
self._remote_path = str(remote_path)
self._last_updated = -float('inf')
super().__init__(local_db_file.name)
def get_raster_tile(self,
keys: Union[Sequence[str], Mapping[str, str]], *,
bounds: Sequence[float] = None,
tile_size: Sequence[int] = None,
preserve_values: bool = False,
asynchronous: bool = False) -> Any:
settings = get_settings()
key_tuple = tuple(self._key_dict_to_sequence(keys))
datasets = self.get_datasets(dict(zip(self.key_names, key_tuple)))
assert len(datasets) == 1
path = datasets[key_tuple]
if tile_size is None:
tile_size = settings.DEFAULT_TILE_SIZE
# make sure all arguments are hashable
task = functools.partial(
self._get_raster_tile,
path,
bounds=tuple(bounds) if bounds else None,
tile_size=tuple(tile_size),
preserve_values=preserve_values,
upsampling_method=settings.UPSAMPLING_METHOD,
downsampling_method=settings.DOWNSAMPLING_METHOD
)
if asynchronous:
return self._executor.submit(task)
else:
return task()
def __init__(self, *args: Any, **kwargs: Any) -> None:
settings = get_settings()
self._raster_cache = LFUCache(
settings.RASTER_CACHE_SIZE,
getsizeof=operator.attrgetter('nbytes')
)
self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=3)
super().__init__(*args, **kwargs)
def metadata(keys: Union[Sequence[str], Mapping[str, str]]) -> Dict[str, Any]:
"""Returns all metadata for a single dataset"""
settings = get_settings()
driver = get_driver(settings.DRIVER_PATH,
provider=settings.DRIVER_PROVIDER)
metadata = driver.get_metadata(keys)
metadata['keys'] = OrderedDict(zip(driver.key_names, keys))
return metadata
def __init__(self, meta_store: MetaStore,
raster_store: RasterStore) -> None:
self.meta_store = meta_store
self.raster_store = raster_store
settings = terracotta.get_settings()
self.LAZY_LOADING_MAX_SHAPE: Tuple[
int, int] = settings.LAZY_LOADING_MAX_SHAPE
def get_raster_tile(self,
path: str,
*,
tile_bounds: Sequence[float] = None,
tile_size: Sequence[int] = None,
preserve_values: bool = False,
asynchronous: bool = False) -> Any:
future: Future[np.ma.MaskedArray]
result: np.ma.MaskedArray
settings = get_settings()
if tile_size is None:
tile_size = settings.DEFAULT_TILE_SIZE
kwargs = dict(
path=path,
tile_bounds=tile_bounds,
tile_size=tuple(tile_size),
preserve_values=preserve_values,
reprojection_method=settings.REPROJECTION_METHOD,
resampling_method=settings.RESAMPLING_METHOD,
target_crs=self._TARGET_CRS,
rio_env_options=self._RIO_ENV_OPTIONS,
)
cache_key = hash(ensure_hashable(kwargs))
try:
with self._cache_lock:
result = self._raster_cache[cache_key]
except KeyError:
pass
else:
if asynchronous:
# wrap result in a future
future = Future()
future.set_result(result)
return future
else:
return result
retrieve_tile = functools.partial(raster.get_raster_tile, **kwargs)
future = submit_to_executor(retrieve_tile)
def cache_callback(future: Future) -> None:
# insert result into global cache if execution was successful
if future.exception() is None:
self._add_to_cache(cache_key, future.result())
if asynchronous:
future.add_done_callback(cache_callback)
return future
else:
result = future.result()
cache_callback(future)
return result
def test_get_rgb_preview(client, use_testdb):
import terracotta
settings = terracotta.get_settings()
rv = client.get(f'/rgb/val21/x/preview.png?r=val22&g=val23&b=val24')
assert rv.status_code == 200
img = Image.open(BytesIO(rv.data))
assert np.asarray(img).shape == (*settings.DEFAULT_TILE_SIZE, 3)
def test_get_singleband_preview(client, use_testdb):
import terracotta
settings = terracotta.get_settings()
rv = client.get(f'/singleband/val11/x/val12/preview.png?colormap=jet')
assert rv.status_code == 200
img = Image.open(BytesIO(rv.data))
assert np.asarray(img).shape == settings.DEFAULT_TILE_SIZE
def point(keys: str, lat: str, lng: str):
keys = keys.split("/")
settings = get_settings()
driver = get_driver(settings.DRIVER_PATH,
provider=settings.DRIVER_PROVIDER)
with driver.connect():
dataset = get_unique_dataset(driver, keys)
result = get_point_data(float(lat), float(lng), dataset[1])
return jsonify(result)
def test_get_singleband_cmap(client, use_testdb, raster_file_xyz):
import terracotta
settings = terracotta.get_settings()
x, y, z = raster_file_xyz
rv = client.get(f'/singleband/val11/x/val12/{z}/{x}/{y}.png?colormap=jet')
assert rv.status_code == 200
img = Image.open(BytesIO(rv.data))
assert np.asarray(img).shape == settings.DEFAULT_TILE_SIZE
def _get_data():
driver = tc.get_driver(tc.get_settings().DRIVER_PATH)
data = {}
with driver.connect():
for k in driver.get_datasets():
title, year = k
if not title in data:
data[title] = {}
data[title][year] = driver.get_metadata(k)
return data
def test_get_rgb(client, use_testdb, raster_file_xyz):
import terracotta
settings = terracotta.get_settings()
x, y, z = raster_file_xyz
rv = client.get(f'/rgb/val21/x/{z}/{x}/{y}.png?r=val22&g=val23&b=val24')
assert rv.status_code == 200
img = Image.open(BytesIO(rv.data))
assert np.asarray(img).shape == (*settings.DEFAULT_TILE_SIZE, 3)
def create_app(debug: bool = False, profile: bool = False) -> Flask:
"""Returns a Flask app"""
from terracotta import get_settings
import terracotta.server.datasets
import terracotta.server.keys
import terracotta.server.colormap
import terracotta.server.metadata
import terracotta.server.rgb
import terracotta.server.singleband
import terracotta.server.compute
new_app = Flask('terracotta.server')
new_app.debug = debug
# extensions might modify the global blueprints, so copy before use
new_tile_api = copy.deepcopy(TILE_API)
new_metadata_api = copy.deepcopy(METADATA_API)
# suppress implicit sort of JSON responses
new_app.config['JSON_SORT_KEYS'] = False
# CORS
settings = get_settings()
CORS(new_tile_api, origins=settings.ALLOWED_ORIGINS_TILES)
CORS(new_metadata_api, origins=settings.ALLOWED_ORIGINS_METADATA)
new_app.register_blueprint(new_tile_api, url_prefix='')
new_app.register_blueprint(new_metadata_api, url_prefix='')
# register routes on API spec
with new_app.test_request_context():
SPEC.path(view=terracotta.server.datasets.get_datasets)
SPEC.path(view=terracotta.server.keys.get_keys)
SPEC.path(view=terracotta.server.colormap.get_colormap)
SPEC.path(view=terracotta.server.metadata.get_metadata)
SPEC.path(view=terracotta.server.rgb.get_rgb)
SPEC.path(view=terracotta.server.rgb.get_rgb_preview)
SPEC.path(view=terracotta.server.singleband.get_singleband)
SPEC.path(view=terracotta.server.singleband.get_singleband_preview)
SPEC.path(view=terracotta.server.compute.get_compute)
SPEC.path(view=terracotta.server.compute.get_compute_preview)
import terracotta.server.spec
new_app.register_blueprint(SPEC_API, url_prefix='')
if profile:
from werkzeug.contrib.profiler import ProfilerMiddleware
# use setattr to work around mypy false-positive (python/mypy#2427)
setattr(new_app, 'wsgi_app',
ProfilerMiddleware(new_app.wsgi_app, restrictions=[30]))
_setup_error_handlers(new_app)
return new_app
def test_get_compute_preview(client, use_testdb):
import terracotta
settings = terracotta.get_settings()
rv = client.get(f'/compute/val21/x/preview.png'
'?expression=v1*v2&v1=val22&v2=val23'
'&stretch_range=[0,10000]')
assert rv.status_code == 200
img = Image.open(BytesIO(rv.data))
assert np.asarray(img).shape == settings.DEFAULT_TILE_SIZE
def singleband(keys: Union[Sequence[str], Mapping[str, str]],
tile_xyz: Tuple[int, int, int] = None,
*,
colormap: Union[str, Mapping[Number, RGB], None] = None,
stretch_range: Tuple[Number, Number] = None,
tile_size: Tuple[int, int] = None) -> BinaryIO:
"""Return singleband image as PNG"""
cmap_or_palette: Union[str, Sequence[RGB], None]
if stretch_range is None:
stretch_min, stretch_max = None, None
else:
stretch_min, stretch_max = stretch_range
preserve_values = isinstance(colormap, collections.Mapping)
settings = get_settings()
if tile_size is None:
tile_size = settings.DEFAULT_TILE_SIZE
driver = get_driver(settings.DRIVER_PATH,
provider=settings.DRIVER_PROVIDER)
with driver.connect():
metadata = driver.get_metadata(keys)
tile_data = xyz.get_tile_data(driver,
keys,
tile_xyz,
tile_size=tile_size,
preserve_values=preserve_values)
if preserve_values:
# bin output image into supplied labels, starting at 1
colormap = cast(Mapping, colormap)
labels, label_colors = list(colormap.keys()), list(colormap.values())
cmap_or_palette = label_colors
out = image.label(tile_data, labels)
else:
# determine stretch range from metadata and arguments
stretch_range_ = list(metadata['range'])
if stretch_min is not None:
stretch_range_[0] = stretch_min
if stretch_max is not None:
stretch_range_[1] = stretch_max
cmap_or_palette = cast(Optional[str], colormap)
out = image.to_uint8(tile_data, *stretch_range_)
return image.array_to_png(out, colormap=cmap_or_palette)
def test_get_singleband_out_of_bounds(client, use_testdb):
import terracotta
settings = terracotta.get_settings()
x, y, z = (0, 0, 10)
rv = client.get(f'/singleband/val11/x/val12/{z}/{x}/{y}.png')
assert rv.status_code == 200
img = Image.open(BytesIO(rv.data))
assert np.asarray(img).shape == settings.DEFAULT_TILE_SIZE
assert np.all(np.asarray(img) == 0)
def empty_image(size: Tuple[int, int]) -> BinaryIO:
"""Return a fully transparent PNG image of given size"""
settings = get_settings()
compress_level = settings.PNG_COMPRESS_LEVEL
img = Image.new(mode='P', size=size, color=0)
sio = BytesIO()
img.save(sio, 'png', compress_level=compress_level, transparency=0)
sio.seek(0)
return sio
def test_singleband_noxyz(use_testdb):
from terracotta import get_settings
from terracotta.handlers import singleband
settings = get_settings()
ds_keys = ['val21', 'x', 'val22']
raw_img = singleband.singleband(ds_keys)
img_data = np.asarray(Image.open(raw_img))
assert img_data.shape == settings.DEFAULT_TILE_SIZE
def test_get_singleband_stretch(client, use_testdb, raster_file_xyz):
import terracotta
settings = terracotta.get_settings()
x, y, z = raster_file_xyz
for stretch_range in ('[0,1]', '[0,null]', '[null, 1]', '[null,null]', 'null'):
rv = client.get(f'/singleband/val11/x/val12/{z}/{x}/{y}.png?stretch_range={stretch_range}')
assert rv.status_code == 200
img = Image.open(BytesIO(rv.data))
assert np.asarray(img).shape == settings.DEFAULT_TILE_SIZE
def test_singleband_explicit_colormap(use_testdb, testdb, raster_file_xyz):
import terracotta
from terracotta.xyz import get_tile_data
from terracotta.handlers import singleband
ds_keys = ['val21', 'x', 'val22']
nodata = 10000
settings = terracotta.get_settings()
driver = terracotta.get_driver(testdb)
with driver.connect():
tile_data = get_tile_data(driver,
ds_keys,
tile_xyz=raster_file_xyz,
preserve_values=True,
tile_size=settings.DEFAULT_TILE_SIZE)
# Get some values from the raster to use for colormap
classes = np.unique(tile_data)
classes = classes[:254]
colormap = {}
for i in range(classes.shape[0]):
val = classes[i]
color = val % 256
colormap[val] = (color, color, color, color)
colormap[nodata] = (100, 100, 100, 100)
raw_img = singleband.singleband(ds_keys,
raster_file_xyz,
colormap=colormap)
img_data = np.asarray(Image.open(raw_img).convert('RGBA'))
# get unstretched data to compare to
with driver.connect():
tile_data = get_tile_data(driver,
ds_keys,
tile_xyz=raster_file_xyz,
preserve_values=True,
tile_size=img_data.shape[:2])
# check that labels are mapped to colors correctly
for cmap_label, cmap_color in colormap.items():
if cmap_label == nodata:
# make sure nodata is still transparent
assert np.all(img_data[tile_data == cmap_label, -1] == 0)
else:
assert np.all(
img_data[tile_data == cmap_label] == np.asarray(cmap_color))
# check that all data outside of labels is transparent
keys_arr = np.array(list(colormap.keys()), dtype=np.int16)
assert np.all(img_data[~np.isin(tile_data, keys_arr), -1] == 0)
def test_singleband_handler(use_testdb, raster_file_xyz, upsampling_method):
import terracotta
terracotta.update_settings(UPSAMPLING_METHOD=upsampling_method)
from terracotta.handlers import datasets, singleband
settings = terracotta.get_settings()
ds = datasets.datasets()
for keys in ds:
raw_img = singleband.singleband(keys, raster_file_xyz)
img_data = np.asarray(Image.open(raw_img))
assert img_data.shape == settings.DEFAULT_TILE_SIZE
def test_get_rgb_stretch(client, use_testdb, raster_file_xyz):
import terracotta
settings = terracotta.get_settings()
x, y, z = raster_file_xyz
for stretch_range in ('[0,10000]', '[0,null]', '[null, 10000]', '[null,null]', 'null'):
rv = client.get(f'/rgb/val21/x/{z}/{x}/{y}.png?r=val22&g=val23&b=val24&'
f'r_range={stretch_range}&b_range={stretch_range}&g_range={stretch_range}')
assert rv.status_code == 200, rv.data
img = Image.open(BytesIO(rv.data))
assert np.asarray(img).shape == (*settings.DEFAULT_TILE_SIZE, 3)
