def test_update_config(): from terracotta import get_settings, update_settings update_settings(DRIVER_PATH='test') new_settings = get_settings() assert new_settings.DRIVER_PATH == 'test' update_settings(DEFAULT_TILE_SIZE=[50, 50]) new_settings = get_settings() assert new_settings.DRIVER_PATH == 'test' and new_settings.DEFAULT_TILE_SIZE == (50, 50)
def test_get_compute(client, use_testdb, raster_file_xyz): import terracotta settings = terracotta.get_settings() # default tile size x, y, z = raster_file_xyz rv = client.get( f'/compute/val21/x/{z}/{x}/{y}.png' '?expression=v1*v2&v1=val22&v2=val23' '&stretch_range=[0,10000]' ) assert rv.status_code == 200 img = Image.open(BytesIO(rv.data)) assert np.asarray(img).shape == settings.DEFAULT_TILE_SIZE # custom tile size rv = client.get( f'/compute/val21/x/{z}/{x}/{y}.png' '?expression=v1*v2&v1=val22&v2=val23' '&stretch_range=[0,10000]' '&tile_size=[128,128]' ) assert rv.status_code == 200 img = Image.open(BytesIO(rv.data)) assert np.asarray(img).shape == (128, 128)
def _update_db(self, remote_path: str, local_path: str) -> None: settings = get_settings() if self._last_updated < time.time() - settings.REMOTE_DB_CACHE_TTL: logger.debug('Remote database cache expired, re-downloading') _update_from_s3(remote_path, local_path) self._last_updated = time.time()
def cli(ctx: click.Context, config: Mapping[str, Any] = None, loglevel: str = None) -> None: """The command line interface for the Terracotta tile server. All flags must be passed before specifying a subcommand. Example: $ terracotta -c config.toml connect localhost:5000 """ if ctx.invoked_subcommand is None: click.echo(ctx.get_help()) # update settings from config file if config is not None: update_settings(**config) # setup logging settings = get_settings() if loglevel is None: loglevel = settings.LOGLEVEL logs.set_logger(loglevel, catch_warnings=True)
def test_compute_consistency(use_testdb, testdb, raster_file_xyz): import terracotta from terracotta.xyz import get_tile_data from terracotta.handlers import compute from terracotta.image import to_uint8 settings = terracotta.get_settings() raw_img = compute.compute( 'v1 + v2', ['val21', 'x'], {'v1': 'val22', 'v2': 'val23'}, stretch_range=(0, 10000), tile_xyz=raster_file_xyz ) img_data = np.asarray(Image.open(raw_img)) assert img_data.shape == settings.DEFAULT_TILE_SIZE driver = terracotta.get_driver(testdb) with driver.connect(): v1 = get_tile_data(driver, ['val21', 'x', 'val22'], raster_file_xyz) v2 = get_tile_data(driver, ['val21', 'x', 'val23'], raster_file_xyz) np.testing.assert_array_equal( img_data, to_uint8(v1 + v2, 0, 10000) )
def __init__(self, *args: Any, **kwargs: Any) -> None: settings = get_settings() self._raster_cache = CompressedLFUCache( settings.RASTER_CACHE_SIZE, compression_level=settings.RASTER_CACHE_COMPRESS_LEVEL) self._cache_lock = threading.RLock() super().__init__(*args, **kwargs)
def test_rgb_handler(use_testdb, raster_file, raster_file_xyz): import terracotta from terracotta.handlers import rgb raw_img = rgb.rgb(['val21', 'x'], ['val22', 'val23', 'val24'], raster_file_xyz) img_data = np.asarray(Image.open(raw_img)) assert img_data.shape == (*terracotta.get_settings().DEFAULT_TILE_SIZE, 3)
def __init__(self, *args: Any, **kwargs: Any) -> None: settings = get_settings() self._raster_cache = CompressedLFUCache( settings.RASTER_CACHE_SIZE, compression_level=settings.RASTER_CACHE_COMPRESS_LEVEL) self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=3) super().__init__(*args, **kwargs)
def __init__(self, remote_path: str) -> None: """Initialize the RemoteSQLiteDriver. This should not be called directly, use :func:`~terracotta.get_driver` instead. Arguments: remote_path: S3 URL in the form ``s3://bucket/key`` to remote SQLite database (has to exist). """ settings = get_settings() self.__rm = os.remove # keep reference to use in __del__ os.makedirs(settings.REMOTE_DB_CACHE_DIR, exist_ok=True) local_db_file = tempfile.NamedTemporaryFile( dir=settings.REMOTE_DB_CACHE_DIR, prefix='tc_s3_db_', suffix='.sqlite', delete=False) local_db_file.close() self._remote_path = str(remote_path) self._last_updated = -float('inf') super().__init__(local_db_file.name)
def get_raster_tile(self, keys: Union[Sequence[str], Mapping[str, str]], *, bounds: Sequence[float] = None, tile_size: Sequence[int] = None, preserve_values: bool = False, asynchronous: bool = False) -> Any: settings = get_settings() key_tuple = tuple(self._key_dict_to_sequence(keys)) datasets = self.get_datasets(dict(zip(self.key_names, key_tuple))) assert len(datasets) == 1 path = datasets[key_tuple] if tile_size is None: tile_size = settings.DEFAULT_TILE_SIZE # make sure all arguments are hashable task = functools.partial( self._get_raster_tile, path, bounds=tuple(bounds) if bounds else None, tile_size=tuple(tile_size), preserve_values=preserve_values, upsampling_method=settings.UPSAMPLING_METHOD, downsampling_method=settings.DOWNSAMPLING_METHOD ) if asynchronous: return self._executor.submit(task) else: return task()
def __init__(self, *args: Any, **kwargs: Any) -> None: settings = get_settings() self._raster_cache = LFUCache( settings.RASTER_CACHE_SIZE, getsizeof=operator.attrgetter('nbytes') ) self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=3) super().__init__(*args, **kwargs)
def metadata(keys: Union[Sequence[str], Mapping[str, str]]) -> Dict[str, Any]: """Returns all metadata for a single dataset""" settings = get_settings() driver = get_driver(settings.DRIVER_PATH, provider=settings.DRIVER_PROVIDER) metadata = driver.get_metadata(keys) metadata['keys'] = OrderedDict(zip(driver.key_names, keys)) return metadata
def __init__(self, meta_store: MetaStore, raster_store: RasterStore) -> None: self.meta_store = meta_store self.raster_store = raster_store settings = terracotta.get_settings() self.LAZY_LOADING_MAX_SHAPE: Tuple[ int, int] = settings.LAZY_LOADING_MAX_SHAPE
def get_raster_tile(self, path: str, *, tile_bounds: Sequence[float] = None, tile_size: Sequence[int] = None, preserve_values: bool = False, asynchronous: bool = False) -> Any: future: Future[np.ma.MaskedArray] result: np.ma.MaskedArray settings = get_settings() if tile_size is None: tile_size = settings.DEFAULT_TILE_SIZE kwargs = dict( path=path, tile_bounds=tile_bounds, tile_size=tuple(tile_size), preserve_values=preserve_values, reprojection_method=settings.REPROJECTION_METHOD, resampling_method=settings.RESAMPLING_METHOD, target_crs=self._TARGET_CRS, rio_env_options=self._RIO_ENV_OPTIONS, ) cache_key = hash(ensure_hashable(kwargs)) try: with self._cache_lock: result = self._raster_cache[cache_key] except KeyError: pass else: if asynchronous: # wrap result in a future future = Future() future.set_result(result) return future else: return result retrieve_tile = functools.partial(raster.get_raster_tile, **kwargs) future = submit_to_executor(retrieve_tile) def cache_callback(future: Future) -> None: # insert result into global cache if execution was successful if future.exception() is None: self._add_to_cache(cache_key, future.result()) if asynchronous: future.add_done_callback(cache_callback) return future else: result = future.result() cache_callback(future) return result
def test_get_rgb_preview(client, use_testdb): import terracotta settings = terracotta.get_settings() rv = client.get(f'/rgb/val21/x/preview.png?r=val22&g=val23&b=val24') assert rv.status_code == 200 img = Image.open(BytesIO(rv.data)) assert np.asarray(img).shape == (*settings.DEFAULT_TILE_SIZE, 3)
def test_get_singleband_preview(client, use_testdb): import terracotta settings = terracotta.get_settings() rv = client.get(f'/singleband/val11/x/val12/preview.png?colormap=jet') assert rv.status_code == 200 img = Image.open(BytesIO(rv.data)) assert np.asarray(img).shape == settings.DEFAULT_TILE_SIZE
def point(keys: str, lat: str, lng: str): keys = keys.split("/") settings = get_settings() driver = get_driver(settings.DRIVER_PATH, provider=settings.DRIVER_PROVIDER) with driver.connect(): dataset = get_unique_dataset(driver, keys) result = get_point_data(float(lat), float(lng), dataset[1]) return jsonify(result)
def test_get_singleband_cmap(client, use_testdb, raster_file_xyz): import terracotta settings = terracotta.get_settings() x, y, z = raster_file_xyz rv = client.get(f'/singleband/val11/x/val12/{z}/{x}/{y}.png?colormap=jet') assert rv.status_code == 200 img = Image.open(BytesIO(rv.data)) assert np.asarray(img).shape == settings.DEFAULT_TILE_SIZE
def _get_data(): driver = tc.get_driver(tc.get_settings().DRIVER_PATH) data = {} with driver.connect(): for k in driver.get_datasets(): title, year = k if not title in data: data[title] = {} data[title][year] = driver.get_metadata(k) return data
def test_get_rgb(client, use_testdb, raster_file_xyz): import terracotta settings = terracotta.get_settings() x, y, z = raster_file_xyz rv = client.get(f'/rgb/val21/x/{z}/{x}/{y}.png?r=val22&g=val23&b=val24') assert rv.status_code == 200 img = Image.open(BytesIO(rv.data)) assert np.asarray(img).shape == (*settings.DEFAULT_TILE_SIZE, 3)
def create_app(debug: bool = False, profile: bool = False) -> Flask: """Returns a Flask app""" from terracotta import get_settings import terracotta.server.datasets import terracotta.server.keys import terracotta.server.colormap import terracotta.server.metadata import terracotta.server.rgb import terracotta.server.singleband import terracotta.server.compute new_app = Flask('terracotta.server') new_app.debug = debug # extensions might modify the global blueprints, so copy before use new_tile_api = copy.deepcopy(TILE_API) new_metadata_api = copy.deepcopy(METADATA_API) # suppress implicit sort of JSON responses new_app.config['JSON_SORT_KEYS'] = False # CORS settings = get_settings() CORS(new_tile_api, origins=settings.ALLOWED_ORIGINS_TILES) CORS(new_metadata_api, origins=settings.ALLOWED_ORIGINS_METADATA) new_app.register_blueprint(new_tile_api, url_prefix='') new_app.register_blueprint(new_metadata_api, url_prefix='') # register routes on API spec with new_app.test_request_context(): SPEC.path(view=terracotta.server.datasets.get_datasets) SPEC.path(view=terracotta.server.keys.get_keys) SPEC.path(view=terracotta.server.colormap.get_colormap) SPEC.path(view=terracotta.server.metadata.get_metadata) SPEC.path(view=terracotta.server.rgb.get_rgb) SPEC.path(view=terracotta.server.rgb.get_rgb_preview) SPEC.path(view=terracotta.server.singleband.get_singleband) SPEC.path(view=terracotta.server.singleband.get_singleband_preview) SPEC.path(view=terracotta.server.compute.get_compute) SPEC.path(view=terracotta.server.compute.get_compute_preview) import terracotta.server.spec new_app.register_blueprint(SPEC_API, url_prefix='') if profile: from werkzeug.contrib.profiler import ProfilerMiddleware # use setattr to work around mypy false-positive (python/mypy#2427) setattr(new_app, 'wsgi_app', ProfilerMiddleware(new_app.wsgi_app, restrictions=[30])) _setup_error_handlers(new_app) return new_app
def test_get_compute_preview(client, use_testdb): import terracotta settings = terracotta.get_settings() rv = client.get(f'/compute/val21/x/preview.png' '?expression=v1*v2&v1=val22&v2=val23' '&stretch_range=[0,10000]') assert rv.status_code == 200 img = Image.open(BytesIO(rv.data)) assert np.asarray(img).shape == settings.DEFAULT_TILE_SIZE
def singleband(keys: Union[Sequence[str], Mapping[str, str]], tile_xyz: Tuple[int, int, int] = None, *, colormap: Union[str, Mapping[Number, RGB], None] = None, stretch_range: Tuple[Number, Number] = None, tile_size: Tuple[int, int] = None) -> BinaryIO: """Return singleband image as PNG""" cmap_or_palette: Union[str, Sequence[RGB], None] if stretch_range is None: stretch_min, stretch_max = None, None else: stretch_min, stretch_max = stretch_range preserve_values = isinstance(colormap, collections.Mapping) settings = get_settings() if tile_size is None: tile_size = settings.DEFAULT_TILE_SIZE driver = get_driver(settings.DRIVER_PATH, provider=settings.DRIVER_PROVIDER) with driver.connect(): metadata = driver.get_metadata(keys) tile_data = xyz.get_tile_data(driver, keys, tile_xyz, tile_size=tile_size, preserve_values=preserve_values) if preserve_values: # bin output image into supplied labels, starting at 1 colormap = cast(Mapping, colormap) labels, label_colors = list(colormap.keys()), list(colormap.values()) cmap_or_palette = label_colors out = image.label(tile_data, labels) else: # determine stretch range from metadata and arguments stretch_range_ = list(metadata['range']) if stretch_min is not None: stretch_range_[0] = stretch_min if stretch_max is not None: stretch_range_[1] = stretch_max cmap_or_palette = cast(Optional[str], colormap) out = image.to_uint8(tile_data, *stretch_range_) return image.array_to_png(out, colormap=cmap_or_palette)
def test_get_singleband_out_of_bounds(client, use_testdb): import terracotta settings = terracotta.get_settings() x, y, z = (0, 0, 10) rv = client.get(f'/singleband/val11/x/val12/{z}/{x}/{y}.png') assert rv.status_code == 200 img = Image.open(BytesIO(rv.data)) assert np.asarray(img).shape == settings.DEFAULT_TILE_SIZE assert np.all(np.asarray(img) == 0)
def empty_image(size: Tuple[int, int]) -> BinaryIO: """Return a fully transparent PNG image of given size""" settings = get_settings() compress_level = settings.PNG_COMPRESS_LEVEL img = Image.new(mode='P', size=size, color=0) sio = BytesIO() img.save(sio, 'png', compress_level=compress_level, transparency=0) sio.seek(0) return sio
def test_singleband_noxyz(use_testdb): from terracotta import get_settings from terracotta.handlers import singleband settings = get_settings() ds_keys = ['val21', 'x', 'val22'] raw_img = singleband.singleband(ds_keys) img_data = np.asarray(Image.open(raw_img)) assert img_data.shape == settings.DEFAULT_TILE_SIZE
def test_get_singleband_stretch(client, use_testdb, raster_file_xyz): import terracotta settings = terracotta.get_settings() x, y, z = raster_file_xyz for stretch_range in ('[0,1]', '[0,null]', '[null, 1]', '[null,null]', 'null'): rv = client.get(f'/singleband/val11/x/val12/{z}/{x}/{y}.png?stretch_range={stretch_range}') assert rv.status_code == 200 img = Image.open(BytesIO(rv.data)) assert np.asarray(img).shape == settings.DEFAULT_TILE_SIZE
def test_singleband_explicit_colormap(use_testdb, testdb, raster_file_xyz): import terracotta from terracotta.xyz import get_tile_data from terracotta.handlers import singleband ds_keys = ['val21', 'x', 'val22'] nodata = 10000 settings = terracotta.get_settings() driver = terracotta.get_driver(testdb) with driver.connect(): tile_data = get_tile_data(driver, ds_keys, tile_xyz=raster_file_xyz, preserve_values=True, tile_size=settings.DEFAULT_TILE_SIZE) # Get some values from the raster to use for colormap classes = np.unique(tile_data) classes = classes[:254] colormap = {} for i in range(classes.shape[0]): val = classes[i] color = val % 256 colormap[val] = (color, color, color, color) colormap[nodata] = (100, 100, 100, 100) raw_img = singleband.singleband(ds_keys, raster_file_xyz, colormap=colormap) img_data = np.asarray(Image.open(raw_img).convert('RGBA')) # get unstretched data to compare to with driver.connect(): tile_data = get_tile_data(driver, ds_keys, tile_xyz=raster_file_xyz, preserve_values=True, tile_size=img_data.shape[:2]) # check that labels are mapped to colors correctly for cmap_label, cmap_color in colormap.items(): if cmap_label == nodata: # make sure nodata is still transparent assert np.all(img_data[tile_data == cmap_label, -1] == 0) else: assert np.all( img_data[tile_data == cmap_label] == np.asarray(cmap_color)) # check that all data outside of labels is transparent keys_arr = np.array(list(colormap.keys()), dtype=np.int16) assert np.all(img_data[~np.isin(tile_data, keys_arr), -1] == 0)
def test_singleband_handler(use_testdb, raster_file_xyz, upsampling_method): import terracotta terracotta.update_settings(UPSAMPLING_METHOD=upsampling_method) from terracotta.handlers import datasets, singleband settings = terracotta.get_settings() ds = datasets.datasets() for keys in ds: raw_img = singleband.singleband(keys, raster_file_xyz) img_data = np.asarray(Image.open(raw_img)) assert img_data.shape == settings.DEFAULT_TILE_SIZE
def test_get_rgb_stretch(client, use_testdb, raster_file_xyz): import terracotta settings = terracotta.get_settings() x, y, z = raster_file_xyz for stretch_range in ('[0,10000]', '[0,null]', '[null, 10000]', '[null,null]', 'null'): rv = client.get(f'/rgb/val21/x/{z}/{x}/{y}.png?r=val22&g=val23&b=val24&' f'r_range={stretch_range}&b_range={stretch_range}&g_range={stretch_range}') assert rv.status_code == 200, rv.data img = Image.open(BytesIO(rv.data)) assert np.asarray(img).shape == (*settings.DEFAULT_TILE_SIZE, 3)