def test_gridspec():
gs = GridSpec(crs=geometry.CRS('EPSG:4326'),
tile_size=(1, 1),
resolution=(-0.1, 0.1),
origin=(10, 10))
poly = geometry.polygon([(10, 12.2), (10.8, 13), (13, 10.8), (12.2, 10),
(10, 12.2)],
crs=geometry.CRS('EPSG:4326'))
cells = {
index: geobox
for index, geobox in list(gs.tiles_from_geopolygon(poly))
}
assert set(cells.keys()) == {(0, 1), (0, 2), (1, 0), (1, 1), (1, 2),
(2, 0), (2, 1)}
assert numpy.isclose(cells[(2, 0)].coordinates['longitude'].values,
numpy.linspace(12.05, 12.95, num=10)).all()
assert numpy.isclose(cells[(2, 0)].coordinates['latitude'].values,
numpy.linspace(10.95, 10.05, num=10)).all()
# check geobox_cache
cache = {}
poly = gs.tile_geobox((3, 4)).extent
(c1, gbox1), = list(gs.tiles_from_geopolygon(poly, geobox_cache=cache))
(c2, gbox2), = list(gs.tiles_from_geopolygon(poly, geobox_cache=cache))
assert c1 == (3, 4) and c2 == c1
assert gbox1 is gbox2
def test_gridspec():
gs = GridSpec(crs=geometry.CRS('EPSG:4326'), tile_size=(1, 1), resolution=(-0.1, 0.1), origin=(10, 10))
poly = geometry.polygon([(10, 12.2), (10.8, 13), (13, 10.8), (12.2, 10), (10, 12.2)], crs=geometry.CRS('EPSG:4326'))
cells = {index: geobox for index, geobox in list(gs.tiles_inside_geopolygon(poly))}
assert set(cells.keys()) == {(0, 1), (0, 2), (1, 0), (1, 1), (1, 2), (2, 0), (2, 1)}
assert numpy.isclose(cells[(2, 0)].coordinates['longitude'].values, numpy.linspace(12.05, 12.95, num=10)).all()
assert numpy.isclose(cells[(2, 0)].coordinates['latitude'].values, numpy.linspace(10.95, 10.05, num=10)).all()
def __init__(self,
resolution: Tuple[int, int] = (-20, 20),
crs: str = "epsg:6933"):
target_crs = CRS(crs)
self.albers_africa_N = GridSpec(
crs=target_crs,
tile_size=(96_000.0, 96_000.0), # default
resolution=resolution,
)
africa = box(-18, -38, 60, 30, "epsg:4326")
self.africa_projected = africa.to_crs(crs, resolution=math.inf)
def change_resolution(self, resolution: Tuple[float, float]):
"""
Modify GridSpec to have different pixel resolution but still covering same tiles as the original.
"""
if not self.is_compatible_resolution(resolution):
raise ValueError(
"Supplied resolution is not compatible with the current GridSpec"
)
gs = self._gridspec
self._gridspec = GridSpec(
gs.crs, gs.tile_size, resolution=resolution, origin=gs.origin
)
def gs_albers():
from datacube.model import GridSpec
import datacube.utils.geometry as geom
return GridSpec(crs=geom.CRS('EPSG:3577'),
tile_size=(100000.0, 100000.0),
resolution=(-25, 25))
def doc2gs(doc: Document) -> GridSpec:
return GridSpec(
crs=CRS(doc["crs"]),
tile_size=tuple(doc["tile_size"]),
resolution=tuple(doc["resolution"]),
origin=tuple(doc["origin"]),
)
def test_wofs_filtered():
cfg = Config('../configs/template_client.yaml')
grid_spec = GridSpec(crs=CRS('EPSG:3577'),
tile_size=(100000, 100000),
resolution=(-25, 25))
cell_index = (17, -39)
wf = WofsFiltered(cfg, grid_spec, cell_index)
confidence = wf.compute_confidence(cell_index)
filtered = wf.compute_confidence_filtered()
# Display images: to be removed later
with Datacube(app='wofs_summary', env='dev') as dc:
gwf = GridWorkflow(dc.index, grid_spec)
indexed_tile = gwf.list_cells(cell_index,
product='wofs_statistical_summary')
# load the data of the tile
dataset = gwf.load(tile=indexed_tile[cell_index],
measurements=['frequency'])
frequency = dataset.data_vars['frequency'].data.ravel().reshape(
grid_spec.tile_resolution)
# Check with previous run
with rasterio.open('confidenceFilteredWOfS_17_-39_epsilon=10.tiff') as f:
data = f.read(1)
plt.subplot(221)
plt.imshow(frequency)
plt.subplot(222)
plt.imshow(data)
plt.subplot(223)
plt.imshow(confidence)
plt.subplot(224)
plt.imshow(filtered)
plt.show()
wf.compute_and_write()
def parse_gridspec(s: str) -> GridSpec:
"""
"albers_africa_10"
"epsg:6936;10;9600"
"epsg:6936;-10x10;9600x9600"
"""
named_gs = GRIDS.get(s)
if named_gs is not None:
return named_gs
crs, res, shape = split_and_check(s, ';', 3)
try:
if 'x' in res:
res = tuple(float(v) for v in split_and_check(res, 'x', 2))
else:
res = float(res)
res = (-res, res)
if 'x' in shape:
shape = parse_range_int(shape, separator='x')
else:
shape = int(shape)
shape = (shape, shape)
except ValueError:
raise ValueError(f"Failed to parse gridspec: {s}")
tsz = tuple(abs(n * res) for n, res in zip(res, shape))
return GridSpec(crs=CRS(crs), tile_size=tsz, resolution=res, origin=(0, 0))
def example_product(name):
if name not in PRODUCT_LIST:
return None
blue = dict(name='blue', dtype='int16', nodata=-999, units='1')
green = dict(name='green', dtype='int16', nodata=-999, units='1', aliases=['verde'])
flags = {"cloud_acca": {"bits": 10, "values": {"0": "cloud", "1": "no_cloud"}},
"contiguous": {"bits": 8, "values": {"0": False, "1": True}},
"cloud_fmask": {"bits": 11, "values": {"0": "cloud", "1": "no_cloud"}},
"nir_saturated": {"bits": 3, "values": {"0": True, "1": False}},
"red_saturated": {"bits": 2, "values": {"0": True, "1": False}},
"blue_saturated": {"bits": 0, "values": {"0": True, "1": False}},
"green_saturated": {"bits": 1, "values": {"0": True, "1": False}},
"swir1_saturated": {"bits": 4, "values": {"0": True, "1": False}},
"swir2_saturated": {"bits": 7, "values": {"0": True, "1": False}},
"cloud_shadow_acca": {"bits": 12, "values": {"0": "cloud_shadow", "1": "no_cloud_shadow"}},
"cloud_shadow_fmask": {"bits": 13, "values": {"0": "cloud_shadow", "1": "no_cloud_shadow"}}}
pixelquality = dict(name='pixelquality', dtype='int16', nodata=0, units='1',
flags_definition=flags)
result = DatasetType(example_metadata_type(),
dict(name=name, description="", metadata_type='eo', metadata={}))
result.grid_spec = GridSpec(crs=geometry.CRS('EPSG:3577'),
tile_size=(100000., 100000.),
resolution=(-25, 25))
if '_pq_' in name:
result.definition = {'name': name, 'measurements': [pixelquality]}
else:
result.definition = {'name': name, 'measurements': [blue, green]}
return result
def _parse_gridspec_string(s: str) -> GridSpec:
"""
"epsg:6936;10;9600"
"epsg:6936;-10x10;9600x9600"
"""
crs, res, shape = split_and_check(s, ";", 3)
try:
if "x" in res:
res = tuple(float(v) for v in split_and_check(res, "x", 2))
else:
res = float(res)
res = (-res, res)
if "x" in shape:
shape = parse_range_int(shape, separator="x")
else:
shape = int(shape)
shape = (shape, shape)
except ValueError:
raise ValueError(f"Failed to parse gridspec: {s}")
tsz = tuple(abs(n * res) for n, res in zip(res, shape))
return GridSpec(crs=CRS(crs), tile_size=tsz, resolution=res, origin=(0, 0))
def get_grid_spec(config):
storage = config['storage']
crs = CRS(storage['crs'])
return GridSpec(
crs=crs,
tile_size=[storage['tile_size'][dim] for dim in crs.dimensions],
resolution=[storage['resolution'][dim] for dim in crs.dimensions])
def web_gs(zoom, tile_size=256):
""" Construct grid spec compatible with TerriaJS requests at a given level.
Tile indexes should be the same as google maps, except that Y component is negative,
this is a limitation of GridSpec class, you can not have tile index direction be
different from axis direction, but this is what google indexing is using.
http://www.maptiler.org/google-maps-coordinates-tile-bounds-projection/
"""
from datacube.utils.geometry import CRS
from datacube.model import GridSpec
from math import pi
R = 6378137
origin = pi * R
res0 = 2 * pi * R / tile_size
res = res0 * (2**(-zoom))
tsz = 2 * pi * R * (2**(-zoom)) # res*tile_size
return GridSpec(
crs=CRS('epsg:3857'),
tile_size=(tsz, tsz),
resolution=(-res, res), # Y,X
origin=(origin - tsz, -origin)) # Y,X
def example_product(name):
result = DatasetType(
example_metadata_type(),
dict(name=name, description="", metadata_type='eo', metadata={}))
result.grid_spec = GridSpec(crs=geometry.CRS('EPSG:3577'),
tile_size=(100000., 100000.),
resolution=(-25, 25))
return result
def _make_grid_spec(storage) -> GridSpec:
"""Make a grid spec based on a storage spec."""
assert 'tile_size' in storage
crs = CRS(storage['crs'])
return GridSpec(
crs=crs,
tile_size=[storage['tile_size'][dim] for dim in crs.dimensions],
resolution=[storage['resolution'][dim] for dim in crs.dimensions])
def gs_bounds(gs: GridSpec, tiles: Tuple[Tuple[int, int],
Tuple[int, int]]) -> Geometry:
"""
Compute Polygon for a selection of tiles.
:param gs: GridSpec
:param tiles: (x_range, y_range)
X,Y ranges are inclusive on the left and exclusive on the right, same as numpy slicing.
"""
((x0, x1), (y0, y1)) = tiles
if gs.resolution[0] < 0:
gb = gs.tile_geobox((x0, y1 - 1))
else:
gb = gs.tile_geobox((x0, y0))
nx = (x1 - x0) * gb.shape[1]
ny = (y1 - y0) * gb.shape[0]
return polygon_from_transform(nx, ny, gb.affine, gb.crs)
def test_gridworkflow_with_time_depth():
"""Test GridWorkflow with time series.
Also test `Tile` methods `split` and `split_by_time`
"""
from mock import MagicMock
import datetime
fakecrs = geometry.CRS('EPSG:4326')
grid = 100 # spatial frequency in crs units
pixel = 10 # square pixel linear dimension in crs units
# if cell(0,0) has lower left corner at grid origin,
# and cell indices increase toward upper right,
# then this will be cell(1,-2).
gridspec = GridSpec(crs=fakecrs,
tile_size=(grid, grid),
resolution=(-pixel, pixel)) # e.g. product gridspec
def make_fake_datasets(num_datasets):
start_time = datetime.datetime(2001, 2, 15)
delta = datetime.timedelta(days=16)
for i in range(num_datasets):
fakedataset = MagicMock()
fakedataset.extent = geometry.box(left=grid,
bottom=-grid,
right=2 * grid,
top=-2 * grid,
crs=fakecrs)
fakedataset.center_time = start_time + (delta * i)
yield fakedataset
fakeindex = PickableMock()
fakeindex.datasets.get_field_names.return_value = [
'time'
] # permit query on time
fakeindex.datasets.search_eager.return_value = list(
make_fake_datasets(100))
# ------ test with time dimension ----
from datacube.api.grid_workflow import GridWorkflow
gw = GridWorkflow(fakeindex, gridspec)
query = dict(product='fake_product_name')
cells = gw.list_cells(**query)
for cell_index, cell in cells.items():
# test Tile.split()
for label, tile in cell.split('time'):
assert tile.shape == (1, 10, 10)
# test Tile.split_by_time()
for year, year_cell in cell.split_by_time(freq='A'):
for t in year_cell.sources.time.values:
assert str(t)[:4] == year
def test_gridspec_upperleft():
""" Test to ensure grid indexes can be counted correctly from bottom left or top left
"""
tile_bbox = geometry.BoundingBox(left=1934400.0,
top=2414800.0,
right=2084400.000,
bottom=2264800.000)
bbox = geometry.BoundingBox(left=1934615,
top=2379460,
right=1937615,
bottom=2376460)
# Upper left - validated against WELD product tile calculator
# http://globalmonitoring.sdstate.edu/projects/weld/tilecalc.php
gs = GridSpec(crs=geometry.CRS('EPSG:5070'),
tile_size=(-150000, 150000),
resolution=(-30, 30),
origin=(3314800.0, -2565600.0))
cells = {index: geobox for index, geobox in list(gs.tiles(bbox))}
assert set(cells.keys()) == {(30, 6)}
assert cells[(30, 6)].extent.boundingbox == tile_bbox
gs = GridSpec(crs=geometry.CRS('EPSG:5070'),
tile_size=(150000, 150000),
resolution=(-30, 30),
origin=(14800.0, -2565600.0))
cells = {index: geobox for index, geobox in list(gs.tiles(bbox))}
assert set(cells.keys()) == {
(30, 15)
} # WELD grid spec has 21 vertical cells -- 21 - 6 = 15
assert cells[(30, 15)].extent.boundingbox == tile_bbox
class AfricaGeobox:
"""
generate the geobox for each tile according to the longitude ande latitude bounds.
"""
def __init__(self,
resolution: Tuple[int, int] = (-20, 20),
crs: str = "epsg:6933"):
target_crs = CRS(crs)
self.albers_africa_N = GridSpec(
crs=target_crs,
tile_size=(96_000.0, 96_000.0), # default
resolution=resolution,
)
africa = box(-18, -38, 60, 30, "epsg:4326")
self.africa_projected = africa.to_crs(crs, resolution=math.inf)
def tile_geobox(self, tile_index: Tuple[int, int]) -> GeoBox:
return self.albers_africa_N.tile_geobox(tile_index)
@property
def geobox_dict(self) -> Dict:
return dict(
self.albers_africa_N.tiles(self.africa_projected.boundingbox))
def test_output_geobox_fail_paths():
from datacube.api.core import output_geobox
gs_nores = GridSpec(crs=geometry.CRS('EPSG:4326'),
tile_size=None,
resolution=None)
with pytest.raises(ValueError):
output_geobox()
with pytest.raises(ValueError):
output_geobox(output_crs='EPSG:4326') # need resolution as well
with pytest.raises(ValueError):
output_geobox(grid_spec=gs_nores) # GridSpec with missing resolution
def mk_utm_gs(
epsg: int,
resolution: Union[Tuple[float, float], float] = 10,
pixels_per_cell: int = 10_000,
origin: Tuple[float, float] = (0, 0),
) -> GridSpec:
if not isinstance(resolution, tuple):
resolution = (-resolution, resolution)
tile_size = tuple([abs(r) * pixels_per_cell for r in resolution])
return GridSpec(
crs=CRS(f"epsg:{epsg}"),
resolution=resolution,
tile_size=tile_size,
origin=origin,
)
def utm_region_code(epsg: Union[int, Tuple[int, int, int]],
tidx: Optional[Tuple[int, int]] = None) -> str:
"""
Examples:
- 32751 -> "51S"
- 32633, 10, 2 -> "33N_10_02"
- 32603, (1, 2) -> "03N_01_02"
"""
if isinstance(epsg, tuple):
tidx = epsg[1:]
epsg = epsg[0]
def test_gridworkflow():
""" Test GridWorkflow with padding option. """
from mock import MagicMock
import datetime
# ----- fake a datacube -----
# e.g. let there be a dataset that coincides with a grid cell
fakecrs = geometry.CRS('EPSG:4326')
grid = 100 # spatial frequency in crs units
pixel = 10 # square pixel linear dimension in crs units
# if cell(0,0) has lower left corner at grid origin,
# and cell indices increase toward upper right,
# then this will be cell(1,-2).
gridspec = GridSpec(crs=fakecrs,
tile_size=(grid, grid),
resolution=(-pixel, pixel)) # e.g. product gridspec
fakedataset = MagicMock()
fakedataset.extent = geometry.box(left=grid,
bottom=-grid,
right=2 * grid,
top=-2 * grid,
crs=fakecrs)
fakedataset.center_time = t = datetime.datetime(2001, 2, 15)
fakedataset.id = uuid.uuid4()
fakeindex = PickableMock()
fakeindex._db = None
fakeindex.datasets.get_field_names.return_value = [
'time'
] # permit query on time
fakeindex.datasets.search_eager.return_value = [fakedataset]
# ------ test without padding ----
from datacube.api.grid_workflow import GridWorkflow
gw = GridWorkflow(fakeindex, gridspec)
# Need to force the fake index otherwise the driver manager will
# only take its _db
gw.index = fakeindex
query = dict(product='fake_product_name',
time=('2001-1-1 00:00:00', '2001-3-31 23:59:59'))
# test backend : that it finds the expected cell/dataset
assert list(gw.cell_observations(**query).keys()) == [(1, -2)]
# again but with geopolygon
assert list(
gw.cell_observations(**query,
geopolygon=gridspec.tile_geobox(
(1, -2)).extent).keys()) == [(1, -2)]
with pytest.raises(ValueError) as e:
list(
gw.cell_observations(**query,
tile_buffer=(1, 1),
geopolygon=gridspec.tile_geobox(
(1, -2)).extent).keys())
assert str(
e.value) == 'Cannot process tile_buffering and geopolygon together.'
# test frontend
assert len(gw.list_tiles(**query)) == 1
# ------ introduce padding --------
assert len(gw.list_tiles(tile_buffer=(20, 20), **query)) == 9
# ------ add another dataset (to test grouping) -----
# consider cell (2,-2)
fakedataset2 = MagicMock()
fakedataset2.extent = geometry.box(left=2 * grid,
bottom=-grid,
right=3 * grid,
top=-2 * grid,
crs=fakecrs)
fakedataset2.center_time = t
fakedataset2.id = uuid.uuid4()
def search_eager(lat=None, lon=None, **kwargs):
return [fakedataset, fakedataset2]
fakeindex.datasets.search_eager = search_eager
# unpadded
assert len(gw.list_tiles(**query)) == 2
ti = numpy.datetime64(t, 'ns')
assert set(gw.list_tiles(**query).keys()) == {(1, -2, ti), (2, -2, ti)}
# padded
assert len(gw.list_tiles(tile_buffer=(20, 20), **
query)) == 12 # not 18=2*9 because of grouping
# -------- inspect particular returned tile objects --------
# check the array shape
tile = gw.list_tiles(**query)[1, -2, ti] # unpadded example
assert grid / pixel == 10
assert tile.shape == (1, 10, 10)
padded_tile = gw.list_tiles(tile_buffer=(20, 20),
**query)[1, -2, ti] # padded example
# assert grid/pixel + 2*gw2.grid_spec.padding == 14 # GREG: understand this
assert padded_tile.shape == (1, 14, 14)
# count the sources
assert len(tile.sources.isel(time=0).item()) == 1
assert len(padded_tile.sources.isel(time=0).item()) == 2
# check the geocoding
assert tile.geobox.alignment == padded_tile.geobox.alignment
assert tile.geobox.affine * (0, 0) == padded_tile.geobox.affine * (2, 2)
assert tile.geobox.affine * (10, 10) == padded_tile.geobox.affine * (
10 + 2, 10 + 2)
# ------- check loading --------
# GridWorkflow accesses the load_data API
# to ultimately convert geobox,sources,measurements to xarray,
# so only thing to check here is the call interface.
measurement = dict(nodata=0, dtype=numpy.int)
fakedataset.type.lookup_measurements.return_value = {'dummy': measurement}
fakedataset2.type = fakedataset.type
from mock import patch
with patch('datacube.api.core.Datacube.load_data') as loader:
data = GridWorkflow.load(tile)
data2 = GridWorkflow.load(padded_tile)
# Note, could also test Datacube.load for consistency (but may require more patching)
assert data is data2 is loader.return_value
assert loader.call_count == 2
# Note, use of positional arguments here is not robust, could spec mock etc.
for (args, kwargs), loadable in zip(loader.call_args_list,
[tile, padded_tile]):
args = list(args)
assert args[0] is loadable.sources
assert args[1] is loadable.geobox
assert list(args[2].values())[0] is measurement
assert 'resampling' in kwargs
# ------- check single cell index extract -------
tile = gw.list_tiles(cell_index=(1, -2), **query)
assert len(tile) == 1
assert tile[1, -2, ti].shape == (1, 10, 10)
assert len(tile[1, -2, ti].sources.values[0]) == 1
padded_tile = gw.list_tiles(cell_index=(1, -2),
tile_buffer=(20, 20),
**query)
assert len(padded_tile) == 1
assert padded_tile[1, -2, ti].shape == (1, 14, 14)
assert len(padded_tile[1, -2, ti].sources.values[0]) == 2
from datacube.utils.geometry import (
CRS,
GeoBox,
apply_affine,
)
from datacube.model import GridSpec
# pylint: disable=invalid-name
epsg4326 = CRS('EPSG:4326')
epsg3577 = CRS('EPSG:3577')
epsg3857 = CRS('EPSG:3857')
AlbersGS = GridSpec(crs=epsg3577,
tile_size=(100000.0, 100000.0),
resolution=(-25, 25),
origin=(0.0, 0.0))
SAMPLE_WKT_WITHOUT_AUTHORITY = '''PROJCS["unnamed",
GEOGCS["unnamed ellipse",
DATUM["unknown",
SPHEROID["unnamed",6378137,0],
EXTENSION["PROJ4_GRIDS","@null"]],
PRIMEM["Greenwich",0],
UNIT["degree",0.0174532925199433]],
PROJECTION["Mercator_2SP"],
PARAMETER["standard_parallel_1",0],
PARAMETER["central_meridian",0],
PARAMETER["false_easting",0],
PARAMETER["false_northing",0],
UNIT["Meter",1]
from typing import Tuple, Optional
from math import pi
from types import SimpleNamespace
import toolz
from datacube.utils.geometry import CRS
from datacube.model import GridSpec, Dataset
from odc.io.text import split_and_check, parse_range_int
epsg3577 = CRS('epsg:3577')
epsg6933 = CRS('epsg:6933')
GRIDS = {
'albers_au_25':
GridSpec(crs=epsg3577,
tile_size=(100_000.0, 100_000.0),
resolution=(-25, 25)),
'albers_africa_10':
GridSpec(crs=epsg6933,
tile_size=(96_000.0, 96_000.0),
resolution=(-10, 10)),
'albers_africa_20':
GridSpec(crs=epsg6933,
tile_size=(96_000.0, 96_000.0),
resolution=(-20, 20)),
'albers_africa_30':
GridSpec(crs=epsg6933,
tile_size=(96_000.0, 96_000.0),
resolution=(-30, 30)),
'albers_africa_60':
GridSpec(crs=epsg6933,
tile_size=(96_000.0, 96_000.0),
epsg3577 = CRS("epsg:3577")
epsg6933 = CRS("epsg:6933")
# Origin was chosen such that there are no negative indexed tiles for any valid
# point within a given CRS's valid region, and also making sure that x=0,y=0
# lines fall on tile edges.
#
# au = gridspec_from_crs(CRS("epsg:3577"),
# tile_size=(96_000, 96_000),
# pad_yx=(5, 5))
#
# So AU tiles with index `y < 5 or x < 5` are outside of the valid range of EPSG:3577.
#
GRIDS = {
"albers_au_25":
GridSpec(crs=epsg3577,
tile_size=(100_000.0, 100_000.0),
resolution=(-25, 25)),
"au":
GridSpec(
crs=epsg3577,
tile_size=(96_000.0, 96_000.0),
resolution=(-96_000, 96_000),
origin=(-5472000.0, -2688000.0),
),
**{
f"au_{n}": GridSpec(
crs=epsg3577,
tile_size=(96_000.0, 96_000.0),
resolution=(-n, n),
origin=(-5472000.0, -2688000.0),
)
import click
from odc import dscache
from odc.dscache.tools.tiling import bin_dataset_stream, bin_by_native_tile, web_gs, extract_native_albers_tile
from datacube.model import GridSpec
import datacube.utils.geometry as geom
GS_ALBERS = GridSpec(crs=geom.CRS('EPSG:3577'),
tile_size=(100000.0, 100000.0),
resolution=(-25, 25))
@click.command('dstiler')
@click.option('--native',
is_flag=True,
help='Use Landsat Path/Row as grouping')
@click.option('--native-albers',
is_flag=True,
help='When datasets are in Albers grid already')
@click.option('--web',
type=int,
help='Use web map tiling regime at supplied zoom level')
@click.argument('dbfile', type=str, nargs=1)
def cli(native, native_albers, web, dbfile):
"""Add spatial grouping to file db.
Default grid is Australian Albers (EPSG:3577) with 100k by 100k tiles. But
you can also group by Landsat path/row (--native), or Google's map tiling
regime (--web zoom_level)
"""
cache = dscache.open_rw(dbfile)
class TaskReader:
def __init__(self,
cache: Union[str, DatasetCache],
product: Optional[OutputProduct] = None):
self._cache_path = None
if isinstance(cache, str):
if cache.startswith("s3://"):
self._cache_path = s3_download(cache)
cache = self._cache_path
cache = DatasetCache.open_ro(cache)
# TODO: verify this things are set in the file
cfg = cache.get_info_dict("stats/config")
grid = cfg["grid"]
gridspec = cache.grids[grid]
self._product = product
self._dscache = cache
self._cfg = cfg
self._grid = grid
self._gridspec = gridspec
self._all_tiles = sorted(idx for idx, _ in cache.tiles(grid))
def is_compatible_resolution(self,
resolution: Tuple[float, float],
tol=1e-8):
for res, sz in zip(resolution, self._gridspec.tile_size):
res = abs(res)
npix = int(sz / res)
if abs(npix * res - sz) > tol:
return False
return True
def change_resolution(self, resolution: Tuple[float, float]):
"""
Modify GridSpec to have different pixel resolution but still covering same tiles as the original.
"""
if not self.is_compatible_resolution(resolution):
raise ValueError(
"Supplied resolution is not compatible with the current GridSpec"
)
gs = self._gridspec
self._gridspec = GridSpec(gs.crs,
gs.tile_size,
resolution=resolution,
origin=gs.origin)
def __del__(self):
if self._cache_path is not None:
os.unlink(self._cache_path)
def __repr__(self) -> str:
grid, path, n = self._grid, str(self._dscache.path), len(
self._all_tiles)
return f"<{path}> grid:{grid} n:{n:,d}"
def _resolve_product(self,
product: Optional[OutputProduct]) -> OutputProduct:
if product is None:
product = self._product
if product is None:
raise ValueError("Product is not supplied and default is not set")
return product
@property
def product(self) -> OutputProduct:
return self._resolve_product(None)
@property
def all_tiles(self) -> List[TileIdx_txy]:
return self._all_tiles
def datasets(self, tile_index: TileIdx_txy) -> Tuple[Dataset, ...]:
return tuple(
ds
for ds in self._dscache.stream_grid_tile(tile_index, self._grid))
def load_task(
self,
tile_index: TileIdx_txy,
product: Optional[OutputProduct] = None,
source: Any = None,
) -> Task:
product = self._resolve_product(product)
dss = self.datasets(tile_index)
tidx_xy = _xy(tile_index)
return Task(
product=product,
tile_index=tidx_xy,
geobox=self._gridspec.tile_geobox(tidx_xy),
time_range=DateTimeRange(tile_index[0]),
datasets=dss,
source=source,
)
def stream(self,
tiles: Iterable[TileIdx_txy],
product: Optional[OutputProduct] = None) -> Iterator[Task]:
product = self._resolve_product(product)
for tidx in tiles:
yield self.load_task(tidx, product)
def stream_from_sqs(
self,
sqs_queue,
product: Optional[OutputProduct] = None,
visibility_timeout: int = 3600,
**kw,
) -> Iterator[Task]:
from odc.aws.queue import get_messages, get_queue
from ._sqs import SQSWorkToken
product = self._resolve_product(product)
if isinstance(sqs_queue, str):
sqs_queue = get_queue(sqs_queue)
for msg in get_messages(sqs_queue,
visibility_timeout=visibility_timeout,
**kw):
# TODO: switch to JSON for SQS message body
token = SQSWorkToken(msg, visibility_timeout)
tidx = parse_task(msg.body)
yield self.load_task(tidx, product, source=token)
def get_grid_spec(self):
return GridSpec(CRS(self.cfg['storage']['crs']),
(self.cfg['storage']['tile_size']['y'],
self.cfg['storage']['tile_size']['x']),
(self.cfg['storage']['resolution']['y'],
self.cfg['storage']['resolution']['x']))
def doc2gs(doc: Document) -> GridSpec:
return GridSpec(crs=CRS(doc['crs']),
tile_size=tuple(doc['tile_size']),
resolution=tuple(doc['resolution']),
origin=tuple(doc['origin']))
