def test_gridspec(): gs = GridSpec(crs=geometry.CRS('EPSG:4326'), tile_size=(1, 1), resolution=(-0.1, 0.1), origin=(10, 10)) poly = geometry.polygon([(10, 12.2), (10.8, 13), (13, 10.8), (12.2, 10), (10, 12.2)], crs=geometry.CRS('EPSG:4326')) cells = { index: geobox for index, geobox in list(gs.tiles_from_geopolygon(poly)) } assert set(cells.keys()) == {(0, 1), (0, 2), (1, 0), (1, 1), (1, 2), (2, 0), (2, 1)} assert numpy.isclose(cells[(2, 0)].coordinates['longitude'].values, numpy.linspace(12.05, 12.95, num=10)).all() assert numpy.isclose(cells[(2, 0)].coordinates['latitude'].values, numpy.linspace(10.95, 10.05, num=10)).all() # check geobox_cache cache = {} poly = gs.tile_geobox((3, 4)).extent (c1, gbox1), = list(gs.tiles_from_geopolygon(poly, geobox_cache=cache)) (c2, gbox2), = list(gs.tiles_from_geopolygon(poly, geobox_cache=cache)) assert c1 == (3, 4) and c2 == c1 assert gbox1 is gbox2
def test_gridspec(): gs = GridSpec(crs=geometry.CRS('EPSG:4326'), tile_size=(1, 1), resolution=(-0.1, 0.1), origin=(10, 10)) poly = geometry.polygon([(10, 12.2), (10.8, 13), (13, 10.8), (12.2, 10), (10, 12.2)], crs=geometry.CRS('EPSG:4326')) cells = {index: geobox for index, geobox in list(gs.tiles_inside_geopolygon(poly))} assert set(cells.keys()) == {(0, 1), (0, 2), (1, 0), (1, 1), (1, 2), (2, 0), (2, 1)} assert numpy.isclose(cells[(2, 0)].coordinates['longitude'].values, numpy.linspace(12.05, 12.95, num=10)).all() assert numpy.isclose(cells[(2, 0)].coordinates['latitude'].values, numpy.linspace(10.95, 10.05, num=10)).all()
def __init__(self, resolution: Tuple[int, int] = (-20, 20), crs: str = "epsg:6933"): target_crs = CRS(crs) self.albers_africa_N = GridSpec( crs=target_crs, tile_size=(96_000.0, 96_000.0), # default resolution=resolution, ) africa = box(-18, -38, 60, 30, "epsg:4326") self.africa_projected = africa.to_crs(crs, resolution=math.inf)
def change_resolution(self, resolution: Tuple[float, float]): """ Modify GridSpec to have different pixel resolution but still covering same tiles as the original. """ if not self.is_compatible_resolution(resolution): raise ValueError( "Supplied resolution is not compatible with the current GridSpec" ) gs = self._gridspec self._gridspec = GridSpec( gs.crs, gs.tile_size, resolution=resolution, origin=gs.origin )
def gs_albers(): from datacube.model import GridSpec import datacube.utils.geometry as geom return GridSpec(crs=geom.CRS('EPSG:3577'), tile_size=(100000.0, 100000.0), resolution=(-25, 25))
def doc2gs(doc: Document) -> GridSpec: return GridSpec( crs=CRS(doc["crs"]), tile_size=tuple(doc["tile_size"]), resolution=tuple(doc["resolution"]), origin=tuple(doc["origin"]), )
def test_wofs_filtered(): cfg = Config('../configs/template_client.yaml') grid_spec = GridSpec(crs=CRS('EPSG:3577'), tile_size=(100000, 100000), resolution=(-25, 25)) cell_index = (17, -39) wf = WofsFiltered(cfg, grid_spec, cell_index) confidence = wf.compute_confidence(cell_index) filtered = wf.compute_confidence_filtered() # Display images: to be removed later with Datacube(app='wofs_summary', env='dev') as dc: gwf = GridWorkflow(dc.index, grid_spec) indexed_tile = gwf.list_cells(cell_index, product='wofs_statistical_summary') # load the data of the tile dataset = gwf.load(tile=indexed_tile[cell_index], measurements=['frequency']) frequency = dataset.data_vars['frequency'].data.ravel().reshape( grid_spec.tile_resolution) # Check with previous run with rasterio.open('confidenceFilteredWOfS_17_-39_epsilon=10.tiff') as f: data = f.read(1) plt.subplot(221) plt.imshow(frequency) plt.subplot(222) plt.imshow(data) plt.subplot(223) plt.imshow(confidence) plt.subplot(224) plt.imshow(filtered) plt.show() wf.compute_and_write()
def parse_gridspec(s: str) -> GridSpec: """ "albers_africa_10" "epsg:6936;10;9600" "epsg:6936;-10x10;9600x9600" """ named_gs = GRIDS.get(s) if named_gs is not None: return named_gs crs, res, shape = split_and_check(s, ';', 3) try: if 'x' in res: res = tuple(float(v) for v in split_and_check(res, 'x', 2)) else: res = float(res) res = (-res, res) if 'x' in shape: shape = parse_range_int(shape, separator='x') else: shape = int(shape) shape = (shape, shape) except ValueError: raise ValueError(f"Failed to parse gridspec: {s}") tsz = tuple(abs(n * res) for n, res in zip(res, shape)) return GridSpec(crs=CRS(crs), tile_size=tsz, resolution=res, origin=(0, 0))
def example_product(name): if name not in PRODUCT_LIST: return None blue = dict(name='blue', dtype='int16', nodata=-999, units='1') green = dict(name='green', dtype='int16', nodata=-999, units='1', aliases=['verde']) flags = {"cloud_acca": {"bits": 10, "values": {"0": "cloud", "1": "no_cloud"}}, "contiguous": {"bits": 8, "values": {"0": False, "1": True}}, "cloud_fmask": {"bits": 11, "values": {"0": "cloud", "1": "no_cloud"}}, "nir_saturated": {"bits": 3, "values": {"0": True, "1": False}}, "red_saturated": {"bits": 2, "values": {"0": True, "1": False}}, "blue_saturated": {"bits": 0, "values": {"0": True, "1": False}}, "green_saturated": {"bits": 1, "values": {"0": True, "1": False}}, "swir1_saturated": {"bits": 4, "values": {"0": True, "1": False}}, "swir2_saturated": {"bits": 7, "values": {"0": True, "1": False}}, "cloud_shadow_acca": {"bits": 12, "values": {"0": "cloud_shadow", "1": "no_cloud_shadow"}}, "cloud_shadow_fmask": {"bits": 13, "values": {"0": "cloud_shadow", "1": "no_cloud_shadow"}}} pixelquality = dict(name='pixelquality', dtype='int16', nodata=0, units='1', flags_definition=flags) result = DatasetType(example_metadata_type(), dict(name=name, description="", metadata_type='eo', metadata={})) result.grid_spec = GridSpec(crs=geometry.CRS('EPSG:3577'), tile_size=(100000., 100000.), resolution=(-25, 25)) if '_pq_' in name: result.definition = {'name': name, 'measurements': [pixelquality]} else: result.definition = {'name': name, 'measurements': [blue, green]} return result
def _parse_gridspec_string(s: str) -> GridSpec: """ "epsg:6936;10;9600" "epsg:6936;-10x10;9600x9600" """ crs, res, shape = split_and_check(s, ";", 3) try: if "x" in res: res = tuple(float(v) for v in split_and_check(res, "x", 2)) else: res = float(res) res = (-res, res) if "x" in shape: shape = parse_range_int(shape, separator="x") else: shape = int(shape) shape = (shape, shape) except ValueError: raise ValueError(f"Failed to parse gridspec: {s}") tsz = tuple(abs(n * res) for n, res in zip(res, shape)) return GridSpec(crs=CRS(crs), tile_size=tsz, resolution=res, origin=(0, 0))
def get_grid_spec(config): storage = config['storage'] crs = CRS(storage['crs']) return GridSpec( crs=crs, tile_size=[storage['tile_size'][dim] for dim in crs.dimensions], resolution=[storage['resolution'][dim] for dim in crs.dimensions])
def web_gs(zoom, tile_size=256): """ Construct grid spec compatible with TerriaJS requests at a given level. Tile indexes should be the same as google maps, except that Y component is negative, this is a limitation of GridSpec class, you can not have tile index direction be different from axis direction, but this is what google indexing is using. http://www.maptiler.org/google-maps-coordinates-tile-bounds-projection/ """ from datacube.utils.geometry import CRS from datacube.model import GridSpec from math import pi R = 6378137 origin = pi * R res0 = 2 * pi * R / tile_size res = res0 * (2**(-zoom)) tsz = 2 * pi * R * (2**(-zoom)) # res*tile_size return GridSpec( crs=CRS('epsg:3857'), tile_size=(tsz, tsz), resolution=(-res, res), # Y,X origin=(origin - tsz, -origin)) # Y,X
def example_product(name): result = DatasetType( example_metadata_type(), dict(name=name, description="", metadata_type='eo', metadata={})) result.grid_spec = GridSpec(crs=geometry.CRS('EPSG:3577'), tile_size=(100000., 100000.), resolution=(-25, 25)) return result
def _make_grid_spec(storage) -> GridSpec: """Make a grid spec based on a storage spec.""" assert 'tile_size' in storage crs = CRS(storage['crs']) return GridSpec( crs=crs, tile_size=[storage['tile_size'][dim] for dim in crs.dimensions], resolution=[storage['resolution'][dim] for dim in crs.dimensions])
def gs_bounds(gs: GridSpec, tiles: Tuple[Tuple[int, int], Tuple[int, int]]) -> Geometry: """ Compute Polygon for a selection of tiles. :param gs: GridSpec :param tiles: (x_range, y_range) X,Y ranges are inclusive on the left and exclusive on the right, same as numpy slicing. """ ((x0, x1), (y0, y1)) = tiles if gs.resolution[0] < 0: gb = gs.tile_geobox((x0, y1 - 1)) else: gb = gs.tile_geobox((x0, y0)) nx = (x1 - x0) * gb.shape[1] ny = (y1 - y0) * gb.shape[0] return polygon_from_transform(nx, ny, gb.affine, gb.crs)
def test_gridworkflow_with_time_depth(): """Test GridWorkflow with time series. Also test `Tile` methods `split` and `split_by_time` """ from mock import MagicMock import datetime fakecrs = geometry.CRS('EPSG:4326') grid = 100 # spatial frequency in crs units pixel = 10 # square pixel linear dimension in crs units # if cell(0,0) has lower left corner at grid origin, # and cell indices increase toward upper right, # then this will be cell(1,-2). gridspec = GridSpec(crs=fakecrs, tile_size=(grid, grid), resolution=(-pixel, pixel)) # e.g. product gridspec def make_fake_datasets(num_datasets): start_time = datetime.datetime(2001, 2, 15) delta = datetime.timedelta(days=16) for i in range(num_datasets): fakedataset = MagicMock() fakedataset.extent = geometry.box(left=grid, bottom=-grid, right=2 * grid, top=-2 * grid, crs=fakecrs) fakedataset.center_time = start_time + (delta * i) yield fakedataset fakeindex = PickableMock() fakeindex.datasets.get_field_names.return_value = [ 'time' ] # permit query on time fakeindex.datasets.search_eager.return_value = list( make_fake_datasets(100)) # ------ test with time dimension ---- from datacube.api.grid_workflow import GridWorkflow gw = GridWorkflow(fakeindex, gridspec) query = dict(product='fake_product_name') cells = gw.list_cells(**query) for cell_index, cell in cells.items(): # test Tile.split() for label, tile in cell.split('time'): assert tile.shape == (1, 10, 10) # test Tile.split_by_time() for year, year_cell in cell.split_by_time(freq='A'): for t in year_cell.sources.time.values: assert str(t)[:4] == year
def test_gridspec_upperleft(): """ Test to ensure grid indexes can be counted correctly from bottom left or top left """ tile_bbox = geometry.BoundingBox(left=1934400.0, top=2414800.0, right=2084400.000, bottom=2264800.000) bbox = geometry.BoundingBox(left=1934615, top=2379460, right=1937615, bottom=2376460) # Upper left - validated against WELD product tile calculator # http://globalmonitoring.sdstate.edu/projects/weld/tilecalc.php gs = GridSpec(crs=geometry.CRS('EPSG:5070'), tile_size=(-150000, 150000), resolution=(-30, 30), origin=(3314800.0, -2565600.0)) cells = {index: geobox for index, geobox in list(gs.tiles(bbox))} assert set(cells.keys()) == {(30, 6)} assert cells[(30, 6)].extent.boundingbox == tile_bbox gs = GridSpec(crs=geometry.CRS('EPSG:5070'), tile_size=(150000, 150000), resolution=(-30, 30), origin=(14800.0, -2565600.0)) cells = {index: geobox for index, geobox in list(gs.tiles(bbox))} assert set(cells.keys()) == { (30, 15) } # WELD grid spec has 21 vertical cells -- 21 - 6 = 15 assert cells[(30, 15)].extent.boundingbox == tile_bbox
class AfricaGeobox: """ generate the geobox for each tile according to the longitude ande latitude bounds. """ def __init__(self, resolution: Tuple[int, int] = (-20, 20), crs: str = "epsg:6933"): target_crs = CRS(crs) self.albers_africa_N = GridSpec( crs=target_crs, tile_size=(96_000.0, 96_000.0), # default resolution=resolution, ) africa = box(-18, -38, 60, 30, "epsg:4326") self.africa_projected = africa.to_crs(crs, resolution=math.inf) def tile_geobox(self, tile_index: Tuple[int, int]) -> GeoBox: return self.albers_africa_N.tile_geobox(tile_index) @property def geobox_dict(self) -> Dict: return dict( self.albers_africa_N.tiles(self.africa_projected.boundingbox))
def test_output_geobox_fail_paths(): from datacube.api.core import output_geobox gs_nores = GridSpec(crs=geometry.CRS('EPSG:4326'), tile_size=None, resolution=None) with pytest.raises(ValueError): output_geobox() with pytest.raises(ValueError): output_geobox(output_crs='EPSG:4326') # need resolution as well with pytest.raises(ValueError): output_geobox(grid_spec=gs_nores) # GridSpec with missing resolution
def mk_utm_gs( epsg: int, resolution: Union[Tuple[float, float], float] = 10, pixels_per_cell: int = 10_000, origin: Tuple[float, float] = (0, 0), ) -> GridSpec: if not isinstance(resolution, tuple): resolution = (-resolution, resolution) tile_size = tuple([abs(r) * pixels_per_cell for r in resolution]) return GridSpec( crs=CRS(f"epsg:{epsg}"), resolution=resolution, tile_size=tile_size, origin=origin, ) def utm_region_code(epsg: Union[int, Tuple[int, int, int]], tidx: Optional[Tuple[int, int]] = None) -> str: """ Examples: - 32751 -> "51S" - 32633, 10, 2 -> "33N_10_02" - 32603, (1, 2) -> "03N_01_02" """ if isinstance(epsg, tuple): tidx = epsg[1:] epsg = epsg[0]
def test_gridworkflow(): """ Test GridWorkflow with padding option. """ from mock import MagicMock import datetime # ----- fake a datacube ----- # e.g. let there be a dataset that coincides with a grid cell fakecrs = geometry.CRS('EPSG:4326') grid = 100 # spatial frequency in crs units pixel = 10 # square pixel linear dimension in crs units # if cell(0,0) has lower left corner at grid origin, # and cell indices increase toward upper right, # then this will be cell(1,-2). gridspec = GridSpec(crs=fakecrs, tile_size=(grid, grid), resolution=(-pixel, pixel)) # e.g. product gridspec fakedataset = MagicMock() fakedataset.extent = geometry.box(left=grid, bottom=-grid, right=2 * grid, top=-2 * grid, crs=fakecrs) fakedataset.center_time = t = datetime.datetime(2001, 2, 15) fakedataset.id = uuid.uuid4() fakeindex = PickableMock() fakeindex._db = None fakeindex.datasets.get_field_names.return_value = [ 'time' ] # permit query on time fakeindex.datasets.search_eager.return_value = [fakedataset] # ------ test without padding ---- from datacube.api.grid_workflow import GridWorkflow gw = GridWorkflow(fakeindex, gridspec) # Need to force the fake index otherwise the driver manager will # only take its _db gw.index = fakeindex query = dict(product='fake_product_name', time=('2001-1-1 00:00:00', '2001-3-31 23:59:59')) # test backend : that it finds the expected cell/dataset assert list(gw.cell_observations(**query).keys()) == [(1, -2)] # again but with geopolygon assert list( gw.cell_observations(**query, geopolygon=gridspec.tile_geobox( (1, -2)).extent).keys()) == [(1, -2)] with pytest.raises(ValueError) as e: list( gw.cell_observations(**query, tile_buffer=(1, 1), geopolygon=gridspec.tile_geobox( (1, -2)).extent).keys()) assert str( e.value) == 'Cannot process tile_buffering and geopolygon together.' # test frontend assert len(gw.list_tiles(**query)) == 1 # ------ introduce padding -------- assert len(gw.list_tiles(tile_buffer=(20, 20), **query)) == 9 # ------ add another dataset (to test grouping) ----- # consider cell (2,-2) fakedataset2 = MagicMock() fakedataset2.extent = geometry.box(left=2 * grid, bottom=-grid, right=3 * grid, top=-2 * grid, crs=fakecrs) fakedataset2.center_time = t fakedataset2.id = uuid.uuid4() def search_eager(lat=None, lon=None, **kwargs): return [fakedataset, fakedataset2] fakeindex.datasets.search_eager = search_eager # unpadded assert len(gw.list_tiles(**query)) == 2 ti = numpy.datetime64(t, 'ns') assert set(gw.list_tiles(**query).keys()) == {(1, -2, ti), (2, -2, ti)} # padded assert len(gw.list_tiles(tile_buffer=(20, 20), ** query)) == 12 # not 18=2*9 because of grouping # -------- inspect particular returned tile objects -------- # check the array shape tile = gw.list_tiles(**query)[1, -2, ti] # unpadded example assert grid / pixel == 10 assert tile.shape == (1, 10, 10) padded_tile = gw.list_tiles(tile_buffer=(20, 20), **query)[1, -2, ti] # padded example # assert grid/pixel + 2*gw2.grid_spec.padding == 14 # GREG: understand this assert padded_tile.shape == (1, 14, 14) # count the sources assert len(tile.sources.isel(time=0).item()) == 1 assert len(padded_tile.sources.isel(time=0).item()) == 2 # check the geocoding assert tile.geobox.alignment == padded_tile.geobox.alignment assert tile.geobox.affine * (0, 0) == padded_tile.geobox.affine * (2, 2) assert tile.geobox.affine * (10, 10) == padded_tile.geobox.affine * ( 10 + 2, 10 + 2) # ------- check loading -------- # GridWorkflow accesses the load_data API # to ultimately convert geobox,sources,measurements to xarray, # so only thing to check here is the call interface. measurement = dict(nodata=0, dtype=numpy.int) fakedataset.type.lookup_measurements.return_value = {'dummy': measurement} fakedataset2.type = fakedataset.type from mock import patch with patch('datacube.api.core.Datacube.load_data') as loader: data = GridWorkflow.load(tile) data2 = GridWorkflow.load(padded_tile) # Note, could also test Datacube.load for consistency (but may require more patching) assert data is data2 is loader.return_value assert loader.call_count == 2 # Note, use of positional arguments here is not robust, could spec mock etc. for (args, kwargs), loadable in zip(loader.call_args_list, [tile, padded_tile]): args = list(args) assert args[0] is loadable.sources assert args[1] is loadable.geobox assert list(args[2].values())[0] is measurement assert 'resampling' in kwargs # ------- check single cell index extract ------- tile = gw.list_tiles(cell_index=(1, -2), **query) assert len(tile) == 1 assert tile[1, -2, ti].shape == (1, 10, 10) assert len(tile[1, -2, ti].sources.values[0]) == 1 padded_tile = gw.list_tiles(cell_index=(1, -2), tile_buffer=(20, 20), **query) assert len(padded_tile) == 1 assert padded_tile[1, -2, ti].shape == (1, 14, 14) assert len(padded_tile[1, -2, ti].sources.values[0]) == 2
from datacube.utils.geometry import ( CRS, GeoBox, apply_affine, ) from datacube.model import GridSpec # pylint: disable=invalid-name epsg4326 = CRS('EPSG:4326') epsg3577 = CRS('EPSG:3577') epsg3857 = CRS('EPSG:3857') AlbersGS = GridSpec(crs=epsg3577, tile_size=(100000.0, 100000.0), resolution=(-25, 25), origin=(0.0, 0.0)) SAMPLE_WKT_WITHOUT_AUTHORITY = '''PROJCS["unnamed", GEOGCS["unnamed ellipse", DATUM["unknown", SPHEROID["unnamed",6378137,0], EXTENSION["PROJ4_GRIDS","@null"]], PRIMEM["Greenwich",0], UNIT["degree",0.0174532925199433]], PROJECTION["Mercator_2SP"], PARAMETER["standard_parallel_1",0], PARAMETER["central_meridian",0], PARAMETER["false_easting",0], PARAMETER["false_northing",0], UNIT["Meter",1]
from typing import Tuple, Optional from math import pi from types import SimpleNamespace import toolz from datacube.utils.geometry import CRS from datacube.model import GridSpec, Dataset from odc.io.text import split_and_check, parse_range_int epsg3577 = CRS('epsg:3577') epsg6933 = CRS('epsg:6933') GRIDS = { 'albers_au_25': GridSpec(crs=epsg3577, tile_size=(100_000.0, 100_000.0), resolution=(-25, 25)), 'albers_africa_10': GridSpec(crs=epsg6933, tile_size=(96_000.0, 96_000.0), resolution=(-10, 10)), 'albers_africa_20': GridSpec(crs=epsg6933, tile_size=(96_000.0, 96_000.0), resolution=(-20, 20)), 'albers_africa_30': GridSpec(crs=epsg6933, tile_size=(96_000.0, 96_000.0), resolution=(-30, 30)), 'albers_africa_60': GridSpec(crs=epsg6933, tile_size=(96_000.0, 96_000.0),
epsg3577 = CRS("epsg:3577") epsg6933 = CRS("epsg:6933") # Origin was chosen such that there are no negative indexed tiles for any valid # point within a given CRS's valid region, and also making sure that x=0,y=0 # lines fall on tile edges. # # au = gridspec_from_crs(CRS("epsg:3577"), # tile_size=(96_000, 96_000), # pad_yx=(5, 5)) # # So AU tiles with index `y < 5 or x < 5` are outside of the valid range of EPSG:3577. # GRIDS = { "albers_au_25": GridSpec(crs=epsg3577, tile_size=(100_000.0, 100_000.0), resolution=(-25, 25)), "au": GridSpec( crs=epsg3577, tile_size=(96_000.0, 96_000.0), resolution=(-96_000, 96_000), origin=(-5472000.0, -2688000.0), ), **{ f"au_{n}": GridSpec( crs=epsg3577, tile_size=(96_000.0, 96_000.0), resolution=(-n, n), origin=(-5472000.0, -2688000.0), )
import click from odc import dscache from odc.dscache.tools.tiling import bin_dataset_stream, bin_by_native_tile, web_gs, extract_native_albers_tile from datacube.model import GridSpec import datacube.utils.geometry as geom GS_ALBERS = GridSpec(crs=geom.CRS('EPSG:3577'), tile_size=(100000.0, 100000.0), resolution=(-25, 25)) @click.command('dstiler') @click.option('--native', is_flag=True, help='Use Landsat Path/Row as grouping') @click.option('--native-albers', is_flag=True, help='When datasets are in Albers grid already') @click.option('--web', type=int, help='Use web map tiling regime at supplied zoom level') @click.argument('dbfile', type=str, nargs=1) def cli(native, native_albers, web, dbfile): """Add spatial grouping to file db. Default grid is Australian Albers (EPSG:3577) with 100k by 100k tiles. But you can also group by Landsat path/row (--native), or Google's map tiling regime (--web zoom_level) """ cache = dscache.open_rw(dbfile)
class TaskReader: def __init__(self, cache: Union[str, DatasetCache], product: Optional[OutputProduct] = None): self._cache_path = None if isinstance(cache, str): if cache.startswith("s3://"): self._cache_path = s3_download(cache) cache = self._cache_path cache = DatasetCache.open_ro(cache) # TODO: verify this things are set in the file cfg = cache.get_info_dict("stats/config") grid = cfg["grid"] gridspec = cache.grids[grid] self._product = product self._dscache = cache self._cfg = cfg self._grid = grid self._gridspec = gridspec self._all_tiles = sorted(idx for idx, _ in cache.tiles(grid)) def is_compatible_resolution(self, resolution: Tuple[float, float], tol=1e-8): for res, sz in zip(resolution, self._gridspec.tile_size): res = abs(res) npix = int(sz / res) if abs(npix * res - sz) > tol: return False return True def change_resolution(self, resolution: Tuple[float, float]): """ Modify GridSpec to have different pixel resolution but still covering same tiles as the original. """ if not self.is_compatible_resolution(resolution): raise ValueError( "Supplied resolution is not compatible with the current GridSpec" ) gs = self._gridspec self._gridspec = GridSpec(gs.crs, gs.tile_size, resolution=resolution, origin=gs.origin) def __del__(self): if self._cache_path is not None: os.unlink(self._cache_path) def __repr__(self) -> str: grid, path, n = self._grid, str(self._dscache.path), len( self._all_tiles) return f"<{path}> grid:{grid} n:{n:,d}" def _resolve_product(self, product: Optional[OutputProduct]) -> OutputProduct: if product is None: product = self._product if product is None: raise ValueError("Product is not supplied and default is not set") return product @property def product(self) -> OutputProduct: return self._resolve_product(None) @property def all_tiles(self) -> List[TileIdx_txy]: return self._all_tiles def datasets(self, tile_index: TileIdx_txy) -> Tuple[Dataset, ...]: return tuple( ds for ds in self._dscache.stream_grid_tile(tile_index, self._grid)) def load_task( self, tile_index: TileIdx_txy, product: Optional[OutputProduct] = None, source: Any = None, ) -> Task: product = self._resolve_product(product) dss = self.datasets(tile_index) tidx_xy = _xy(tile_index) return Task( product=product, tile_index=tidx_xy, geobox=self._gridspec.tile_geobox(tidx_xy), time_range=DateTimeRange(tile_index[0]), datasets=dss, source=source, ) def stream(self, tiles: Iterable[TileIdx_txy], product: Optional[OutputProduct] = None) -> Iterator[Task]: product = self._resolve_product(product) for tidx in tiles: yield self.load_task(tidx, product) def stream_from_sqs( self, sqs_queue, product: Optional[OutputProduct] = None, visibility_timeout: int = 3600, **kw, ) -> Iterator[Task]: from odc.aws.queue import get_messages, get_queue from ._sqs import SQSWorkToken product = self._resolve_product(product) if isinstance(sqs_queue, str): sqs_queue = get_queue(sqs_queue) for msg in get_messages(sqs_queue, visibility_timeout=visibility_timeout, **kw): # TODO: switch to JSON for SQS message body token = SQSWorkToken(msg, visibility_timeout) tidx = parse_task(msg.body) yield self.load_task(tidx, product, source=token)
def get_grid_spec(self): return GridSpec(CRS(self.cfg['storage']['crs']), (self.cfg['storage']['tile_size']['y'], self.cfg['storage']['tile_size']['x']), (self.cfg['storage']['resolution']['y'], self.cfg['storage']['resolution']['x']))
def doc2gs(doc: Document) -> GridSpec: return GridSpec(crs=CRS(doc['crs']), tile_size=tuple(doc['tile_size']), resolution=tuple(doc['resolution']), origin=tuple(doc['origin']))