def geobox_info(extent, valid_data=None):
image_bounds = extent.boundingbox
data_bounds = valid_data.boundingbox if valid_data else image_bounds
ul = geometry.point(data_bounds.left, data_bounds.top, crs=extent.crs).to_crs(CRS('EPSG:4326'))
ur = geometry.point(data_bounds.right, data_bounds.top, crs=extent.crs).to_crs(CRS('EPSG:4326'))
lr = geometry.point(data_bounds.right, data_bounds.bottom, crs=extent.crs).to_crs(CRS('EPSG:4326'))
ll = geometry.point(data_bounds.left, data_bounds.bottom, crs=extent.crs).to_crs(CRS('EPSG:4326'))
doc = {
'extent': {
'coord': {
'ul': {'lon': ul.points[0][0], 'lat': ul.points[0][1]},
'ur': {'lon': ur.points[0][0], 'lat': ur.points[0][1]},
'lr': {'lon': lr.points[0][0], 'lat': lr.points[0][1]},
'll': {'lon': ll.points[0][0], 'lat': ll.points[0][1]},
}
},
'grid_spatial': {
'projection': {
'spatial_reference': str(extent.crs),
'geo_ref_points': {
'ul': {'x': image_bounds.left, 'y': image_bounds.top},
'ur': {'x': image_bounds.right, 'y': image_bounds.top},
'll': {'x': image_bounds.left, 'y': image_bounds.bottom},
'lr': {'x': image_bounds.right, 'y': image_bounds.bottom},
}
}
}
}
if valid_data:
doc['grid_spatial']['projection']['valid_data'] = valid_data.__geo_interface__
return doc
def test_gridspec_upperleft():
""" Test to ensure grid indexes can be counted correctly from bottom left or top left
"""
tile_bbox = BoundingBox(left=1934400.0,
top=2414800.0,
right=2084400.000,
bottom=2264800.000)
bbox = BoundingBox(left=1934615,
top=2379460,
right=1937615,
bottom=2376460)
# Upper left - validated against WELD product tile calculator
# http://globalmonitoring.sdstate.edu/projects/weld/tilecalc.php
gs = GridSpec(crs=CRS('EPSG:5070'),
tile_size=(-150000, 150000),
resolution=(-30, 30),
origin=(3314800.0, -2565600.0))
cells = {index: geobox for index, geobox in list(gs.tiles(bbox))}
assert set(cells.keys()) == {(30, 6)}
assert cells[(30, 6)].extent.boundingbox == tile_bbox
gs = GridSpec(crs=CRS('EPSG:5070'),
tile_size=(150000, 150000),
resolution=(-30, 30),
origin=(14800.0, -2565600.0))
cells = {index: geobox for index, geobox in list(gs.tiles(bbox))}
assert set(cells.keys()) == {
(30, 15)
} # WELD grid spec has 21 vertical cells -- 21 - 6 = 15
assert cells[(30, 15)].extent.boundingbox == tile_bbox
def test_geobox():
points_list = [
[(148.2697, -35.20111), (149.31254, -35.20111),
(149.31254, -36.331431), (148.2697, -36.331431)],
[(148.2697, 35.20111), (149.31254, 35.20111), (149.31254, 36.331431),
(148.2697, 36.331431)],
[(-148.2697, 35.20111), (-149.31254, 35.20111),
(-149.31254, 36.331431), (-148.2697, 36.331431)],
[(-148.2697, -35.20111), (-149.31254, -35.20111),
(-149.31254, -36.331431), (-148.2697, -36.331431),
(148.2697, -35.20111)],
]
for points in points_list:
polygon = geometry.polygon(points, crs=CRS('EPSG:3577'))
resolution = (-25, 25)
geobox = GeoBox.from_geopolygon(polygon, resolution)
assert abs(resolution[0]) > abs(geobox.extent.boundingbox.left -
polygon.boundingbox.left)
assert abs(resolution[0]) > abs(geobox.extent.boundingbox.right -
polygon.boundingbox.right)
assert abs(resolution[1]) > abs(geobox.extent.boundingbox.top -
polygon.boundingbox.top)
assert abs(resolution[1]) > abs(geobox.extent.boundingbox.bottom -
polygon.boundingbox.bottom)
def get_grid_spec(config):
storage = config['storage']
crs = CRS(storage['crs'])
return GridSpec(
crs=crs,
tile_size=[storage['tile_size'][dim] for dim in crs.dimensions],
resolution=[storage['resolution'][dim] for dim in crs.dimensions])
def test_write_dataset_to_netcdf(tmpnetcdf_filename):
affine = Affine.scale(0.1, 0.1) * Affine.translation(20, 30)
geobox = GeoBox(100, 100, affine, CRS(GEO_PROJ))
dataset = xarray.Dataset(attrs={
'extent': geobox.extent,
'crs': geobox.crs
})
for name, coord in geobox.coordinates.items():
dataset[name] = (name, coord.labels, {'units': coord.units})
dataset['B10'] = (geobox.dimensions,
numpy.arange(10000).reshape(geobox.shape), {
'nodata': 0,
'units': '1'
})
write_dataset_to_netcdf(dataset, {'foo': 'bar'},
{'B10': {
'attrs': {
'abc': 'xyz'
}
}}, Path(tmpnetcdf_filename))
with netCDF4.Dataset(tmpnetcdf_filename) as nco:
nco.set_auto_mask(False)
assert 'B10' in nco.variables
var = nco.variables['B10']
assert (var[:] == dataset['B10'].values).all()
assert 'foo' in nco.ncattrs()
assert nco.getncattr('foo') == 'bar'
assert 'abc' in var.ncattrs()
assert var.getncattr('abc') == 'xyz'
def test_crs_equality():
a = CRS(
"""PROJCS["unnamed",GEOGCS["Unknown datum based upon the custom spheroid",
DATUM["Not specified (based on custom spheroid)",SPHEROID["Custom spheroid",6371007.181,0]],
PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],
PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],
PARAMETER["false_northing",0],UNIT["Meter",1]]""")
b = CRS(
"""PROJCS["unnamed",GEOGCS["unnamed ellipse",DATUM["unknown",SPHEROID["unnamed",6371007.181,0]],
PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],PROJECTION["Sinusoidal"],
PARAMETER["longitude_of_center",0],PARAMETER["false_easting",0],
PARAMETER["false_northing",0],UNIT["Meter",1]]""")
c = CRS(
'+a=6371007.181 +b=6371007.181 +units=m +y_0=0 +proj=sinu +lon_0=0 +no_defs +x_0=0'
)
assert a == b
assert a == c
assert b == c
assert a != CRS('EPSG:4326')
a = CRS(
"""GEOGCS["GEOCENTRIC DATUM of AUSTRALIA",DATUM["GDA94",SPHEROID["GRS80",6378137,298.257222101]],
PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]]""")
b = CRS(
"""GEOGCS["GRS 1980(IUGG, 1980)",DATUM["unknown",SPHEROID["GRS80",6378137,298.257222101]],
PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]]""")
c = CRS('+proj=longlat +no_defs +ellps=GRS80')
assert a == b
assert a == c
assert b == c
def test_gridspec():
gs = GridSpec(crs=CRS('EPSG:4326'),
tile_size=(1, 1),
resolution=(-0.1, 0.1),
origin=(10, 10))
poly = geometry.polygon([(10, 12.2), (10.8, 13), (13, 10.8), (12.2, 10),
(10, 12.2)],
crs=CRS('EPSG:4326'))
cells = {
index: geobox
for index, geobox in list(gs.tiles_inside_geopolygon(poly))
}
assert set(cells.keys()) == {(0, 1), (0, 2), (1, 0), (1, 1), (1, 2),
(2, 0), (2, 1)}
assert numpy.isclose(cells[(2, 0)].coordinates['longitude'].values,
numpy.linspace(12.05, 12.95, num=10)).all()
assert numpy.isclose(cells[(2, 0)].coordinates['latitude'].values,
numpy.linspace(10.95, 10.05, num=10)).all()
def __init__(self):
self.crs = CRS('EPSG:4326')
self.transform = Affine(0.25, 0, 100, 0, -0.25, -30)
self.nodata = -999
self.shape = (613, 597)
self.data = numpy.full(self.shape, self.nodata, dtype='int16')
self.data[:256, :256] = 100
self.data[:256, 256:512] = 200
self.data[256:512, :256] = 300
self.data[256:512, 256:512] = 400
def _write_geographical_extents_attributes(nco, extent):
geo_extents = extent.to_crs(CRS("EPSG:4326"))
nco.geospatial_bounds = geo_extents.wkt
nco.geospatial_bounds_crs = "EPSG:4326"
geo_bounds = geo_extents.boundingbox
nco.geospatial_lat_min = geo_bounds.bottom
nco.geospatial_lat_max = geo_bounds.top
nco.geospatial_lat_units = "degrees_north"
nco.geospatial_lon_min = geo_bounds.left
nco.geospatial_lon_max = geo_bounds.right
nco.geospatial_lon_units = "degrees_east"
def _write_geographical_extents_attributes(nco, extent):
geo_extents = extent.to_crs(CRS("EPSG:4326")).points
geo_extents.append(geo_extents[0])
nco.geospatial_bounds = "POLYGON((" + ", ".join("{0} {1}".format(*p) for p in geo_extents) + "))"
nco.geospatial_bounds_crs = "EPSG:4326"
nco.geospatial_lat_min = min(lat for lon, lat in geo_extents)
nco.geospatial_lat_max = max(lat for lon, lat in geo_extents)
nco.geospatial_lat_units = "degrees_north"
nco.geospatial_lon_min = min(lon for lon, lat in geo_extents)
nco.geospatial_lon_max = max(lon for lon, lat in geo_extents)
nco.geospatial_lon_units = "degrees_east"
def check_open_with_api(index):
from datacube.api.core import Datacube
datacube = Datacube(index=index)
input_type_name = 'ls5_nbar_albers'
input_type = datacube.index.datasets.types.get_by_name(input_type_name)
geobox = GeoBox(200, 200, Affine(25, 0.0, 1500000, 0.0, -25, -3900000), CRS('EPSG:3577'))
observations = datacube.product_observations(product='ls5_nbar_albers', geopolygon=geobox.extent)
sources = datacube.product_sources(observations, lambda ds: ds.center_time, 'time',
'seconds since 1970-01-01 00:00:00')
data = datacube.product_data(sources, geobox, input_type.measurements.values())
assert data.blue.shape == (1, 200, 200)
def set_geobox_info(doc, crs, extent):
bb = extent.boundingbox
gp = GeoPolygon([(bb.left, bb.top), (bb.right, bb.top),
(bb.right, bb.bottom), (bb.left, bb.bottom)],
crs).to_crs(CRS('EPSG:4326'))
doc.update({
'extent': {
'coord': {
'ul': {
'lon': gp.points[0][0],
'lat': gp.points[0][1]
},
'ur': {
'lon': gp.points[1][0],
'lat': gp.points[1][1]
},
'lr': {
'lon': gp.points[2][0],
'lat': gp.points[2][1]
},
'll': {
'lon': gp.points[3][0],
'lat': gp.points[3][1]
},
}
},
'grid_spatial': {
'projection': {
'spatial_reference': str(crs),
'geo_ref_points': {
'ul': {
'x': bb.left,
'y': bb.top
},
'ur': {
'x': bb.right,
'y': bb.top
},
'll': {
'x': bb.left,
'y': bb.bottom
},
'lr': {
'x': bb.right,
'y': bb.bottom
},
}
}
}
})
def check_open_with_api(index):
from datacube import Datacube
dc = Datacube(index=index)
input_type_name = 'ls5_nbar_albers'
input_type = dc.index.products.get_by_name(input_type_name)
geobox = GeoBox(200, 200, Affine(25, 0.0, 1500000, 0.0, -25, -3900000),
CRS('EPSG:3577'))
observations = dc.find_datasets(product='ls5_nbar_albers',
geopolygon=geobox.extent)
group_by = query_group_by('time')
sources = dc.group_datasets(observations, group_by)
data = dc.load_data(sources, geobox, input_type.measurements.values())
assert data.blue.shape == (1, 200, 200)
def _get_geobox(observations,
output_crs=None,
resolution=None,
like=None,
align=None,
**query):
crs = CRS(output_crs) if output_crs else query_crs_like(
like) or get_crs(observations)
geopolygon = query_geopolygon(
**query) or query_geopolygon_like(like) or get_bounds(
observations, crs)
resolution = resolution or query_resolution_like(
like) or get_resolution(observations)
geobox = GeoBox.from_geopolygon(geopolygon, resolution, crs, align)
return geobox
def open(self):
if self._descriptor['path']:
if Path(self._descriptor['path']).is_absolute():
filename = self._descriptor['path']
else:
filename = str(
self.local_path.parent.joinpath(self._descriptor['path']))
else:
filename = str(self.local_path)
getmethefile(filename)
for nasty_format in ('netcdf', 'hdf'):
if nasty_format in self.format.lower():
filename = 'file://%s:%s:%s' % (self.format, filename,
self._descriptor['layer'])
bandnumber = None
break
else:
bandnumber = self._descriptor.get('layer', 1)
try:
_LOG.debug("openening %s, band %s", filename, bandnumber)
with rasterio.open(filename) as src:
if bandnumber is None:
if 'netcdf' in self.format.lower():
bandnumber = self.wheres_my_band(src, self.time)
else:
bandnumber = 1
# print("in storage.py in DatasetSource.open")
# print (src.transform)
# print (src.affine)
a = tuple(src.transform)
# print (Affine(a[0], a[1], a[2], a[3], a[4], a[5]))
self.transform = src.affine
self.crs = CRS(str(src.crs_wkt))
self.nodata = src.nodatavals[0] or self._bandinfo.get('nodata')
yield rasterio.band(src, bandnumber)
except Exception as e:
_LOG.error("Error opening source dataset: %s", filename)
raise e
def open(self):
if self._descriptor['path']:
if Path(self._descriptor['path']).is_absolute():
filename = self._descriptor['path']
else:
filename = str(
self.local_path.parent.joinpath(self._descriptor['path']))
else:
filename = str(self.local_path)
for nasty_format in ('netcdf', 'hdf'):
if nasty_format in self.format.lower():
filename = 'file://%s:%s:%s' % (self.format, filename,
self._descriptor['layer'])
bandnumber = None
break
else:
bandnumber = self._descriptor.get('layer', 1)
try:
_LOG.debug("openening %s, band %s", filename, bandnumber)
with rasterio.open(filename) as src:
if bandnumber is None:
if 'netcdf' in self.format.lower():
bandnumber = self.wheres_my_band(src, self.time)
else:
bandnumber = 1
self.transform = src.affine
try:
self.crs = CRS(_rasterio_crs_wkt(src))
except ValueError:
pass
self.dtype = numpy.dtype(src.dtypes[0])
self.nodata = self.dtype.type(
src.nodatavals[0] if src.nodatavals[0] is not None else
self._bandinfo.get('nodata'))
yield rasterio.band(src, bandnumber)
except Exception as e:
_LOG.error("Error opening source dataset: %s", filename)
raise e
def solar_vector(p, time, crs):
poly = GeoPolygon([p, (p[0], p[1] + 100)], crs).to_crs(CRS('EPSG:4326'))
lon, lat = poly.points[0]
dlon = poly.points[1][0] - lon
dlat = poly.points[1][1] - lat
# azimuth north to east of the vertical direction of the crs
vert_az = math.atan2(dlon * math.cos(math.radians(lat)), dlat)
observer = ephem.Observer()
observer.lat = math.radians(lat)
observer.lon = math.radians(lon)
observer.date = time
sun = ephem.Sun(observer)
sun_az = sun.az - vert_az
x = math.sin(sun_az) * math.cos(sun.alt)
y = -math.cos(sun_az) * math.cos(sun.alt)
z = math.sin(sun.alt)
return x, y, z, sun_az, sun.alt
def load(self,
product=None,
measurements=None,
output_crs=None,
resolution=None,
resampling=None,
stack=False,
dask_chunks=None,
like=None,
fuse_func=None,
align=None,
datasets=None,
**query):
"""
Load data as an ``xarray`` object. Each measurement will be a data variable in the :class:`xarray.Dataset`.
See the `xarray documentation <http://xarray.pydata.org/en/stable/data-structures.html>`_ for usage of the
:class:`xarray.Dataset` and :class:`xarray.DataArray` objects.
**Product and Measurements**
A product can be specified using the product name, or by search fields that uniquely describe a single
product.
::
product='ls5_ndvi_albers'
See :meth:`list_products` for the list of products with their names and properties.
A product can also be selected by searched using fields, but must only match one product.
::
platform='LANDSAT_5',
product_type='ndvi'
The ``measurements`` argument is a list of measurement names, as listed in :meth:`list_measurements`.
If not provided, all measurements for the product will be returned.
::
measurements=['red', 'nir', swir2']
**Dimensions**
Spatial dimensions can specified using the ``longitude``/``latitude`` and ``x``/``y`` fields.
The CRS of this query is assumed to be WGS84/EPSG:4326 unless the ``crs`` field is supplied,
even if the stored data is in another projection or the `output_crs` is specified.
The dimensions ``longitude``/``latitude`` and ``x``/``y`` can be used interchangeably.
::
latitude=(-34.5, -35.2), longitude=(148.3, 148.7)
or ::
x=(1516200, 1541300), y=(-3867375, -3867350), crs='EPSG:3577'
The ``time`` dimension can be specified using a tuple of datetime objects or strings with
`YYYY-MM-DD hh:mm:ss` format. E.g::
time=('2001-04', '2001-07')
For EO-specific datasets that are based around scenes, the time dimension can be reduced to the day level,
using solar day to keep scenes together.
::
group_by='solar_day'
For data that has different values for the scene overlap the requires more complex rules for combining data,
such as GA's Pixel Quality dataset, a function can be provided to the merging into a single time slice.
::
def pq_fuser(dest, src):
valid_bit = 8
valid_val = (1 << valid_bit)
no_data_dest_mask = ~(dest & valid_val).astype(bool)
np.copyto(dest, src, where=no_data_dest_mask)
both_data_mask = (valid_val & dest & src).astype(bool)
np.copyto(dest, src & dest, where=both_data_mask)
**Output**
If the `stack` argument is supplied, the returned data is stacked in a single ``DataArray``.
A new dimension is created with the name supplied.
This requires all of the data to be of the same datatype.
To reproject or resample the data, supply the ``output_crs``, ``resolution``, ``resampling`` and ``align``
fields.
To reproject data to 25m resolution for EPSG:3577::
dc.load(product='ls5_nbar_albers', x=(148.15, 148.2), y=(-35.15, -35.2), time=('1990', '1991'),
output_crs='EPSG:3577`, resolution=(-25, 25), resampling='cubic')
:param str product: the product to be included.
:param measurements:
measurements name or list of names to be included, as listed in :meth:`list_measurements`.
If a list is specified, the measurements will be returned in the order requested.
By default all available measurements are included.
:type measurements: list(str), optional
:param query:
Search parameters for products and dimension ranges as described above.
:param str output_crs:
The CRS of the returned data. If no CRS is supplied, the CRS of the stored data is used.
:param (float,float) resolution:
A tuple of the spatial resolution of the returned data.
This includes the direction (as indicated by a positive or negative number).
Typically when using most CRSs, the first number would be negative.
:param str resampling:
The resampling method to use if re-projection is required.
Valid values are: ``'nearest', 'cubic', 'bilinear', 'cubic_spline', 'lanczos', 'average'``
Defaults to ``'nearest'``.
:param (float,float) align:
Load data such that point 'align' lies on the pixel boundary.
Units are in the co-ordinate space of the output CRS.
Default is (0,0)
:param stack: The name of the new dimension used to stack the measurements.
If provided, the data is returned as a :class:`xarray.DataArray` rather than a :class:`xarray.Dataset`.
If only one measurement is returned, the dimension name is not used and the dimension is dropped.
:type stack: str or bool
:param dict dask_chunks:
If the data should be lazily loaded using :class:`dask.array.Array`,
specify the chunking size in each output dimension.
See the documentation on using `xarray with dask <http://xarray.pydata.org/en/stable/dask.html>`_
for more information.
:param xarray.Dataset like:
Uses the output of a previous ``load()`` to form the basis of a request for another product.
E.g.::
pq = dc.load(product='ls5_pq_albers', like=nbar_dataset)
:param str group_by:
When specified, perform basic combining/reducing of the data.
:param fuse_func:
Function used to fuse/combine/reduce data with the ``group_by`` parameter. By default,
data is simply copied over the top of each other, in a relatively undefined manner. This function can
perform a specific combining step, eg. for combining GA PQ data.
:param datasets:
Optional. If this is a non-empty list of :class:`datacube.model.Dataset` objects, these will be loaded
instead of performing a database lookup.
:return: Requested data in a :class:`xarray.Dataset`.
As a :class:`xarray.DataArray` if the ``stack`` variable is supplied.
:rtype: :class:`xarray.Dataset` or :class:`xarray.DataArray`
"""
observations = datasets or self.find_datasets(
product=product, like=like, **query)
if not observations:
return None if stack else xarray.Dataset()
if like:
assert output_crs is None, "'like' and 'output_crs' are not supported together"
assert resolution is None, "'like' and 'resolution' are not supported together"
assert align is None, "'like' and 'align' are not supported together"
geobox = like.geobox
else:
if output_crs:
if not resolution:
raise RuntimeError(
"Must specify 'resolution' when specifying 'output_crs'"
)
crs = CRS(output_crs)
else:
grid_spec = self.index.products.get_by_name(product).grid_spec
if not grid_spec or not grid_spec.crs:
raise RuntimeError(
"Product has no default CRS. Must specify 'output_crs' and 'resolution'"
)
crs = grid_spec.crs
if not resolution:
if not grid_spec.resolution:
raise RuntimeError(
"Product has no default resolution. Must specify 'resolution'"
)
resolution = grid_spec.resolution
align = align or grid_spec.alignment
geobox = GeoBox.from_geopolygon(
query_geopolygon(**query) or get_bounds(observations, crs),
resolution, crs, align)
group_by = query_group_by(**query)
grouped = self.group_datasets(observations, group_by)
measurements = self.index.products.get_by_name(
product).lookup_measurements(measurements)
measurements = set_resampling_method(measurements, resampling)
result = self.load_data(grouped,
geobox,
measurements.values(),
fuse_func=fuse_func,
dask_chunks=dask_chunks)
if not stack:
return result
else:
if not isinstance(stack, string_types):
stack = 'measurement'
return result.to_array(dim=stack)
def test_gridworkflow():
""" Test GridWorkflow with padding option. """
from mock import MagicMock
import datetime
# ----- fake a datacube -----
# e.g. let there be a dataset that coincides with a grid cell
fakecrs = CRS('EPSG:4326')
grid = 100 # spatial frequency in crs units
pixel = 10 # square pixel linear dimension in crs units
# if cell(0,0) has lower left corner at grid origin,
# and cell indices increase toward upper right,
# then this will be cell(1,-2).
gridspec = GridSpec(crs=fakecrs,
tile_size=(grid, grid),
resolution=(-pixel, pixel)) # e.g. product gridspec
fakedataset = MagicMock()
fakedataset.extent = geometry.box(left=grid,
bottom=-grid,
right=2 * grid,
top=-2 * grid,
crs=fakecrs)
fakedataset.center_time = t = datetime.datetime(2001, 2, 15)
fakeindex = MagicMock()
fakeindex.datasets.get_field_names.return_value = [
'time'
] # permit query on time
fakeindex.datasets.search_eager.return_value = [fakedataset]
# ------ test without padding ----
from datacube.api.grid_workflow import GridWorkflow
gw = GridWorkflow(fakeindex, gridspec)
query = dict(product='fake_product_name',
time=('2001-1-1 00:00:00', '2001-3-31 23:59:59'))
# test backend : that it finds the expected cell/dataset
assert list(gw.cell_observations(**query).keys()) == [(1, -2)]
# test frontend
assert len(gw.list_tiles(**query)) == 1
# ------ introduce padding --------
assert len(gw.list_tiles(tile_buffer=(20, 20), **query)) == 9
# ------ add another dataset (to test grouping) -----
# consider cell (2,-2)
fakedataset2 = MagicMock()
fakedataset2.extent = geometry.box(left=2 * grid,
bottom=-grid,
right=3 * grid,
top=-2 * grid,
crs=fakecrs)
fakedataset2.center_time = t
def search_eager(lat=None, lon=None, **kwargs):
return [fakedataset, fakedataset2]
fakeindex.datasets.search_eager = search_eager
# unpadded
assert len(gw.list_tiles(**query)) == 2
ti = numpy.datetime64(t, 'ns')
assert set(gw.list_tiles(**query).keys()) == {(1, -2, ti), (2, -2, ti)}
# padded
assert len(gw.list_tiles(tile_buffer=(20, 20), **
query)) == 12 # not 18=2*9 because of grouping
# -------- inspect particular returned tile objects --------
# check the array shape
tile = gw.list_tiles(**query)[1, -2, ti] # unpadded example
assert grid / pixel == 10
assert tile.shape == (1, 10, 10)
padded_tile = gw.list_tiles(tile_buffer=(20, 20),
**query)[1, -2, ti] # padded example
# assert grid/pixel + 2*gw2.grid_spec.padding == 14 # GREG: understand this
assert padded_tile.shape == (1, 14, 14)
# count the sources
assert len(tile.sources.isel(time=0).item()) == 1
assert len(padded_tile.sources.isel(time=0).item()) == 2
# check the geocoding
assert tile.geobox.alignment == padded_tile.geobox.alignment
assert tile.geobox.affine * (0, 0) == padded_tile.geobox.affine * (2, 2)
assert tile.geobox.affine * (10, 10) == padded_tile.geobox.affine * (
10 + 2, 10 + 2)
# ------- check loading --------
# GridWorkflow accesses the load_data API
# to ultimately convert geobox,sources,measurements to xarray,
# so only thing to check here is the call interface.
measurement = dict(nodata=0, dtype=numpy.int)
fakedataset.type.lookup_measurements.return_value = {'dummy': measurement}
fakedataset2.type = fakedataset.type
from mock import patch
with patch('datacube.api.core.Datacube.load_data') as loader:
data = GridWorkflow.load(tile)
data2 = GridWorkflow.load(padded_tile)
# Note, could also test Datacube.load for consistency (but may require more patching)
assert data is data2 is loader.return_value
assert loader.call_count == 2
# Note, use of positional arguments here is not robust, could spec mock etc.
for (args, kwargs), loadable in zip(loader.call_args_list,
[tile, padded_tile]):
args = list(args)
assert args[0] is loadable.sources
assert args[1] is loadable.geobox
assert list(args[2])[0] is measurement
# ------- check single cell index extract -------
tile = gw.list_tiles(cell_index=(1, -2), **query)
assert len(tile) == 1
assert tile[1, -2, ti].shape == (1, 10, 10)
assert len(tile[1, -2, ti].sources.values[0]) == 1
padded_tile = gw.list_tiles(cell_index=(1, -2),
tile_buffer=(20, 20),
**query)
assert len(padded_tile) == 1
assert padded_tile[1, -2, ti].shape == (1, 14, 14)
assert len(padded_tile[1, -2, ti].sources.values[0]) == 2
from __future__ import print_function, absolute_import
from textwrap import dedent
import netCDF4
import numpy
from osgeo import osr
from datacube.model import Variable, CRS
from datacube.storage.netcdf_writer import create_netcdf, create_coordinate, create_variable, netcdfy_data, \
create_grid_mapping_variable, flag_mask_meanings
GEO_PROJ = CRS(
'GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],'
'AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],'
'AUTHORITY["EPSG","4326"]]')
ALBERS_PROJ = CRS("""PROJCS["GDA94 / Australian Albers",
GEOGCS["GDA94",
DATUM["Geocentric_Datum_of_Australia_1994",
SPHEROID["GRS 1980",6378137,298.257222101,
AUTHORITY["EPSG","7019"]],
TOWGS84[0,0,0,0,0,0,0],
AUTHORITY["EPSG","6283"]],
PRIMEM["Greenwich",0,
AUTHORITY["EPSG","8901"]],
UNIT["degree",0.01745329251994328,
AUTHORITY["EPSG","9122"]],
AUTHORITY["EPSG","4283"]],
UNIT["metre",1,
AUTHORITY["EPSG","9001"]],
import fiona
import shapely.ops
from shapely.geometry import shape, mapping
from datacube.utils.geometry import Geometry
from datacube.model import CRS
import sys
from datacube import Datacube
product_name = sys.argv[1]
shapefile = sys.argv[2]
with fiona.open(shapefile) as input_region:
joined = shapely.ops.unary_union(
list(shape(geom['geometry']) for geom in input_region))
final = joined.convex_hull
crs = CRS(input_region.crs_wkt)
boundary_polygon = Geometry(mapping(final), crs)
dc = Datacube()
product = dc.index.products.get_by_name(product_name)
query_tiles = set(tile_index for tile_index, tile_geobox in
product.grid_spec.tiles_inside_geopolygon(boundary_polygon))
print(query_tiles)
# for tile_index, _ in product.grid_spec.tiles_inside_geopolygon(boundary_polygon):
# print(tile_index)