def to_lat_long_extent(left,
bottom,
right,
top,
spatial_reference,
new_crs="EPSG:4326"):
crs = CRS(spatial_reference)
abox = box(left, bottom, right, top, crs)
projected = abox.to_crs(CRS(new_crs))
proj = projected.boundingbox
left, bottom, right, top = proj.left, proj.bottom, proj.right, proj.top
coord = {
'ul': {
'lon': left,
'lat': top
},
'ur': {
'lon': right,
'lat': top
},
'll': {
'lon': left,
'lat': bottom
},
'lr': {
'lon': right,
'lat': bottom
},
}
return coord
def __call__(self, product, time, group_by) -> Tile:
# Do for a specific poly whose boundary is known
output_crs = CRS(self.storage['crs'])
filtered_item = [
'geopolygon', 'lon', 'lat', 'longitude', 'latitude', 'x', 'y'
]
filtered_dict = {
k: v
for k, v in filter(lambda t: t[0] in filtered_item,
self.input_region.items())
}
if 'feature_id' in self.input_region:
filtered_dict['geopolygon'] = Geometry(
self.input_region['geom_feat'],
CRS(self.input_region['crs_txt']))
geopoly = filtered_dict['geopolygon']
else:
geopoly = query_geopolygon(**self.input_region)
datasets = self.dc.find_datasets(product=product,
time=time,
group_by=group_by,
**filtered_dict)
group_by = query_group_by(group_by=group_by)
sources = self.dc.group_datasets(datasets, group_by)
output_resolution = [
self.storage['resolution'][dim] for dim in output_crs.dimensions
]
geopoly = geopoly.to_crs(output_crs)
geobox = GeoBox.from_geopolygon(geopoly, resolution=output_resolution)
return Tile(sources, geobox)
def rio_crs_to_odc(crs):
from datacube.utils.geometry import CRS
if crs.is_epsg_code:
return CRS('epsg:{}'.format(crs.to_epsg()))
return CRS(crs.wkt)
def test_crs():
CRS = geometry.CRS
custom_crs = geometry.CRS("""PROJCS["unnamed",
GEOGCS["Unknown datum based upon the custom spheroid",
DATUM["Not specified (based on custom spheroid)", SPHEROID["Custom spheroid",6371007.181,0]],
PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433]],
PROJECTION["Sinusoidal"],
PARAMETER["longitude_of_center",0],
PARAMETER["false_easting",0],
PARAMETER["false_northing",0],
UNIT["Meter",1]]""")
crs = epsg3577
assert crs.geographic is False
assert crs.projected is True
assert crs.dimensions == ('y', 'x')
assert crs.epsg == 3577
assert crs.units == ('metre', 'metre')
assert isinstance(repr(crs), str)
crs = epsg4326
assert crs.geographic is True
assert crs.projected is False
assert crs.dimensions == ('latitude', 'longitude')
assert crs.epsg == 4326
crs2 = CRS(crs)
assert crs2 == crs
assert crs.proj is crs2.proj
assert epsg4326.valid_region == geometry.box(-180, -90, 180, 90, epsg4326)
assert epsg3857.valid_region.crs == epsg4326
xmin, _, xmax, _ = epsg3857.valid_region.boundingbox
assert (xmin, xmax) == (-180, 180)
assert custom_crs.valid_region is None
assert epsg3577 == epsg3577
assert epsg3577 == 'EPSG:3577'
assert (epsg3577 != epsg3577) is False
assert (epsg3577 == epsg4326) is False
assert (epsg3577 == 'EPSG:4326') is False
assert epsg3577 != epsg4326
assert epsg3577 != 'EPSG:4326'
bad_crs = [
'cupcakes',
('PROJCS["unnamed",'
'GEOGCS["WGS 84", DATUM["WGS_1984", SPHEROID["WGS 84",6378137,298.257223563, AUTHORITY["EPSG","7030"]],'
'AUTHORITY["EPSG","6326"]], PRIMEM["Greenwich",0, AUTHORITY["EPSG","8901"]],'
'UNIT["degree",0.0174532925199433, AUTHORITY["EPSG","9122"]], AUTHORITY["EPSG","4326"]]]'
)
]
for bad in bad_crs:
with pytest.raises(geometry.CRSError):
CRS(bad)
with pytest.warns(DeprecationWarning):
assert str(epsg3857) == epsg3857.crs_str
def eo3_lonlat_bbox(doc, tol=None):
epsg4326 = CRS('epsg:4326')
crs = CRS(doc['crs'])
grids = doc['grids']
geometry = doc.get('geometry')
if geometry is None:
return bbox_union(
grid2polygon(grid, crs).to_crs(epsg4326, tol).boundingbox
for grid in grids.values())
else:
return Geometry(geometry, crs).to_crs(epsg4326, tol).boundingbox
def _dc_crs(crs: Optional[RioCRS]) -> Optional[CRS]:
""" Convert RIO version of CRS to datacube
"""
if crs is None:
return None
if not crs.is_valid:
return None
if crs.is_epsg_code:
return CRS('epsg:{}'.format(crs.to_epsg()))
return CRS(crs.wkt)
def fake_data_2x2x2():
from datacube.utils.geometry import CRS
fake_data = {
'attrs': {
'crs': CRS('EPSG:3577')
},
'coords': {
'time': {
'attrs': {
'units': 'seconds since 1970-01-01 00:00:00'
},
'data': [
datetime(2001, 1, 31, 23, 59, 59),
datetime(2001, 12, 30, 23, 59, 59)
],
'dims': ('time', )
},
'x': {
'attrs': {
'units': 'metre'
},
'data': [1456789.5, 1456790.5],
'dims': ('x', )
},
'y': {
'attrs': {
'units': 'metre'
},
'data': [-4098765.5, -4098766.5],
'dims': ('y', )
}
},
'data_vars': {
'FOO': {
'attrs': {
'crs': CRS('EPSG:3577'),
'nodata': -1,
'units': 'percent'
},
'data': np.ones((2, 2, 2), dtype=np.int8),
'dims': ('time', 'y', 'x')
},
},
'dims': {
'time': 2,
'x': 2,
'y': 2
}
}
return fake_data
def geographic_extent(self):
"""
:rtype: geometry.Geometry
"""
if self.crs.geographic:
return self.extent
return self.extent.to_crs(CRS('EPSG:4326'))
def rasterfile_to_xarray(file, geobox=None, name=None, nodata=None):
"""Blit like"""
with rasterio.open(file) as src:
assert src.indexes == (1, ) # assume single band
if geobox is None:
from datacube.utils.geometry import GeoBox, CRS
# TODO: fix this heinousness
global crs
global affine
crs = src.crs
affine = src.transform
geobox = GeoBox(
width=src.width,
height=src.height,
affine=src.
affine, # .transform is a list, .affine is an object
crs=CRS(src.crs.wkt))
array = src.read(1)
else:
band = rasterio.band(
src, 1) # do not attempt to read entire extent into memory
array = np.empty((geobox.height, geobox.width), dtype=band.dtype)
rasterio.warp.reproject(source=band,
destination=array,
dst_crs=geobox.crs.crs_str,
dst_transform=geobox.affine,
dst_nodata=nodata)
return numpy_to_xarray(array, geobox, name)
def geom_from_file(filename, feature_id):
"""
The geometry of a feature
:param filename: name of shape file
:param feature_id: the id of the wanted feature
"""
import fiona
with fiona.open(filename) as input_region:
geom_list = []
geopolygon_list = []
feature_list = []
crs = CRS(input_region.crs_wkt)
for feature in input_region:
if feature_id is not None and feature_id != {}:
if feature['properties']['ID'] in feature_id:
geom = feature['geometry']
return feature['properties'], geom, input_region.crs_wkt, Geometry(geom, crs)
else:
geom = feature['geometry']
feature_list.append(feature['properties'])
geom_list.append(geom)
geopolygon_list.append(Geometry(geom, crs))
return feature_list, geom_list, input_region.crs_wkt, geopolygon_list
_LOG.info("No geometry found")
def parse_gridspec(s: str) -> GridSpec:
"""
"albers_africa_10"
"epsg:6936;10;9600"
"epsg:6936;-10x10;9600x9600"
"""
named_gs = GRIDS.get(s)
if named_gs is not None:
return named_gs
crs, res, shape = split_and_check(s, ';', 3)
try:
if 'x' in res:
res = tuple(float(v) for v in split_and_check(res, 'x', 2))
else:
res = float(res)
res = (-res, res)
if 'x' in shape:
shape = parse_range_int(shape, separator='x')
else:
shape = int(shape)
shape = (shape, shape)
except ValueError:
raise ValueError(f"Failed to parse gridspec: {s}")
tsz = tuple(abs(n * res) for n, res in zip(res, shape))
return GridSpec(crs=CRS(crs), tile_size=tsz, resolution=res, origin=(0, 0))
def __call__(self, index, product, time, group_by) -> Tile:
# Do for a specific poly whose boundary is known
output_crs = CRS(self.storage['crs'])
filtered_items = [
'geopolygon', 'lon', 'lat', 'longitude', 'latitude', 'x', 'y'
]
filtered_dict = {
k: v
for k, v in self.input_region.items() if k in filtered_items
}
if self.feature is not None:
filtered_dict['geopolygon'] = self.feature.geopolygon
geopoly = filtered_dict['geopolygon']
else:
geopoly = query_geopolygon(**self.input_region)
dc = Datacube(index=index)
datasets = dc.find_datasets(product=product,
time=time,
group_by=group_by,
**filtered_dict)
group_by = query_group_by(group_by=group_by)
sources = dc.group_datasets(datasets, group_by)
output_resolution = [
self.storage['resolution'][dim] for dim in output_crs.dimensions
]
geopoly = geopoly.to_crs(output_crs)
geobox = GeoBox.from_geopolygon(geopoly, resolution=output_resolution)
return Tile(sources, geobox)
def dataset_shape(ds: Dataset) -> Tuple[Optional[Polygon], bool]:
"""
Get a usable extent from the dataset (if possible), and return
whether the original was valid.
"""
log = _LOG.bind(dataset_id=ds.id)
try:
extent = ds.extent
except AttributeError:
# `ds.extent` throws an exception on telemetry datasets,
# as they have no grid_spatial. It probably shouldn't.
return None, False
if extent is None:
log.warn('invalid_dataset.empty_extent')
return None, False
geom = shapely.geometry.asShape(extent.to_crs(CRS(_TARGET_CRS)))
if not geom.is_valid:
log.warn('invalid_dataset.invalid_extent',
reason_text=shapely.validation.explain_validity(geom))
# A zero distance may be used to “tidy” a polygon.
clean = geom.buffer(0.0)
assert clean.geom_type == 'Polygon'
assert clean.is_valid
return clean, False
if geom.is_empty:
_LOG.warn('invalid_dataset.empty_extent_geom', dataset_id=ds.id)
return None, False
return geom, True
def web_gs(zoom, tile_size=256):
""" Construct grid spec compatible with TerriaJS requests at a given level.
Tile indexes should be the same as google maps, except that Y component is negative,
this is a limitation of GridSpec class, you can not have tile index direction be
different from axis direction, but this is what google indexing is using.
http://www.maptiler.org/google-maps-coordinates-tile-bounds-projection/
"""
from datacube.utils.geometry import CRS
from datacube.model import GridSpec
from math import pi
R = 6378137
origin = pi * R
res0 = 2 * pi * R / tile_size
res = res0 * (2**(-zoom))
tsz = 2 * pi * R * (2**(-zoom)) # res*tile_size
return GridSpec(
crs=CRS('epsg:3857'),
tile_size=(tsz, tsz),
resolution=(-res, res), # Y,X
origin=(origin - tsz, -origin)) # Y,X
def make_sample_netcdf(tmpdir):
"""Make a test Geospatial NetCDF file, 4000x4000 int16 random data, in a variable named `sample`.
Return the GDAL access string."""
sample_nc = str(tmpdir.mkdir('netcdfs').join('sample.nc'))
geobox = GeoBox(4000,
4000,
affine=Affine(25.0, 0.0, 1200000, 0.0, -25.0, -4200000),
crs=CRS('EPSG:3577'))
sample_data = np.random.randint(10000, size=(4000, 4000), dtype=np.int16)
variables = {
'sample':
Variable(sample_data.dtype,
nodata=-999,
dims=geobox.dimensions,
units=1)
}
nco = create_netcdf_storage_unit(sample_nc,
geobox.crs,
geobox.coordinates,
variables=variables,
variable_params={})
nco['sample'][:] = sample_data
nco.close()
return "NetCDF:%s:sample" % sample_nc, geobox, sample_data
def doc2gs(doc: Document) -> GridSpec:
return GridSpec(
crs=CRS(doc["crs"]),
tile_size=tuple(doc["tile_size"]),
resolution=tuple(doc["resolution"]),
origin=tuple(doc["origin"]),
)
def calculate_bounds_geotransform(dataset):
# Convert rasterio CRS into datacube CRS object
if isinstance(dataset.crs, xr.DataArray):
crs_dict = dataset.crs.to_dict()
_crs = CRS(str(crs_dict['attrs']['spatial_ref']))
# Leave the CRS as it is (datacube CRS object)
elif isinstance(dataset.crs, datacube.utils.geometry._base.CRS):
_crs = dataset.crs
else:
raise Exception(
'dataset.crs datatype not know (please check calculate_bounds_geotransform)'
)
#crs_dict = dataset.crs.to_dict()
#_crs = CRS(str(crs_dict['attrs']['spatial_ref']))
dims = _crs.dimensions
xres, xoff = data_resolution_and_offset(dataset[dims[1]])
yres, yoff = data_resolution_and_offset(dataset[dims[0]])
GeoTransform = [xoff, xres, 0.0, yoff, 0.0, yres]
left, right = dataset[dims[1]][0] - 0.5 * xres, dataset[
dims[1]][-1] + 0.5 * xres
bottom, top = dataset[dims[0]][0] - 0.5 * yres, dataset[
dims[0]][-1] + 0.5 * yres
return {
'left': left,
'right': right,
'bottom': bottom,
'top': top,
'GeoTransform': GeoTransform
}
def mk_sample_xr_dataset(crs="EPSG:3578",
shape=(33, 74),
resolution=None,
xy=(0, 0),
time='2020-02-13T11:12:13.1234567Z',
name='band',
dtype='int16',
nodata=-999,
units='1'):
""" Note that resolution is in Y,X order to match that of GeoBox.
shape (height, width)
resolution (y: float, x: float) - in YX, to match GeoBox/shape notation
xy (x: float, y: float) -- location of the top-left corner of the top-left pixel in CRS units
"""
if isinstance(crs, str):
crs = CRS(crs)
if resolution is None:
resolution = (-10, 10) if crs is None or crs.projected else (-0.01, 0.01)
t_coords = {}
if time is not None:
t_coords['time'] = mk_time_coord([time])
transform = Affine.translation(*xy)*Affine.scale(*resolution[::-1])
h, w = shape
geobox = GeoBox(w, h, transform, crs)
return Datacube.create_storage(t_coords, geobox, [Measurement(name=name, dtype=dtype, nodata=nodata, units=units)])
def eo3_lonlat_bbox(doc, tol=None):
epsg4326 = CRS('epsg:4326')
crs = doc.get('crs')
grids = doc.get('grids')
if crs is None or grids is None:
raise ValueError("Input must have crs and grids")
crs = CRS(crs)
geometry = doc.get('geometry')
if geometry is None:
return bbox_union(
grid2polygon(grid, crs).to_crs(epsg4326, tol).boundingbox
for grid in grids.values())
else:
return Geometry(geometry, crs).to_crs(epsg4326, tol).boundingbox
def _parse_gridspec_string(s: str) -> GridSpec:
"""
"epsg:6936;10;9600"
"epsg:6936;-10x10;9600x9600"
"""
crs, res, shape = split_and_check(s, ";", 3)
try:
if "x" in res:
res = tuple(float(v) for v in split_and_check(res, "x", 2))
else:
res = float(res)
res = (-res, res)
if "x" in shape:
shape = parse_range_int(shape, separator="x")
else:
shape = int(shape)
shape = (shape, shape)
except ValueError:
raise ValueError(f"Failed to parse gridspec: {s}")
tsz = tuple(abs(n * res) for n, res in zip(res, shape))
return GridSpec(crs=CRS(crs), tile_size=tsz, resolution=res, origin=(0, 0))
def grid_spec(self):
"""
Grid specification for this product
:rtype: GridSpec
"""
if 'storage' not in self.definition:
return None
storage = self.definition['storage']
if 'crs' not in storage:
return None
crs = CRS(str(storage['crs']).strip())
tile_size = None
if 'tile_size' in storage:
tile_size = [storage['tile_size'][dim] for dim in crs.dimensions]
resolution = None
if 'resolution' in storage:
resolution = [storage['resolution'][dim] for dim in crs.dimensions]
origin = None
if 'origin' in storage:
origin = [storage['origin'][dim] for dim in crs.dimensions]
return GridSpec(crs=crs,
tile_size=tile_size,
resolution=resolution,
origin=origin)
def test_wofs_filtered():
cfg = Config('../configs/template_client.yaml')
grid_spec = GridSpec(crs=CRS('EPSG:3577'),
tile_size=(100000, 100000),
resolution=(-25, 25))
cell_index = (17, -39)
wf = WofsFiltered(cfg, grid_spec, cell_index)
confidence = wf.compute_confidence(cell_index)
filtered = wf.compute_confidence_filtered()
# Display images: to be removed later
with Datacube(app='wofs_summary', env='dev') as dc:
gwf = GridWorkflow(dc.index, grid_spec)
indexed_tile = gwf.list_cells(cell_index,
product='wofs_statistical_summary')
# load the data of the tile
dataset = gwf.load(tile=indexed_tile[cell_index],
measurements=['frequency'])
frequency = dataset.data_vars['frequency'].data.ravel().reshape(
grid_spec.tile_resolution)
# Check with previous run
with rasterio.open('confidenceFilteredWOfS_17_-39_epsilon=10.tiff') as f:
data = f.read(1)
plt.subplot(221)
plt.imshow(frequency)
plt.subplot(222)
plt.imshow(data)
plt.subplot(223)
plt.imshow(confidence)
plt.subplot(224)
plt.imshow(filtered)
plt.show()
wf.compute_and_write()
def geom_from_file(filename, feature_id):
"""
The geometry of a feature
:param filename: name of shape file
:param feature_id: the id of the wanted feature
"""
import fiona
geometry_list = []
geopolygon_list = []
feature_list = []
find_feature = False
with fiona.open(filename) as input_region:
crs = CRS(input_region.crs_wkt)
for feature in input_region:
find_feature = False
properties = feature['properties']
if feature_id is None or properties.get(
'ID') in feature_id or properties.get('id') in feature_id:
feature_list.append(properties)
find_feature = True
if int(feature.get('id')) in feature_id:
feature_list.append(feature)
find_feature = True
if find_feature:
geometry = feature['geometry']
geopolygon = Geometry(geometry, crs)
geometry_list.append(geometry)
geopolygon_list.append(geopolygon)
if not geometry_list:
_LOG.info("No geometry found")
return feature_list, geometry_list, input_region.crs_wkt, geopolygon_list
def grid2polygon(grid, crs):
h, w = grid['shape']
transform = Affine(*grid['transform'][:6])
if isinstance(crs, str):
crs = CRS(crs)
return polygon_from_transform(w, h, transform, crs)
def tile_to_geojsonfeature(tile, **props):
geometry = tile.geobox.extent.to_crs(CRS('EPSG:4326')).__geo_interface__
feature = {
'type': 'Feature',
'geometry': geometry,
'properties': dict(count=len(tile.sources), **props)
}
return feature
def test_crs_compat():
import rasterio.crs
crs = CRS("epsg:3577")
assert crs.epsg == 3577
crs2 = CRS(crs)
assert crs.epsg == crs2.epsg
crs_rio = rasterio.crs.CRS(init='epsg:3577')
assert CRS(crs_rio).epsg == 3577
assert (CRS(crs_rio) == crs_rio) is True
assert rasterio.crs.CRS.from_user_input(crs).to_epsg() == 3577
with pytest.raises(ValueError):
CRS(("random", "tuple"))
def _make_grid_spec(storage) -> GridSpec:
"""Make a grid spec based on a storage spec."""
assert 'tile_size' in storage
crs = CRS(storage['crs'])
return GridSpec(
crs=crs,
tile_size=[storage['tile_size'][dim] for dim in crs.dimensions],
resolution=[storage['resolution'][dim] for dim in crs.dimensions])
def grid2polygon(grid, crs):
grid = _norm_grid(grid)
h, w = grid.shape
transform = grid.transform
if isinstance(crs, str):
crs = CRS(crs)
return polygon_from_transform(w, h, transform, crs)
def _extent_point_projector(crs):
crs = CRS(crs)
def reproject_point(pos):
pos = point(pos['lon'], pos['lat'], CRS('EPSG:4326'))
coords = pos.to_crs(crs).coords[0]
return {'x': coords[0], 'y': coords[1]}
return reproject_point
def test_vector_to_crs(orig_point, orig_vect):
"""
Given a random point (in EPSG:3577) and vector, convert to EPSG:4326
and back
Ensure you get your starting values back (approximately)
"""
inter_point, inter_vect = vector_to_crs(orig_point,
orig_vect,
original_crs=CRS('EPSG:3577'),
destination_crs=CRS('EPSG:4326'))
new_point, new_vect = vector_to_crs(inter_point,
inter_vect,
original_crs=CRS('EPSG:4326'),
destination_crs=CRS('EPSG:3577'))
assert orig_point == pytest.approx(new_point, abs=1e-6)
assert orig_vect == pytest.approx(new_vect, abs=1e-6)