def list_grs_schemas(self):
"""Retrieve a list of available Grid Schema on Brazil Data Cube database."""
schemas = GridRefSys.query().all()
return [
dict(**Serializer.serialize(schema), crs=schema.crs)
for schema in schemas
], 200
def test_create_grid(db):
fields = _prepare_grs_fields()
with db.session.begin_nested():
grs = GridRefSys.create_geometry_table(**fields,
srid=4326,
schema='public')
db.session.add(grs)
db.session.commit()
assert grs and grs.name == fields['table_name']
assert grs.geom_table is not None
assert grs.crs == '+proj=longlat +datum=WGS84 +no_defs '
with pytest.raises(RuntimeError) as e:
_ = GridRefSys.create_geometry_table(**fields,
srid=4326,
schema='public',
extend_existing=True)
assert str(e.value) == f'Table {fields["table_name"]} already exists'
def get_grs_schema(self, grs_id):
"""Retrieve a Grid Schema definition with tiles associated."""
schema = GridRefSys.query().filter(GridRefSys.id == grs_id).first()
if schema is None:
return 'GRS {} not found.'.format(grs_id), 404
geom_table = schema.geom_table
tiles = db.session.query(
geom_table.c.tile,
func.ST_AsGeoJSON(func.ST_Transform(geom_table.c.geom, 4326), 6, 3).cast(sqlalchemy.JSON).label('geom_wgs84')
).all()
dump_grs = Serializer.serialize(schema)
dump_grs['tiles'] = [dict(id=t.tile, geom_wgs84=t.geom_wgs84) for t in tiles]
return dump_grs, 200
def create_grs_schema(cls,
name,
description,
projection,
meridian,
degreesx,
degreesy,
bbox,
srid=100001):
"""Create a Brazil Data Cube Grid Schema."""
grid_mapping, proj4 = create_grids(names=[name],
projection=projection,
meridian=meridian,
degreesx=[degreesx],
degreesy=[degreesy],
bbox=bbox,
srid=srid)
grid = grid_mapping[name]
with db.session.begin_nested():
crs = CRS.from_proj4(proj4)
data = dict(auth_name='Albers Equal Area',
auth_srid=srid,
srid=srid,
srtext=crs.to_wkt(),
proj4text=proj4)
spatial_index, _ = get_or_create_model(SpatialRefSys,
defaults=data,
srid=srid)
grs = GridRefSys.create_geometry_table(table_name=name,
features=grid['features'],
srid=srid)
grs.description = description
db.session.add(grs)
for tile_obj in grid['tiles']:
tile = Tile(**tile_obj, grs=grs)
db.session.add(tile)
db.session.commit()
return 'Grid {} created with successfully'.format(name), 201
def get_grs_schema(cls,
grs_id,
bbox: Tuple[float, float, float, float] = None,
tiles=None):
"""Retrieve a Grid Schema definition with tiles associated."""
schema: GridRefSys = GridRefSys.query().filter(
GridRefSys.id == grs_id).first()
if schema is None:
return 'GRS {} not found.'.format(grs_id), 404
geom_table = schema.geom_table
srid_column = get_srid_column(geom_table.c, default_srid=4326)
where = []
if bbox is not None:
x_min, y_min, x_max, y_max = bbox
where.append(
func.ST_Intersects(
func.ST_MakeEnvelope(x_min, y_min, x_max, y_max, 4326),
func.ST_Transform(
func.ST_SetSRID(geom_table.c.geom, srid_column),
4326)))
if tiles:
where.append(geom_table.c.tile.in_(tiles))
tiles = db.session.query(
geom_table.c.tile,
func.ST_AsGeoJSON(
func.ST_Transform(
func.ST_SetSRID(geom_table.c.geom, srid_column), 4326), 6,
3).cast(
sqlalchemy.JSON).label('geom_wgs84')).filter(*where).all()
dump_grs = Serializer.serialize(schema)
dump_grs['tiles'] = [
dict(id=t.tile, geom_wgs84=t.geom_wgs84) for t in tiles
]
return dump_grs, 200
def create_grs_schema(cls,
name,
description,
projection,
meridian,
degreesx,
degreesy,
bbox,
srid=100001):
"""Create a Brazil Data Cube Grid Schema."""
bbox = bbox.split(',')
bbox_obj = {
"w": float(bbox[0]),
"n": float(bbox[1]),
"e": float(bbox[2]),
"s": float(bbox[3])
}
tile_srs_p4 = "+proj=longlat +ellps=GRS80 +datum=GRS80 +no_defs"
if projection == 'aea':
tile_srs_p4 = "+proj=aea +lat_0=-12 +lon_0={} +lat_1=-2 +lat_2=-22 +x_0=5000000 +y_0=10000000 +ellps=GRS80 +units=m +no_defs".format(
meridian)
elif projection == 'sinu':
tile_srs_p4 = "+proj=sinu +lon_0={} +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs".format(
meridian)
# Number of tiles and base tile
num_tiles_x = int(360. / degreesx)
num_tiles_y = int(180. / degreesy)
h_base = num_tiles_x / 2
v_base = num_tiles_y / 2
# Tile size in meters (dx,dy) at center of system (argsmeridian,0.)
src_crs = '+proj=longlat +ellps=GRS80 +datum=GRS80 +no_defs'
dst_crs = tile_srs_p4
xs = [(meridian - degreesx / 2), (meridian + degreesx / 2), meridian,
meridian, 0.]
ys = [0., 0., -degreesy / 2, degreesy / 2, 0.]
out = transform(src_crs, dst_crs, xs, ys, zs=None)
x1 = out[0][0]
x2 = out[0][1]
y1 = out[1][2]
y2 = out[1][3]
dx = x2 - x1
dy = y2 - y1
# Coordinates of WRS center (0.,0.) - top left coordinate of (h_base,v_base)
x_center = out[0][4]
y_center = out[1][4]
# Border coordinates of WRS grid
x_min = x_center - dx * h_base
y_max = y_center + dy * v_base
# Upper Left is (xl,yu) Bottom Right is (xr,yb)
xs = [bbox_obj['w'], bbox_obj['e'], meridian, meridian]
ys = [0., 0., bbox_obj['n'], bbox_obj['s']]
out = transform(src_crs, dst_crs, xs, ys, zs=None)
xl = out[0][0]
xr = out[0][1]
yu = out[1][2]
yb = out[1][3]
h_min = int((xl - x_min) / dx)
h_max = int((xr - x_min) / dx)
v_min = int((y_max - yu) / dy)
v_max = int((y_max - yb) / dy)
tiles = []
features = []
dst_crs = '+proj=longlat +ellps=GRS80 +datum=GRS80 +no_defs'
src_crs = tile_srs_p4
for ix in range(h_min, h_max + 1):
x1 = x_min + ix * dx
x2 = x1 + dx
for iy in range(v_min, v_max + 1):
y1 = y_max - iy * dy
y2 = y1 - dy
# Evaluate the bounding box of tile in longlat
xs = [x1, x2, x2, x1]
ys = [y1, y1, y2, y2]
out = rasterio.warp.transform(src_crs,
dst_crs,
xs,
ys,
zs=None)
polygon = from_shape(Polygon([(x1, y2), (x2, y2), (x2, y1),
(x1, y1), (x1, y2)]),
srid=SRID_ALBERS_EQUAL_AREA)
# Insert tile
tile_name = '{0:03d}{1:03d}'.format(ix, iy)
tiles.append(dict(name=tile_name))
features.append(dict(tile=tile_name, geom=polygon))
with db.session.begin_nested():
crs = CRS.from_proj4(tile_srs_p4)
data = dict(auth_name='Albers Equal Area',
auth_srid=srid,
srid=srid,
srtext=crs.to_wkt(),
proj4text=tile_srs_p4)
spatial_index, _ = get_or_create_model(SpatialRefSys,
defaults=data,
srid=srid)
grs = GridRefSys.create_geometry_table(table_name=name,
features=features,
srid=SRID_ALBERS_EQUAL_AREA)
grs.description = description
db.session.add(grs)
[db.session.add(Tile(**tile, grs=grs)) for tile in tiles]
db.session.commit()
return 'Grid {} created with successfully'.format(name), 201
def _create_cube_definition(cls, cube_id: str, params: dict) -> dict:
"""Create a data cube definition.
Basically, the definition consists in `Collection` and `Band` attributes.
Note:
It does not try to create when data cube already exists.
Args:
cube_id - Data cube
params - Dict of required values to create data cube. See @validators.py
Returns:
A serialized data cube information.
"""
cube_parts = get_cube_parts(cube_id)
function = cube_parts.composite_function
cube_id = cube_parts.datacube
cube = Collection.query().filter(
Collection.name == cube_id,
Collection.version == params['version']).first()
grs = GridRefSys.query().filter(
GridRefSys.name == params['grs']).first()
if grs is None:
abort(404, f'Grid {params["grs"]} not found.')
cube_function = CompositeFunction.query().filter(
CompositeFunction.alias == function).first()
if cube_function is None:
abort(404, f'Function {function} not found.')
data = dict(name='Meter', symbol='m')
resolution_meter, _ = get_or_create_model(ResolutionUnit,
defaults=data,
symbol='m')
mime_type, _ = get_or_create_model(MimeType,
defaults=dict(name=COG_MIME_TYPE),
name=COG_MIME_TYPE)
if cube is None:
cube = Collection(
name=cube_id,
title=params['title'],
temporal_composition_schema=params['temporal_composition']
if function != 'IDT' else None,
composite_function_id=cube_function.id,
grs=grs,
_metadata=params['metadata'],
description=params['description'],
collection_type='cube',
is_public=params.get('public', True),
version=params['version'])
cube.save(commit=False)
bands = []
default_bands = (CLEAR_OBSERVATION_NAME.lower(),
TOTAL_OBSERVATION_NAME.lower(),
PROVENANCE_NAME.lower())
band_map = dict()
for band in params['bands']:
name = band['name'].strip()
if name in default_bands:
continue
is_not_cloud = params['quality_band'] != band['name']
if band['name'] == params['quality_band']:
data_type = 'uint8'
else:
data_type = band['data_type']
band_model = Band(name=name,
common_name=band['common_name'],
collection=cube,
min_value=0,
max_value=10000 if is_not_cloud else 4,
nodata=-9999 if is_not_cloud else 255,
scale=0.0001 if is_not_cloud else 1,
data_type=data_type,
resolution_x=params['resolution'],
resolution_y=params['resolution'],
resolution_unit_id=resolution_meter.id,
description='',
mime_type_id=mime_type.id)
if band.get('metadata'):
band_model._metadata = cls._validate_band_metadata(
deepcopy(band['metadata']), band_map)
band_model.save(commit=False)
bands.append(band_model)
band_map[name] = band_model
if band_model._metadata:
for _band_origin_id in band_model._metadata['expression'][
'bands']:
band_provenance = BandSRC(band_src_id=_band_origin_id,
band_id=band_model.id)
band_provenance.save(commit=False)
quicklook = Quicklook(
red=band_map[params['bands_quicklook'][0]].id,
green=band_map[params['bands_quicklook'][1]].id,
blue=band_map[params['bands_quicklook'][2]].id,
collection=cube)
quicklook.save(commit=False)
# Create default Cube Bands
if function != 'IDT':
_ = cls.get_or_create_band(cube.id,
**CLEAR_OBSERVATION_ATTRIBUTES,
resolution_unit_id=resolution_meter.id,
resolution_x=params['resolution'],
resolution_y=params['resolution'])
_ = cls.get_or_create_band(cube.id,
**TOTAL_OBSERVATION_ATTRIBUTES,
resolution_unit_id=resolution_meter.id,
resolution_x=params['resolution'],
resolution_y=params['resolution'])
if function == 'STK':
_ = cls.get_or_create_band(
cube.id,
**PROVENANCE_ATTRIBUTES,
resolution_unit_id=resolution_meter.id,
resolution_x=params['resolution'],
resolution_y=params['resolution'])
if params.get('is_combined') and function != 'MED':
_ = cls.get_or_create_band(cube.id,
**DATASOURCE_ATTRIBUTES,
resolution_unit_id=resolution_meter.id,
resolution_x=params['resolution'],
resolution_y=params['resolution'])
return CollectionForm().dump(cube)
def get_grid_by_name(grid_name: str):
"""Try to get a grid, otherwise raises 404."""
grid = GridRefSys.query().filter(GridRefSys.name == grid_name).first()
if not grid:
raise NotFound(f'Grid {grid_name} not found.')
return grid
def get_collection_items(
collection_id=None,
roles=None,
item_id=None,
bbox=None,
datetime=None,
ids=None,
collections=None,
intersects=None,
page=1,
limit=10,
query=None,
**kwargs,
) -> Pagination:
"""Retrieve a list of collection items based on filters.
:param collection_id: Single Collection ID to include in the search for items.
Only Items in one of the provided Collection will be searched, defaults to None
:type collection_id: str, optional
:param item_id: item identifier, defaults to None
:type item_id: str, optional
:param bbox: bounding box for intersection [west, north, east, south], defaults to None
:type bbox: list, optional
:param datetime: Single date+time, or a range ('/' seperator), formatted to RFC 3339, section 5.6.
Use double dots '..' for open date ranges, defaults to None. If the start or end date of an image
generated by a temporal composition intersects the given datetime or range it will be included in the
result.
:type datetime: str, optional
:param ids: Array of Item ids to return. All other filter parameters that further restrict the
number of search results are ignored, defaults to None
:type ids: list, optional
:param collections: Array of Collection IDs to include in the search for items.
Only Items in one of the provided Collections will be searched, defaults to None
:type collections: list, optional
:param intersects: Searches items by performing intersection between their geometry and provided GeoJSON geometry.
All GeoJSON geometry types must be supported., defaults to None
:type intersects: dict, optional
:param page: The page offset of results, defaults to 1
:type page: int, optional
:param limit: The maximum number of results to return (page size), defaults to 10
:type limit: int, optional
:return: list of collectio items
:rtype: list
"""
columns = [
func.concat(Collection.name, "-",
Collection.version).label("collection"),
Collection.collection_type,
Collection._metadata.label("meta"),
Item._metadata.label("item_meta"),
Item.name.label("item"),
Item.id,
Item.collection_id,
Item.start_date.label("start"),
Item.end_date.label("end"),
Item.assets,
Item.created,
Item.updated,
cast(Item.cloud_cover, Float).label("cloud_cover"),
func.ST_AsGeoJSON(Item.geom).label("geom"),
func.ST_XMin(Item.geom).label("xmin"),
func.ST_XMax(Item.geom).label("xmax"),
func.ST_YMin(Item.geom).label("ymin"),
func.ST_YMax(Item.geom).label("ymax"),
Tile.name.label("tile"),
]
if roles is None:
roles = []
where = [
Collection.id == Item.collection_id,
or_(Collection.is_public.is_(True),
Collection.id.in_([int(r.split(":")[0]) for r in roles])),
]
collections_where = _where_collections(collection_id, collections)
collections_where.append(
or_(Collection.is_public.is_(True),
Collection.id.in_([int(r.split(":")[0]) for r in roles])))
outer_join = [(Tile, [Item.tile_id == Tile.id])]
_geom_tables = []
_collections = Collection.query().filter(*collections_where).all()
if bbox or intersects:
grids = GridRefSys.query().filter(
GridRefSys.id.in_([c.grid_ref_sys_id
for c in _collections])).all()
for grid in grids:
geom_table = grid.geom_table
if geom_table is None:
continue
_geom_tables.append(geom_table)
if ids is not None:
if isinstance(ids, str):
ids = ids.split(",")
where += [Item.name.in_(ids)]
else:
where += _where_collections(collection_id, collections)
if item_id is not None:
where += [Item.name.like(item_id)]
if query:
filters = create_query_filter(query)
if filters:
where += filters
if intersects is not None:
# Intersect with native grid if there is
geom_expr = func.ST_GeomFromGeoJSON(str(intersects))
grids_where, joins = intersect_grids(geom_expr,
geom_tables=_geom_tables)
where += grids_where
outer_join += joins
elif bbox is not None:
try:
if isinstance(bbox, str):
bbox = bbox.split(",")
bbox = [float(x) for x in bbox]
if bbox[0] == bbox[2] or bbox[1] == bbox[3]:
raise InvalidBoundingBoxError("")
geom_expr = func.ST_MakeEnvelope(bbox[0], bbox[1], bbox[2],
bbox[3],
func.ST_SRID(Item.geom))
grid_where, joins = intersect_grids(geom_expr,
geom_tables=_geom_tables)
where += grid_where
outer_join += joins
except (ValueError, InvalidBoundingBoxError) as e:
abort(400, f"'{bbox}' is not a valid bbox.")
if datetime is not None:
if "/" in datetime:
matches_open = ("..", "")
time_start, time_end = datetime.split("/")
if time_start in matches_open: # open start
date_filter = [
or_(Item.start_date <= time_end,
Item.end_date <= time_end)
]
elif time_end in matches_open: # open end
date_filter = [
or_(Item.start_date >= time_start,
Item.end_date >= time_start)
]
else: # closed range
date_filter = [
or_(
and_(Item.start_date >= time_start,
Item.start_date <= time_end),
and_(Item.end_date >= time_start,
Item.end_date <= time_end),
and_(Item.start_date < time_start,
Item.end_date > time_end),
)
]
else:
date_filter = [
and_(Item.start_date <= datetime,
Item.end_date >= datetime)
]
where += date_filter
query = session.query(*columns)
for entity, join_conditions in outer_join:
query = query.outerjoin(entity, *join_conditions)
try:
query = query.filter(*where).order_by(Item.start_date.desc(), Item.id)
result = query.paginate(page=int(page),
per_page=int(limit),
error_out=False,
max_per_page=BDC_STAC_MAX_LIMIT)
return result
except Exception as err:
msg = str(err)
if hasattr(err, "orig"):
msg = str(err.orig)
abort(400, msg.rstrip())
def check_scenes(cls, collections: str, start_date: datetime, end_date: datetime,
catalog: str = None, dataset: str = None,
grid: str = None, tiles: list = None, bbox: list = None, catalog_kwargs=None, only_tiles=False):
"""Check for the scenes in remote provider and compares with the Collection Builder."""
bbox_list = []
if grid and tiles:
grid = GridRefSys.query().filter(GridRefSys.name == grid).first_or_404(f'Grid "{grid}" not found.')
geom_table = grid.geom_table
rows = db.session.query(
geom_table.c.tile,
func.ST_Xmin(func.ST_Transform(geom_table.c.geom, 4326)).label('xmin'),
func.ST_Ymin(func.ST_Transform(geom_table.c.geom, 4326)).label('ymin'),
func.ST_Xmax(func.ST_Transform(geom_table.c.geom, 4326)).label('xmax'),
func.ST_Ymax(func.ST_Transform(geom_table.c.geom, 4326)).label('ymax'),
).filter(geom_table.c.tile.in_(tiles)).all()
for row in rows:
bbox_list.append((row.tile, (row.xmin, row.ymin, row.xmax, row.ymax)))
else:
bbox_list.append(('', bbox))
instance, provider = get_provider(catalog)
collection_map = dict()
collection_ids = list()
for _collection in collections:
collection, version = _collection.split('-')
collection = Collection.query().filter(
Collection.name == collection,
Collection.version == version
).first_or_404(f'Collection "{collection}-{version}" not found.')
collection_ids.append(collection.id)
collection_map[_collection] = collection
options = dict(start_date=start_date, end_date=end_date)
if catalog_kwargs:
options.update(catalog_kwargs)
redis = current_app.redis
output = dict(
collections={cname: dict(total_scenes=0, total_missing=0, missing_external=[]) for cname in collections}
)
items = {cid: set() for cid in collection_ids}
external_scenes = set()
for tile, _bbox in bbox_list:
with redis.pipeline() as pipe:
if only_tiles:
entry = tile
options['tile'] = tile
else:
options['bbox'] = _bbox
entry = _bbox
periods = _generate_periods(start_date.replace(tzinfo=None), end_date.replace(tzinfo=None))
for period_start, period_end in periods:
_items = db.session.query(Item.name, Item.collection_id).filter(
Item.collection_id.in_(collection_ids),
func.ST_Intersects(
func.ST_MakeEnvelope(
*_bbox, func.ST_SRID(Item.geom)
),
Item.geom
),
or_(
and_(Item.start_date >= period_start, Item.start_date <= period_end),
and_(Item.end_date >= period_start, Item.end_date <= period_end),
and_(Item.start_date < period_start, Item.end_date > period_end),
)
).order_by(Item.name).all()
for item in _items:
items[item.collection_id].add(item.name)
options['start_date'] = period_start.strftime('%Y-%m-%d')
options['end_date'] = period_end.strftime('%Y-%m-%d')
key = f'scenes:{catalog}:{dataset}:{period_start.strftime("%Y%m%d")}_{period_end.strftime("%Y%m%d")}_{entry}'
pipe.get(key)
provider_scenes = []
if not redis.exists(key):
provider_scenes = provider.search(dataset, **options)
provider_scenes = [s.scene_id for s in provider_scenes]
pipe.set(key, json.dumps(provider_scenes))
external_scenes = external_scenes.union(set(provider_scenes))
cached_scenes = pipe.execute()
for cache in cached_scenes:
# When cache is True, represents set the value were cached.
if cache is not None and cache is not True:
external_scenes = external_scenes.union(set(json.loads(cache)))
output['total_external'] = len(external_scenes)
for _collection_name, _collection in collection_map.items():
_items = set(items[_collection.id])
diff = list(external_scenes.difference(_items))
output['collections'][_collection_name]['total_scenes'] = len(_items)
output['collections'][_collection_name]['total_missing'] = len(diff)
output['collections'][_collection_name]['missing_external'] = diff
for cname, _internal_collection in collection_map.items():
if cname != _collection_name:
diff = list(_items.difference(set(items[_internal_collection.id])))
output['collections'][_collection_name][f'total_missing_{cname}'] = len(diff)
output['collections'][_collection_name][f'missing_{cname}'] = diff
return output
