def apply(self, query):
return query.filter(
models.School.location.intersects(
func.ST_MakeEnvelope(self.values['left'],
self.values['bottom'],
self.values['right'],
self.values['top'])))
def suggest_larger_areas(self):
ret = []
for e in reversed(self.is_in() or []):
osm_type, osm_id, bounds = e['type'], e['id'], e['bounds']
if osm_type == self.osm_type and osm_id == self.osm_id:
continue
box = func.ST_MakeEnvelope(bounds['minlon'], bounds['minlat'],
bounds['maxlon'], bounds['maxlat'],
4326)
q = func.ST_Area(box.cast(Geography))
bbox_area = session.query(q).scalar()
area_in_sq_km = bbox_area / (1000 * 1000)
if area_in_sq_km < 10 or area_in_sq_km > 40_000:
continue
place = Place.from_osm(osm_type, osm_id)
if not place:
continue
place.admin_level = e['tags'].get(
'admin_level') or None if 'tags' in e else None
ret.append(place)
ret.sort(key=lambda place: place.area_in_sq_km)
return ret
def chunk(self):
chunk_size = utils.calc_chunk_size(self.area_in_sq_km)
chunks = self.chunk_n(chunk_size)
print('chunk size:', chunk_size)
files = []
need_pause = False
for chunk_num, (ymin, ymax, xmin, xmax) in enumerate(chunks):
q = self.items
# note: different order for coordinates, xmin first, not ymin
q = q.filter(
cast(Item.location, Geometry).contained(
func.ST_MakeEnvelope(xmin, ymin, xmax, ymax)))
tags = set()
for item in q:
tags |= set(item.tags)
tags.difference_update(skip_tags)
tags = matcher.simplify_tags(tags)
filename = '{}_{:03d}_{:03d}.xml'.format(self.place_id, chunk_num,
len(chunks))
print(chunk_num, q.count(), len(tags), filename, list(tags))
full = os.path.join('overpass', filename)
if not (tags):
print('no tags, skipping')
continue
if not (tags):
continue
files.append(full)
if os.path.exists(full):
continue
oql_bbox = '{:f},{:f},{:f},{:f}'.format(ymin, xmin, ymax, xmax)
if need_pause:
seconds = 2
print('waiting {:d} seconds'.format(seconds))
time.sleep(seconds)
oql = overpass.oql_for_area(self.overpass_type,
self.osm_id,
tags,
oql_bbox,
None,
include_self=(chunk_num == 0))
r = overpass.run_query_persistent(oql)
if not r:
print(oql)
assert r
open(full, 'wb').write(r.content)
need_pause = True
cmd = ['osmium', 'merge'] + files + ['-o', self.overpass_filename]
print(' '.join(cmd))
subprocess.run(cmd)
def get(self):
parser = reqparse.RequestParser()
parser.add_argument('request_fields', action='append')
#xmin
parser.add_argument('sw_longitude', type=float)
#ymin
parser.add_argument('sw_latitude', type=float)
#xmax
parser.add_argument('ne_longitude', type=float)
#ymax
parser.add_argument('ne_latitude', type=float)
parser.add_argument('keyword', type=str)
parser.add_argument('current_user_id', type=int)
args = parser.parse_args()
pin_query = Pin.query
sw_longitude = args['sw_longitude']
sw_latitude = args['sw_latitude']
ne_longitude = args['ne_longitude']
ne_latitude = args['ne_latitude']
pin_query = pin_query.filter(func.ST_Contains(func.ST_MakeEnvelope(sw_longitude, sw_latitude, ne_longitude, ne_latitude, 4326), Pin.geo))
if(args['request_fields']):
request_fields = tuple(args['request_fields'])
pin_schema = PinSchema(only=request_fields)
else:
pin_schema = PinSchema()
if(args['keyword']):
keyword = args['keyword']
combined_search_vector = ( Pin.search_vector | User.search_vector | func.coalesce(Tag.search_vector, u'') )
pin_query = pin_query.join(User).outerjoin(PinTag).outerjoin(Tag).filter(combined_search_vector.match(parse_search_query(keyword)))
pins = pin_query.all()
if(args['current_user_id']):
for pin in pins:
if(pin.votes):
for vote in pin.votes:
if(vote.user_id == args['current_user_id']):
pin.vote_by_current_user = vote
if(not pins):
return {"message" :"Pin not found"}, HTTP_NOT_FOUND
try:
pin_json = pin_schema.dump(pins, many=True).data
return pin_json
except AttributeError as err:
return {"message" : {"request_fields" : format(err)} }, HTTP_BAD_REQUEST
def get_places_in_rect(self, rect, sort_close_to=None):
query = self.session.query(TestPlace)
query = query.filter(
func.ST_Contains(
func.ST_MakeEnvelope(rect[0], rect[1], rect[2], rect[3], 4326),
cast(TestPlace.coord, Geometry)))
if sort_close_to is not None:
query = query.order_by(
func.ST_DistanceSphere(
func.ST_GeomFromText('POINT({} {} 4326)'.format(
sort_close_to[0], sort_close_to[1])),
cast(TestPlace.coord, Geometry),
))
places = query.limit(100)
return [place for place in places]
def suggest_larger_areas(place_identifier):
top = get_place(place_identifier)
for place in top.go_bigger():
area_in_sq_km = place.area_in_sq_km
print(f'{area_in_sq_km:>10.1f} sq km {place.name}')
return
for e in reversed(top.is_in()):
# pprint(e)
osm_type, osm_id = e['type'], e['id']
if osm_type == top.osm_type and osm_id == top.osm_id:
continue
level = e['tags'].get('admin_level')
# {'minlat': 49, 'minlon': -14, 'maxlat': 61.061, 'maxlon': 2}
box = func.ST_MakeEnvelope(e['bounds']['minlon'],
e['bounds']['minlat'],
e['bounds']['maxlon'],
e['bounds']['maxlat'], 4326)
bbox_area = database.session.query(func.ST_Area(
box.cast(Geography))).scalar()
area_in_sq_km = bbox_area / (1000 * 1000)
if area_in_sq_km > 20_000:
continue
place = Place.from_osm(osm_type, osm_id)
area_in_sq_km = place.area_in_sq_km
print(f'{area_in_sq_km:>10.1f} sq km', e['type'], f"{e['id']:10d}",
level, e['tags']['name'])
continue
print(f'{place.area_in_sq_km:>10.1f} sq km {place.name:30s}')
continue
hit = nominatim.reverse(e['type'], e['id'], polygon_text=0)
pprint(hit)
print()
sleep(2)
def polygons(place_identifier):
place = get_place(place_identifier)
chunk_size = utils.calc_chunk_size(place.area_in_sq_km)
place_geojson = (database.session.query(func.ST_AsGeoJSON(
Place.geom, 4)).filter(Place.place_id == place.place_id).scalar())
# print(place_geojson)
for chunk in place.chunk_n(chunk_size):
print(', '.join('{:.3f}'.format(i) for i in chunk))
(ymin, ymax, xmin, xmax) = chunk
clip = func.ST_Intersection(
Place.geom, func.ST_MakeEnvelope(xmin, ymin, xmax, ymax))
chunk_geojson = (database.session.query(func.ST_AsGeoJSON(
clip, 4)).filter(Place.place_id == place.place_id).scalar())
print(chunk_geojson)
def point():
bbox = request.args.get('bbox')
page = request.args.get('page', type=int)
rows = request.args.get('rows', app.Config.POINTS_PER_PAGE, type=int)
if bbox:
split = bbox.split(',')
points = Point.query.filter(
func.ST_Contains(
func.ST_MakeEnvelope(split[0], split[1], split[2], split[3],
3857), Point.geom))
if page is None:
points = Point.query.all()
else:
points = Point.query.paginate(page, rows, False).items
data = {
'type': 'FeatureCollection',
'features': [incident.to_small_dict() for incident in points],
'totalCount': Point.query.count()
}
return jsonify(data)
def amenity():
if request.method == 'POST':
data = request.get_json() or {}
data["type"] = "amenity"
missing_fields = validate_amenity_data(data)
if (len(missing_fields) > 0):
return make_missing_fields_error(missing_fields)
amenity = Amenity()
amenity.from_dict(data)
db.session.add(amenity)
db.session.commit()
response = jsonify(amenity.to_small_dict())
response.status_code = 201
return response
# Handle GET request
bbox = request.args.get('bbox')
filter = request.args.get('filter', default=False, type=boolean)
page = request.args.get('page', type=int)
rows = request.args.get('rows', app.Config.POINTS_PER_PAGE, type=int)
if bbox:
split = bbox.split(',')
amenities = Amenity.query.filter(
func.ST_Contains(
func.ST_MakeEnvelope(split[0], split[1], split[2], split[3],
3857), Amenity.geom))
if page is None:
amenities = Amenity.query.all()
else:
amenities = Amenity.query.paginate(page, rows, False).items
if filter == True:
amenities = _filter_amenities(amenities)
data = {
'type': 'FeatureCollection',
'features': [amenity.to_small_dict() for amenity in amenities],
'totalCount': Amenity.query.count()
}
return jsonify(data)
def get_collection_items(
collection_id=None,
roles=[],
item_id=None,
bbox=None,
datetime=None,
ids=None,
collections=None,
intersects=None,
page=1,
limit=10,
query=None,
**kwargs,
):
"""Retrieve a list of collection items based on filters.
:param collection_id: Single Collection ID to include in the search for items.
Only Items in one of the provided Collection will be searched, defaults to None
:type collection_id: str, optional
:param item_id: item identifier, defaults to None
:type item_id: str, optional
:param bbox: bounding box for intersection [west, north, east, south], defaults to None
:type bbox: list, optional
:param datetime: Single date+time, or a range ('/' seperator), formatted to RFC 3339, section 5.6.
Use double dots '..' for open date ranges, defaults to None. If the start or end date of an image
generated by a temporal composition intersects the given datetime or range it will be included in the
result.
:type datetime: str, optional
:param ids: Array of Item ids to return. All other filter parameters that further restrict the
number of search results are ignored, defaults to None
:type ids: list, optional
:param collections: Array of Collection IDs to include in the search for items.
Only Items in one of the provided Collections will be searched, defaults to None
:type collections: list, optional
:param intersects: Searches items by performing intersection between their geometry and provided GeoJSON geometry.
All GeoJSON geometry types must be supported., defaults to None
:type intersects: dict, optional
:param page: The page offset of results, defaults to 1
:type page: int, optional
:param limit: The maximum number of results to return (page size), defaults to 10
:type limit: int, optional
:return: list of collectio items
:rtype: list
"""
columns = [
func.concat(Collection.name, "-",
Collection.version).label("collection"),
Collection.collection_type,
Collection._metadata.label("meta"),
Item._metadata.label("item_meta"),
Item.name.label("item"),
Item.id,
Item.collection_id,
Item.start_date.label("start"),
Item.end_date.label("end"),
Item.assets,
Item.created,
Item.updated,
cast(Item.cloud_cover, Float).label("cloud_cover"),
func.ST_AsGeoJSON(Item.geom).label("geom"),
func.Box2D(Item.geom).label("bbox"),
Tile.name.label("tile"),
]
where = [
Collection.id == Item.collection_id,
or_(Collection.is_public.is_(True),
Collection.id.in_([int(r.split(":")[0]) for r in roles])),
]
if ids is not None:
where += [Item.name.in_(ids.split(","))]
else:
if collections is not None:
where += [
func.concat(Collection.name, "-",
Collection.version).in_(collections.split(","))
]
elif collection_id is not None:
where += [
func.concat(Collection.name, "-",
Collection.version) == collection_id
]
if item_id is not None:
where += [Item.name.like(item_id)]
if query:
filters = create_query_filter(query)
if filters:
where += filters
if intersects is not None:
where += [
func.ST_Intersects(func.ST_GeomFromGeoJSON(str(intersects)),
Item.geom)
]
elif bbox is not None:
try:
split_bbox = [float(x) for x in bbox.split(",")]
if split_bbox[0] == split_bbox[2] or split_bbox[
1] == split_bbox[3]:
raise InvalidBoundingBoxError("")
where += [
func.ST_Intersects(
func.ST_MakeEnvelope(
split_bbox[0],
split_bbox[1],
split_bbox[2],
split_bbox[3],
func.ST_SRID(Item.geom),
),
Item.geom,
)
]
except:
raise (
InvalidBoundingBoxError(f"'{bbox}' is not a valid bbox."))
if datetime is not None:
date_filter = None
if "/" in datetime:
matches_open = ("..", "")
time_start, time_end = datetime.split("/")
if time_start in matches_open: # open start
date_filter = [
or_(Item.start_date <= time_end,
Item.end_date <= time_end)
]
elif time_end in matches_open: # open end
date_filter = [
or_(Item.start_date >= time_start,
Item.end_date >= time_start)
]
else: # closed range
date_filter = [
or_(
and_(Item.start_date >= time_start,
Item.start_date <= time_end),
and_(Item.end_date >= time_start,
Item.end_date <= time_end),
and_(Item.start_date < time_start,
Item.end_date > time_end),
)
]
else:
date_filter = [
and_(Item.start_date <= datetime,
Item.end_date >= datetime)
]
where += date_filter
outer = [Item.tile_id == Tile.id]
query = session.query(*columns).outerjoin(
Tile, *outer).filter(*where).order_by(Item.start_date.desc(), Item.id)
result = query.paginate(page=int(page),
per_page=int(limit),
error_out=False,
max_per_page=BDC_STAC_MAX_LIMIT)
return result
def envelope(bbox):
# note: different order for coordinates, xmin first, not ymin
ymin, ymax, xmin, xmax = bbox
return func.ST_MakeEnvelope(xmin, ymin, xmax, ymax, 4326)
def search_items(
self,
*,
product_name: Optional[str] = None,
time: Optional[Tuple[datetime, datetime]] = None,
bbox: Tuple[float, float, float, float] = None,
limit: int = 500,
offset: int = 0,
full_dataset: bool = False,
dataset_ids: Sequence[UUID] = None,
require_geometry=True,
ordered=True,
) -> Generator[DatasetItem, None, None]:
"""
Search datasets using Cubedash's spatial table
Returned as DatasetItem records, with optional embedded full Datasets
(if full_dataset==True)
Returned results are always sorted by (center_time, id)
"""
geom = func.ST_Transform(DATASET_SPATIAL.c.footprint, 4326)
columns = [
geom.label("geometry"),
func.Box2D(geom).label("bbox"),
# TODO: dataset label?
DATASET_SPATIAL.c.region_code.label("region_code"),
DATASET_SPATIAL.c.creation_time,
DATASET_SPATIAL.c.center_time,
]
# If fetching the whole dataset, we need to join the ODC dataset table.
if full_dataset:
query: Select = select(
(*columns, *_utils.DATASET_SELECT_FIELDS)).select_from(
DATASET_SPATIAL.join(
ODC_DATASET,
onclause=ODC_DATASET.c.id == DATASET_SPATIAL.c.id))
# Otherwise query purely from the spatial table.
else:
query: Select = select((*columns, DATASET_SPATIAL.c.id,
DATASET_SPATIAL.c.dataset_type_ref
)).select_from(DATASET_SPATIAL)
if time:
query = query.where(
func.tstzrange(
_utils.default_utc(time[0]),
_utils.default_utc(time[1]),
"[]",
type_=TSTZRANGE,
).contains(DATASET_SPATIAL.c.center_time))
if bbox:
query = query.where(
func.ST_Transform(DATASET_SPATIAL.c.footprint,
4326).intersects(
func.ST_MakeEnvelope(*bbox)))
if product_name:
query = query.where(DATASET_SPATIAL.c.dataset_type_ref == select(
[ODC_DATASET_TYPE.c.id]).where(
ODC_DATASET_TYPE.c.name == product_name))
if dataset_ids:
query = query.where(DATASET_SPATIAL.c.id.in_(dataset_ids))
if require_geometry:
query = query.where(DATASET_SPATIAL.c.footprint != None)
if ordered:
query = query.order_by(DATASET_SPATIAL.c.center_time,
DATASET_SPATIAL.c.id)
query = query.limit(limit).offset(
# TODO: Offset/limit isn't particularly efficient for paging...
offset)
for r in self._engine.execute(query):
yield DatasetItem(
dataset_id=r.id,
bbox=_box2d_to_bbox(r.bbox) if r.bbox else None,
product_name=self.index.products.get(r.dataset_type_ref).name,
geometry=_get_shape(r.geometry),
region_code=r.region_code,
creation_time=r.creation_time,
center_time=r.center_time,
odc_dataset=(_utils.make_dataset_from_select_fields(
self.index, r) if full_dataset else None),
)
def get_collection_items(
collection_id=None,
roles=None,
item_id=None,
bbox=None,
datetime=None,
ids=None,
collections=None,
intersects=None,
page=1,
limit=10,
query=None,
**kwargs,
) -> Pagination:
"""Retrieve a list of collection items based on filters.
:param collection_id: Single Collection ID to include in the search for items.
Only Items in one of the provided Collection will be searched, defaults to None
:type collection_id: str, optional
:param item_id: item identifier, defaults to None
:type item_id: str, optional
:param bbox: bounding box for intersection [west, north, east, south], defaults to None
:type bbox: list, optional
:param datetime: Single date+time, or a range ('/' seperator), formatted to RFC 3339, section 5.6.
Use double dots '..' for open date ranges, defaults to None. If the start or end date of an image
generated by a temporal composition intersects the given datetime or range it will be included in the
result.
:type datetime: str, optional
:param ids: Array of Item ids to return. All other filter parameters that further restrict the
number of search results are ignored, defaults to None
:type ids: list, optional
:param collections: Array of Collection IDs to include in the search for items.
Only Items in one of the provided Collections will be searched, defaults to None
:type collections: list, optional
:param intersects: Searches items by performing intersection between their geometry and provided GeoJSON geometry.
All GeoJSON geometry types must be supported., defaults to None
:type intersects: dict, optional
:param page: The page offset of results, defaults to 1
:type page: int, optional
:param limit: The maximum number of results to return (page size), defaults to 10
:type limit: int, optional
:return: list of collectio items
:rtype: list
"""
columns = [
func.concat(Collection.name, "-",
Collection.version).label("collection"),
Collection.collection_type,
Collection._metadata.label("meta"),
Item._metadata.label("item_meta"),
Item.name.label("item"),
Item.id,
Item.collection_id,
Item.start_date.label("start"),
Item.end_date.label("end"),
Item.assets,
Item.created,
Item.updated,
cast(Item.cloud_cover, Float).label("cloud_cover"),
func.ST_AsGeoJSON(Item.geom).label("geom"),
func.ST_XMin(Item.geom).label("xmin"),
func.ST_XMax(Item.geom).label("xmax"),
func.ST_YMin(Item.geom).label("ymin"),
func.ST_YMax(Item.geom).label("ymax"),
Tile.name.label("tile"),
]
if roles is None:
roles = []
where = [
Collection.id == Item.collection_id,
or_(Collection.is_public.is_(True),
Collection.id.in_([int(r.split(":")[0]) for r in roles])),
]
collections_where = _where_collections(collection_id, collections)
collections_where.append(
or_(Collection.is_public.is_(True),
Collection.id.in_([int(r.split(":")[0]) for r in roles])))
outer_join = [(Tile, [Item.tile_id == Tile.id])]
_geom_tables = []
_collections = Collection.query().filter(*collections_where).all()
if bbox or intersects:
grids = GridRefSys.query().filter(
GridRefSys.id.in_([c.grid_ref_sys_id
for c in _collections])).all()
for grid in grids:
geom_table = grid.geom_table
if geom_table is None:
continue
_geom_tables.append(geom_table)
if ids is not None:
if isinstance(ids, str):
ids = ids.split(",")
where += [Item.name.in_(ids)]
else:
where += _where_collections(collection_id, collections)
if item_id is not None:
where += [Item.name.like(item_id)]
if query:
filters = create_query_filter(query)
if filters:
where += filters
if intersects is not None:
# Intersect with native grid if there is
geom_expr = func.ST_GeomFromGeoJSON(str(intersects))
grids_where, joins = intersect_grids(geom_expr,
geom_tables=_geom_tables)
where += grids_where
outer_join += joins
elif bbox is not None:
try:
if isinstance(bbox, str):
bbox = bbox.split(",")
bbox = [float(x) for x in bbox]
if bbox[0] == bbox[2] or bbox[1] == bbox[3]:
raise InvalidBoundingBoxError("")
geom_expr = func.ST_MakeEnvelope(bbox[0], bbox[1], bbox[2],
bbox[3],
func.ST_SRID(Item.geom))
grid_where, joins = intersect_grids(geom_expr,
geom_tables=_geom_tables)
where += grid_where
outer_join += joins
except (ValueError, InvalidBoundingBoxError) as e:
abort(400, f"'{bbox}' is not a valid bbox.")
if datetime is not None:
if "/" in datetime:
matches_open = ("..", "")
time_start, time_end = datetime.split("/")
if time_start in matches_open: # open start
date_filter = [
or_(Item.start_date <= time_end,
Item.end_date <= time_end)
]
elif time_end in matches_open: # open end
date_filter = [
or_(Item.start_date >= time_start,
Item.end_date >= time_start)
]
else: # closed range
date_filter = [
or_(
and_(Item.start_date >= time_start,
Item.start_date <= time_end),
and_(Item.end_date >= time_start,
Item.end_date <= time_end),
and_(Item.start_date < time_start,
Item.end_date > time_end),
)
]
else:
date_filter = [
and_(Item.start_date <= datetime,
Item.end_date >= datetime)
]
where += date_filter
query = session.query(*columns)
for entity, join_conditions in outer_join:
query = query.outerjoin(entity, *join_conditions)
try:
query = query.filter(*where).order_by(Item.start_date.desc(), Item.id)
result = query.paginate(page=int(page),
per_page=int(limit),
error_out=False,
max_per_page=BDC_STAC_MAX_LIMIT)
return result
except Exception as err:
msg = str(err)
if hasattr(err, "orig"):
msg = str(err.orig)
abort(400, msg.rstrip())
def check_scenes(cls, collections: str, start_date: datetime, end_date: datetime,
catalog: str = None, dataset: str = None,
grid: str = None, tiles: list = None, bbox: list = None, catalog_kwargs=None, only_tiles=False):
"""Check for the scenes in remote provider and compares with the Collection Builder."""
bbox_list = []
if grid and tiles:
grid = GridRefSys.query().filter(GridRefSys.name == grid).first_or_404(f'Grid "{grid}" not found.')
geom_table = grid.geom_table
rows = db.session.query(
geom_table.c.tile,
func.ST_Xmin(func.ST_Transform(geom_table.c.geom, 4326)).label('xmin'),
func.ST_Ymin(func.ST_Transform(geom_table.c.geom, 4326)).label('ymin'),
func.ST_Xmax(func.ST_Transform(geom_table.c.geom, 4326)).label('xmax'),
func.ST_Ymax(func.ST_Transform(geom_table.c.geom, 4326)).label('ymax'),
).filter(geom_table.c.tile.in_(tiles)).all()
for row in rows:
bbox_list.append((row.tile, (row.xmin, row.ymin, row.xmax, row.ymax)))
else:
bbox_list.append(('', bbox))
instance, provider = get_provider(catalog)
collection_map = dict()
collection_ids = list()
for _collection in collections:
collection, version = _collection.split('-')
collection = Collection.query().filter(
Collection.name == collection,
Collection.version == version
).first_or_404(f'Collection "{collection}-{version}" not found.')
collection_ids.append(collection.id)
collection_map[_collection] = collection
options = dict(start_date=start_date, end_date=end_date)
if catalog_kwargs:
options.update(catalog_kwargs)
redis = current_app.redis
output = dict(
collections={cname: dict(total_scenes=0, total_missing=0, missing_external=[]) for cname in collections}
)
items = {cid: set() for cid in collection_ids}
external_scenes = set()
for tile, _bbox in bbox_list:
with redis.pipeline() as pipe:
if only_tiles:
entry = tile
options['tile'] = tile
else:
options['bbox'] = _bbox
entry = _bbox
periods = _generate_periods(start_date.replace(tzinfo=None), end_date.replace(tzinfo=None))
for period_start, period_end in periods:
_items = db.session.query(Item.name, Item.collection_id).filter(
Item.collection_id.in_(collection_ids),
func.ST_Intersects(
func.ST_MakeEnvelope(
*_bbox, func.ST_SRID(Item.geom)
),
Item.geom
),
or_(
and_(Item.start_date >= period_start, Item.start_date <= period_end),
and_(Item.end_date >= period_start, Item.end_date <= period_end),
and_(Item.start_date < period_start, Item.end_date > period_end),
)
).order_by(Item.name).all()
for item in _items:
items[item.collection_id].add(item.name)
options['start_date'] = period_start.strftime('%Y-%m-%d')
options['end_date'] = period_end.strftime('%Y-%m-%d')
key = f'scenes:{catalog}:{dataset}:{period_start.strftime("%Y%m%d")}_{period_end.strftime("%Y%m%d")}_{entry}'
pipe.get(key)
provider_scenes = []
if not redis.exists(key):
provider_scenes = provider.search(dataset, **options)
provider_scenes = [s.scene_id for s in provider_scenes]
pipe.set(key, json.dumps(provider_scenes))
external_scenes = external_scenes.union(set(provider_scenes))
cached_scenes = pipe.execute()
for cache in cached_scenes:
# When cache is True, represents set the value were cached.
if cache is not None and cache is not True:
external_scenes = external_scenes.union(set(json.loads(cache)))
output['total_external'] = len(external_scenes)
for _collection_name, _collection in collection_map.items():
_items = set(items[_collection.id])
diff = list(external_scenes.difference(_items))
output['collections'][_collection_name]['total_scenes'] = len(_items)
output['collections'][_collection_name]['total_missing'] = len(diff)
output['collections'][_collection_name]['missing_external'] = diff
for cname, _internal_collection in collection_map.items():
if cname != _collection_name:
diff = list(_items.difference(set(items[_internal_collection.id])))
output['collections'][_collection_name][f'total_missing_{cname}'] = len(diff)
output['collections'][_collection_name][f'missing_{cname}'] = diff
return output
def get_collection_items(collection_id=None,
roles=[],
item_id=None,
bbox=None,
time=None,
ids=None,
collections=None,
cubes=None,
intersects=None,
page=1,
limit=10,
query=None,
**kwargs):
"""Retrieve a list of collection items based on filters.
:param collection_id: Single Collection ID to include in the search for items.
Only Items in one of the provided Collection will be searched, defaults to None
:type collection_id: str, optional
:param item_id: item identifier, defaults to None
:type item_id: str, optional
:param bbox: bounding box for intersection [west, north, east, south], defaults to None
:type bbox: list, optional
:param time: Single date+time, or a range ('/' seperator), formatted to RFC 3339, section 5.6, defaults to None
:type time: str, optional
:param ids: Array of Item ids to return. All other filter parameters that further restrict the
number of search results are ignored, defaults to None
:type ids: list, optional
:param collections: Array of Collection IDs to include in the search for items.
Only Items in one of the provided Collections will be searched, defaults to None
:type collections: list, optional
:param cubes: Bool indicating if only cubes should be returned, defaults to None
:type cubes: bool, optional
:param intersects: Searches items by performing intersection between their geometry and provided GeoJSON geometry.
All GeoJSON geometry types must be supported., defaults to None
:type intersects: dict, optional
:param page: The page offset of results, defaults to 1
:type page: int, optional
:param limit: The maximum number of results to return (page size), defaults to 10
:type limit: int, optional
:return: list of collectio items
:rtype: list
"""
columns = [
Collection.name.label('collection'),
Item.name.label('item'),
Item.start_date.label('start'),
Item.end_date.label('end'), Item.assets,
func.ST_AsGeoJSON(Item.geom).label('geom'),
func.Box2D(Item.geom).label('bbox'),
Tile.name.label('tile')
]
where = [
Collection.id == Item.collection_id, Item.tile_id == Tile.id,
or_(Collection.is_public.is_(True),
Collection.id.in_([int(r.split(':')[0]) for r in roles]))
]
if ids is not None:
where += [Item.id.in_(ids.split(','))]
elif item_id is not None:
where += [Item.id.like(item_id)]
else:
if collections is not None:
where += [Collection.name.in_(collections.split(','))]
elif collection_id is not None:
where += [Collection.name.like(collection_id)]
if intersects is not None:
where += [
func.ST_Intersects(func.ST_GeomFromGeoJSON(str(intersects)),
Item.geom)
]
if query:
filters = create_query_filter(query)
if filters:
where += filters
if bbox is not None:
try:
split_bbox = [float(x) for x in bbox.split(',')]
where += [
func.ST_Intersects(
func.ST_MakeEnvelope(split_bbox[0], split_bbox[1],
split_bbox[2], split_bbox[3],
func.ST_SRID(Item.geom)),
Item.geom)
]
except:
raise (
InvalidBoundingBoxError(f"'{bbox}' is not a valid bbox.'"))
if time is not None:
if "/" in time:
time_start, time_end = time.split("/")
time_end = datetime.fromisoformat(time_end)
where += [
or_(Item.end_date <= time_end, Item.start_date <= time_end)
]
else:
time_start = datetime.fromisoformat(time)
where += [
or_(Item.start_date >= time_start, Item.end_date >= time_start)
]
query = session.query(*columns).filter(*where).order_by(
Item.start_date.desc())
result = query.paginate(page=int(page),
per_page=int(limit),
error_out=False,
max_per_page=int(BDC_STAC_MAX_LIMIT))
return result
