def _safe_read_date(d):
if d:
return _utils.default_utc(dateutil.parser.parse(d))
return None
def expect_values(
s: TimePeriodOverview,
dataset_count: int,
footprint_count: int,
time_range: Range,
newest_creation_time: datetime,
timeline_period: str,
timeline_count: int,
crses: Set[str],
size_bytes: Optional[int],
region_dataset_counts: Dict = None,
):
__tracebackhide__ = True
was_timeline_error = False
was_regions_error = False
try:
assert s.dataset_count == dataset_count, "wrong dataset count"
assert s.footprint_count == footprint_count, "wrong footprint count"
if s.footprint_count is not None and s.footprint_count > 0:
assert (s.footprint_geometry
is not None), "No footprint, despite footprint count"
assert s.footprint_geometry.area > 0, "Empty footprint"
assert s.time_range == time_range, "wrong dataset time range"
assert s.newest_dataset_creation_time == default_utc(
newest_creation_time), "wrong newest dataset creation"
assert s.timeline_period == timeline_period, (
f"Should be a {timeline_period}, "
f"not {s.timeline_period} timeline")
assert (s.summary_gen_time
is not None), "Missing summary_gen_time (there's a default)"
assert s.crses == crses, "Wrong dataset CRSes"
if size_bytes is None:
assert s.size_bytes is None, "Expected null size_bytes"
else:
assert s.size_bytes == size_bytes, "Wrong size_bytes"
assert s.summary_gen_time is not None, "Missing summary_gen_time"
was_timeline_error = True
if s.timeline_dataset_counts is None:
if timeline_count is not None:
raise AssertionError(
f"null timeline_dataset_counts. "
f"Expected entry with {timeline_count} records.")
else:
assert (len(s.timeline_dataset_counts) == timeline_count
), "wrong timeline entry count"
assert (sum(s.region_dataset_counts.values()) == s.dataset_count
), "region dataset count doesn't match total dataset count"
assert (sum(s.timeline_dataset_counts.values()) == s.dataset_count
), "timeline count doesn't match dataset count"
was_timeline_error = False
if region_dataset_counts is not None:
was_regions_error = True
if s.region_dataset_counts is None:
if region_dataset_counts is not None:
raise AssertionError(
f"No region counts found. "
f"Expected entry with {len(region_dataset_counts)} records."
)
else:
assert region_dataset_counts == s.region_dataset_counts
was_regions_error = False
except AssertionError:
print(f"""Got:
dataset_count {s.dataset_count}
footprint_count {s.footprint_count}
time range:
- {repr(s.time_range.begin.astimezone(tzutc()))}
- {repr(s.time_range.end.astimezone(tzutc()))}
newest: {repr(s.newest_dataset_creation_time.astimezone(tzutc()))}
crses: {repr(s.crses)}
size_bytes: {s.size_bytes}
timeline
period: {s.timeline_period}
dataset_counts: {None if s.timeline_dataset_counts is None else len(s.timeline_dataset_counts)}
""")
if was_timeline_error:
print("timeline keys:")
for day, count in s.timeline_dataset_counts.items():
print(f"\t{repr(day)}: {count}")
if was_regions_error:
print("region keys:")
for region, count in s.region_dataset_counts.items():
print(f"\t{repr(region)}: {count}")
raise
def search_items(
self,
*,
product_name: Optional[str] = None,
time: Optional[Tuple[datetime, datetime]] = None,
bbox: Tuple[float, float, float, float] = None,
limit: int = 500,
offset: int = 0,
full_dataset: bool = False,
dataset_ids: Sequence[UUID] = None,
require_geometry=True,
ordered=True,
) -> Generator[DatasetItem, None, None]:
"""
Search datasets using Cubedash's spatial table
Returned as DatasetItem records, with optional embedded full Datasets
(if full_dataset==True)
Returned results are always sorted by (center_time, id)
"""
geom = func.ST_Transform(DATASET_SPATIAL.c.footprint, 4326)
columns = [
geom.label("geometry"),
func.Box2D(geom).label("bbox"),
# TODO: dataset label?
DATASET_SPATIAL.c.region_code.label("region_code"),
DATASET_SPATIAL.c.creation_time,
DATASET_SPATIAL.c.center_time,
]
# If fetching the whole dataset, we need to join the ODC dataset table.
if full_dataset:
query: Select = select(
(*columns, *_utils.DATASET_SELECT_FIELDS)).select_from(
DATASET_SPATIAL.join(
ODC_DATASET,
onclause=ODC_DATASET.c.id == DATASET_SPATIAL.c.id))
# Otherwise query purely from the spatial table.
else:
query: Select = select((*columns, DATASET_SPATIAL.c.id,
DATASET_SPATIAL.c.dataset_type_ref
)).select_from(DATASET_SPATIAL)
if time:
query = query.where(
func.tstzrange(
_utils.default_utc(time[0]),
_utils.default_utc(time[1]),
"[]",
type_=TSTZRANGE,
).contains(DATASET_SPATIAL.c.center_time))
if bbox:
query = query.where(
func.ST_Transform(DATASET_SPATIAL.c.footprint,
4326).intersects(
func.ST_MakeEnvelope(*bbox)))
if product_name:
query = query.where(DATASET_SPATIAL.c.dataset_type_ref == select(
[ODC_DATASET_TYPE.c.id]).where(
ODC_DATASET_TYPE.c.name == product_name))
if dataset_ids:
query = query.where(DATASET_SPATIAL.c.id.in_(dataset_ids))
if require_geometry:
query = query.where(DATASET_SPATIAL.c.footprint != None)
if ordered:
query = query.order_by(DATASET_SPATIAL.c.center_time,
DATASET_SPATIAL.c.id)
query = query.limit(limit).offset(
# TODO: Offset/limit isn't particularly efficient for paging...
offset)
for r in self._engine.execute(query):
yield DatasetItem(
dataset_id=r.id,
bbox=_box2d_to_bbox(r.bbox) if r.bbox else None,
product_name=self.index.products.get(r.dataset_type_ref).name,
geometry=_get_shape(r.geometry),
region_code=r.region_code,
creation_time=r.creation_time,
center_time=r.center_time,
odc_dataset=(_utils.make_dataset_from_select_fields(
self.index, r) if full_dataset else None),
)