def update(
self,
product_name: Optional[str],
year: Optional[int] = None,
month: Optional[int] = None,
day: Optional[int] = None,
generate_missing_children: Optional[bool] = True,
force_refresh: Optional[bool] = False,
):
"""Update the given summary and return the new one"""
product = self._product(product_name)
get_child = self.get_or_update if generate_missing_children else self.get
if year and month and day:
# Don't store days, they're quick.
return self._summariser.calculate_summary(
product_name, _utils.as_time_range(year, month, day))
elif year and month:
summary = self._summariser.calculate_summary(
product_name, _utils.as_time_range(year, month))
elif year:
summary = TimePeriodOverview.add_periods(
get_child(product_name,
year,
month_,
None,
force_refresh=force_refresh)
for month_ in range(1, 13))
elif product_name:
if product.dataset_count > 0:
years = range(product.time_earliest.year,
product.time_latest.year + 1)
else:
years = []
summary = TimePeriodOverview.add_periods(
get_child(product_name,
year_,
None,
None,
force_refresh=force_refresh) for year_ in years)
else:
summary = TimePeriodOverview.add_periods(
get_child(product.name,
None,
None,
None,
force_refresh=force_refresh)
for product in self.all_dataset_types())
self._do_put(product_name, year, month, day, summary)
for listener in self._update_listeners:
listener(product_name, year, month, day, summary)
return summary
def test_add_period_list():
total = TimePeriodOverview.add_periods([])
assert total.dataset_count == 0
joined = TimePeriodOverview.add_periods([_overview(), _overview(), total])
assert joined.dataset_count == _overview().dataset_count * 2
assert _overview().footprint_geometry.area == pytest.approx(
joined.footprint_geometry.area)
assert sum(joined.region_dataset_counts.values()) == joined.dataset_count
assert sum(joined.timeline_dataset_counts.values()) == joined.dataset_count
assert joined.crses == _overview().crses
assert joined.size_bytes == _overview().size_bytes * 2
assert sorted(joined.region_dataset_counts.keys()) == ["1_2", "3_4", "4_5"]
def _summary_from_row(res):
timeline_dataset_counts = (Counter(
dict(
zip(res["timeline_dataset_start_days"],
res["timeline_dataset_counts"])))
if res["timeline_dataset_start_days"] else None)
region_dataset_counts = (Counter(
dict(zip(res["regions"], res["region_dataset_counts"])))
if res["regions"] else None)
return TimePeriodOverview(
dataset_count=res["dataset_count"],
# : Counter
timeline_dataset_counts=timeline_dataset_counts,
region_dataset_counts=region_dataset_counts,
timeline_period=res["timeline_period"],
# : Range
time_range=Range(res["time_earliest"], res["time_latest"])
if res["time_earliest"] else None,
# shapely.geometry.base.BaseGeometry
footprint_geometry=(None if res["footprint_geometry"] is None else
geo_shape.to_shape(res["footprint_geometry"])),
footprint_crs=(None if res["footprint_geometry"] is None
or res["footprint_geometry"].srid == -1 else
"EPSG:{}".format(res["footprint_geometry"].srid)),
size_bytes=res["size_bytes"],
footprint_count=res["footprint_count"],
# The most newly created dataset
newest_dataset_creation_time=res["newest_dataset_creation_time"],
# When this summary was last generated
summary_gen_time=res["generation_time"],
crses=set(res["crses"]) if res["crses"] is not None else None,
)
def _put(
self,
product_name: Optional[str],
year: Optional[int],
month: Optional[int],
day: Optional[int],
summary: TimePeriodOverview,
):
product = self._product(product_name)
start_day, period = self._start_day(year, month, day)
row = _summary_to_row(summary)
ret = self._engine.execute(
postgres.insert(TIME_OVERVIEW).returning(
TIME_OVERVIEW.c.generation_time).on_conflict_do_update(
index_elements=["product_ref", "start_day", "period_type"],
set_=row,
where=and_(
TIME_OVERVIEW.c.product_ref == product.id_,
TIME_OVERVIEW.c.start_day == start_day,
TIME_OVERVIEW.c.period_type == period,
),
).values(product_ref=product.id_,
start_day=start_day,
period_type=period,
**row))
[gen_time] = ret.fetchone()
summary.summary_gen_time = gen_time
def _summary_from_row(res):
timeline_dataset_counts = Counter(
dict(
zip(res['timeline_dataset_start_days'],
res['timeline_dataset_counts']))
) if res['timeline_dataset_start_days'] else None
region_dataset_counts = Counter(
dict(zip(res['regions'],
res['region_dataset_counts']))) if res['regions'] else None
return TimePeriodOverview(
dataset_count=res['dataset_count'],
# : Counter
timeline_dataset_counts=timeline_dataset_counts,
region_dataset_counts=region_dataset_counts,
timeline_period=res['timeline_period'],
# : Range
time_range=Range(res['time_earliest'], res['time_latest'])
if res['time_earliest'] else None,
# shapely.geometry.base.BaseGeometry
footprint_geometry=(None if res['footprint_geometry'] is None else
geo_shape.to_shape(res['footprint_geometry'])),
footprint_crs=(None if res['footprint_geometry'] is None
or res['footprint_geometry'].srid == -1 else
'EPSG:{}'.format(res['footprint_geometry'].srid)),
size_bytes=res['size_bytes'],
footprint_count=res['footprint_count'],
# The most newly created dataset
newest_dataset_creation_time=res['newest_dataset_creation_time'],
# When this summary was last generated
summary_gen_time=res['generation_time'],
crses=set(res['crses']) if res['crses'] is not None else None,
)
def _overview(
product_name: str = "test_product",
year: int = None,
month: int = None,
day: int = None,
):
orig = TimePeriodOverview(
product_name=product_name,
year=year,
month=month,
day=day,
dataset_count=4,
timeline_dataset_counts=Counter([
datetime(2017, 1, 2, tzinfo=tz.tzutc()),
datetime(2017, 1, 3, tzinfo=tz.tzutc()),
datetime(2017, 1, 3, tzinfo=tz.tzutc()),
datetime(2017, 1, 1, tzinfo=tz.tzutc()),
]),
region_dataset_counts=Counter(["1_2", "1_2", "3_4", "4_5"]),
timeline_period="day",
time_range=Range(
datetime(2017, 1, 2, tzinfo=tz.tzutc()),
datetime(2017, 2, 3, tzinfo=tz.tzutc()),
),
footprint_geometry=geo.Polygon([
# ll:
(-29.882_024, 113.105_949),
# lr:
(-29.930_607, 115.464_187),
# ur:
(-27.849_244, 115.494_523),
# ul
(-27.804_641, 113.18267),
]),
footprint_crs="EPSG:3577",
footprint_count=3,
newest_dataset_creation_time=datetime(2018,
1,
1,
1,
1,
1,
tzinfo=tz.tzutc()),
crses={"epsg:1234"},
size_bytes=123_400_000,
product_refresh_time=datetime(2018, 2, 3, 1, 1, 1, tzinfo=tz.tzutc()),
)
return orig
def _overview():
orig = TimePeriodOverview(
dataset_count=4,
timeline_dataset_counts=Counter([
datetime(2017, 1, 2, tzinfo=tz.tzutc()),
datetime(2017, 1, 3, tzinfo=tz.tzutc()),
datetime(2017, 1, 3, tzinfo=tz.tzutc()),
datetime(2017, 1, 1, tzinfo=tz.tzutc())
]),
region_dataset_counts=Counter([
"1_2",
"1_2",
"3_4",
"4_5",
]),
timeline_period='day',
time_range=Range(datetime(2017, 1, 2, tzinfo=tz.tzutc()),
datetime(2017, 2, 3, tzinfo=tz.tzutc())),
footprint_geometry=geo.Polygon([
# ll:
(-29.882024, 113.105949),
# lr:
(-29.930607, 115.464187),
# ur:
(-27.849244, 115.494523),
# ul
(-27.804641, 113.18267),
]),
footprint_crs='EPSG:3577',
footprint_count=3,
newest_dataset_creation_time=datetime(2018,
1,
1,
1,
1,
1,
tzinfo=tz.tzutc()),
crses={'epsg:1234'},
size_bytes=123_400_000,
)
return orig
wofs_time_summary = TimePeriodOverview(
product_name="wofs_summary",
year=None,
month=None,
day=None,
dataset_count=1244,
timeline_dataset_counts=Counter(
{
datetime(
1970, 1, 1, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 1244,
datetime(
1970, 1, 2, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 3, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 4, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 5, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 6, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 7, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 8, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 9, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 10, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 11, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 12, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 13, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 14, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 15, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 16, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 17, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 18, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 19, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 20, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 21, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 22, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 23, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 24, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 25, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 26, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 27, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 28, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 29, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 30, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
datetime(
1970, 1, 31, 0, 0, tzinfo=FixedOffsetTimezone(offset=600, name=None)
): 0,
}
),
region_dataset_counts=Counter(
{
"-10_-11": 1,
"-10_-12": 1,
"-10_-13": 1,
"-10_-14": 1,
"-10_-15": 1,
"-10_-16": 1,
"-10_-17": 1,
"-10_-18": 1,
"-10_-19": 1,
"-10_-20": 1,
"-10_-21": 1,
"-10_-22": 1,
"-10_-23": 1,
"-10_-24": 1,
"-10_-25": 1,
"-10_-26": 1,
"-10_-27": 1,
"-10_-28": 1,
"-10_-29": 1,
"-10_-30": 1,
"-10_-31": 1,
"-10_-32": 1,
"-10_-33": 1,
"-10_-34": 1,
"-10_-35": 1,
"-10_-36": 1,
"-10_-37": 1,
"-10_-38": 1,
"-10_-39": 1,
"-10_-40": 1,
"-11_-11": 1,
"-11_-12": 1,
"-11_-13": 1,
"-11_-14": 1,
"-11_-15": 1,
"-11_-16": 1,
"-11_-17": 1,
"-11_-18": 1,
"-11_-19": 1,
"-11_-20": 1,
"-11_-21": 1,
"-11_-22": 1,
"-11_-23": 1,
"-11_-24": 1,
"-11_-25": 1,
"-11_-26": 1,
"-11_-27": 1,
"-11_-28": 1,
"-11_-29": 1,
"-11_-30": 1,
"-11_-31": 1,
"-11_-32": 1,
"-11_-33": 1,
"-11_-34": 1,
"-11_-35": 1,
"-11_-36": 1,
"-11_-37": 1,
"-11_-38": 1,
"-11_-39": 1,
"-11_-40": 1,
"-12_-11": 1,
"-12_-12": 1,
"-12_-13": 1,
"-12_-14": 1,
"-12_-15": 1,
"-12_-16": 1,
"-12_-17": 1,
"-12_-18": 1,
"-12_-19": 1,
"-12_-20": 1,
"-12_-21": 1,
"-12_-22": 1,
"-12_-23": 1,
"-12_-24": 1,
"-12_-25": 1,
"-12_-26": 1,
"-12_-27": 1,
"-12_-28": 1,
"-12_-29": 1,
"-12_-30": 1,
"-12_-31": 1,
"-12_-32": 1,
"-12_-33": 1,
"-12_-34": 1,
"-12_-35": 1,
"-12_-36": 1,
"-12_-37": 1,
"-12_-38": 1,
"-12_-39": 1,
"-12_-40": 1,
"-12_-41": 1,
"-12_-42": 1,
"-13_-11": 1,
"-13_-12": 1,
"-13_-13": 1,
"-13_-14": 1,
"-13_-15": 1,
"-13_-16": 1,
"-13_-17": 1,
"-13_-19": 1,
"-13_-20": 1,
"-13_-21": 1,
"-13_-22": 1,
"-13_-23": 1,
"-13_-24": 1,
"-13_-25": 1,
"-13_-26": 1,
"-13_-27": 1,
"-13_-28": 1,
"-13_-29": 1,
"-13_-30": 1,
"-13_-31": 1,
"-13_-32": 1,
"-13_-33": 1,
"-13_-34": 1,
"-13_-35": 1,
"-13_-36": 1,
"-13_-37": 1,
"-13_-38": 1,
"-13_-39": 1,
"-13_-40": 1,
"-13_-41": 1,
"-13_-42": 1,
"-14_-12": 1,
"-14_-13": 1,
"-14_-15": 1,
"-14_-16": 1,
"-14_-17": 1,
"-14_-18": 1,
"-14_-19": 1,
"-14_-20": 1,
"-14_-21": 1,
"-14_-22": 1,
"-14_-23": 1,
"-14_-24": 1,
"-14_-25": 1,
"-14_-26": 1,
"-14_-27": 1,
"-14_-28": 1,
"-14_-29": 1,
"-14_-30": 1,
"-14_-31": 1,
"-14_-32": 1,
"-14_-33": 1,
"-14_-34": 1,
"-14_-35": 1,
"-14_-36": 1,
"-14_-37": 1,
"-14_-38": 1,
"-14_-39": 1,
"-14_-40": 1,
"-14_-41": 1,
"-15_-12": 1,
"-15_-13": 1,
"-15_-19": 1,
"-15_-20": 1,
"-15_-21": 1,
"-15_-22": 1,
"-15_-23": 1,
"-15_-24": 1,
"-15_-25": 1,
"-15_-26": 1,
"-15_-27": 1,
"-15_-28": 1,
"-15_-29": 1,
"-15_-30": 1,
"-15_-31": 1,
"-15_-32": 1,
"-15_-33": 1,
"-15_-34": 1,
"-15_-35": 1,
"-15_-36": 1,
"-15_-37": 1,
"-15_-38": 1,
"-15_-39": 1,
"-15_-40": 1,
"-16_-12": 1,
"-16_-13": 1,
"-16_-19": 1,
"-16_-20": 1,
"-16_-21": 1,
"-16_-22": 1,
"-16_-23": 1,
"-16_-24": 1,
"-16_-25": 1,
"-16_-26": 1,
"-16_-27": 1,
"-16_-28": 1,
"-16_-29": 1,
"-16_-30": 1,
"-16_-31": 1,
"-16_-32": 1,
"-16_-33": 1,
"-16_-34": 1,
"-16_-35": 1,
"-16_-36": 1,
"-16_-37": 1,
"-16_-38": 1,
"-16_-39": 1,
"-16_-40": 1,
"-17_-12": 1,
"-17_-13": 1,
"-17_-22": 1,
"-17_-23": 1,
"-17_-24": 1,
"-17_-25": 1,
"-17_-26": 1,
"-17_-27": 1,
"-17_-28": 1,
"-17_-29": 1,
"-17_-30": 1,
"-17_-31": 1,
"-17_-32": 1,
"-17_-33": 1,
"-17_-34": 1,
"-17_-35": 1,
"-17_-36": 1,
"-17_-37": 1,
"-17_-38": 1,
"-17_-39": 1,
"-17_-40": 1,
"-18_-22": 1,
"-18_-23": 1,
"-18_-24": 1,
"-18_-25": 1,
"-18_-26": 1,
"-18_-27": 1,
"-18_-28": 1,
"-18_-29": 1,
"-18_-30": 1,
"-18_-31": 1,
"-18_-32": 1,
"-18_-33": 1,
"-18_-34": 1,
"-18_-35": 1,
"-18_-36": 1,
"-18_-37": 1,
"-18_-38": 1,
"-18_-39": 1,
"-18_-40": 1,
"-19_-22": 1,
"-19_-23": 1,
"-19_-24": 1,
"-19_-25": 1,
"-19_-26": 1,
"-19_-27": 1,
"-19_-28": 1,
"-19_-29": 1,
"-19_-30": 1,
"-19_-31": 1,
"-19_-32": 1,
"-19_-33": 1,
"-19_-34": 1,
"-19_-35": 1,
"-1_-11": 1,
"-1_-12": 1,
"-1_-13": 1,
"-1_-14": 1,
"-1_-15": 1,
"-1_-16": 1,
"-1_-17": 1,
"-1_-18": 1,
"-1_-19": 1,
"-1_-20": 1,
"-1_-21": 1,
"-1_-22": 1,
"-1_-23": 1,
"-1_-24": 1,
"-1_-25": 1,
"-1_-26": 1,
"-1_-27": 1,
"-1_-28": 1,
"-1_-29": 1,
"-1_-30": 1,
"-1_-31": 1,
"-1_-32": 1,
"-1_-33": 1,
"-1_-34": 1,
"-1_-35": 1,
"-1_-36": 1,
"-1_-37": 1,
"-20_-24": 1,
"-20_-25": 1,
"-20_-26": 1,
"-20_-27": 1,
"-20_-28": 1,
"-2_-11": 1,
"-2_-12": 1,
"-2_-13": 1,
"-2_-14": 1,
"-2_-15": 1,
"-2_-16": 1,
"-2_-17": 1,
"-2_-18": 1,
"-2_-19": 1,
"-2_-20": 1,
"-2_-21": 1,
"-2_-22": 1,
"-2_-23": 1,
"-2_-24": 1,
"-2_-25": 1,
"-2_-26": 1,
"-2_-27": 1,
"-2_-28": 1,
"-2_-29": 1,
"-2_-30": 1,
"-2_-31": 1,
"-2_-32": 1,
"-2_-33": 1,
"-2_-34": 1,
"-2_-35": 1,
"-2_-36": 1,
"-3_-11": 1,
"-3_-12": 1,
"-3_-13": 1,
"-3_-14": 1,
"-3_-15": 1,
"-3_-16": 1,
"-3_-17": 1,
"-3_-18": 1,
"-3_-19": 1,
"-3_-20": 1,
"-3_-21": 1,
"-3_-22": 1,
"-3_-23": 1,
"-3_-24": 1,
"-3_-25": 1,
"-3_-26": 1,
"-3_-27": 1,
"-3_-28": 1,
"-3_-29": 1,
"-3_-30": 1,
"-3_-31": 1,
"-3_-32": 1,
"-3_-33": 1,
"-3_-34": 1,
"-3_-35": 1,
"-3_-36": 1,
"-4_-14": 1,
"-4_-15": 1,
"-4_-16": 1,
"-4_-17": 1,
"-4_-18": 1,
"-4_-19": 1,
"-4_-20": 1,
"-4_-21": 1,
"-4_-22": 1,
"-4_-23": 1,
"-4_-24": 1,
"-4_-25": 1,
"-4_-26": 1,
"-4_-27": 1,
"-4_-28": 1,
"-4_-29": 1,
"-4_-30": 1,
"-4_-31": 1,
"-4_-32": 1,
"-4_-33": 1,
"-4_-34": 1,
"-4_-35": 1,
"-4_-36": 1,
"-4_-37": 1,
"-4_-38": 1,
"-5_-13": 1,
"-5_-14": 1,
"-5_-15": 1,
"-5_-16": 1,
"-5_-17": 1,
"-5_-18": 1,
"-5_-19": 1,
"-5_-20": 1,
"-5_-21": 1,
"-5_-22": 1,
"-5_-23": 1,
"-5_-24": 1,
"-5_-25": 1,
"-5_-26": 1,
"-5_-27": 1,
"-5_-28": 1,
"-5_-29": 1,
"-5_-30": 1,
"-5_-31": 1,
"-5_-32": 1,
"-5_-33": 1,
"-5_-34": 1,
"-5_-35": 1,
"-5_-36": 1,
"-5_-37": 1,
"-5_-38": 1,
"-6_-13": 1,
"-6_-14": 1,
"-6_-15": 1,
"-6_-16": 1,
"-6_-17": 1,
"-6_-18": 1,
"-6_-19": 1,
"-6_-20": 1,
"-6_-21": 1,
"-6_-22": 1,
"-6_-23": 1,
"-6_-24": 1,
"-6_-25": 1,
"-6_-26": 1,
"-6_-27": 1,
"-6_-28": 1,
"-6_-29": 1,
"-6_-30": 1,
"-6_-31": 1,
"-6_-32": 1,
"-6_-33": 1,
"-6_-34": 1,
"-6_-35": 1,
"-6_-36": 1,
"-6_-37": 1,
"-6_-38": 1,
"-7_-11": 1,
"-7_-12": 1,
"-7_-13": 1,
"-7_-14": 1,
"-7_-15": 1,
"-7_-16": 1,
"-7_-17": 1,
"-7_-18": 1,
"-7_-19": 1,
"-7_-20": 1,
"-7_-21": 1,
"-7_-22": 1,
"-7_-23": 1,
"-7_-24": 1,
"-7_-25": 1,
"-7_-26": 1,
"-7_-27": 1,
"-7_-28": 1,
"-7_-29": 1,
"-7_-30": 1,
"-7_-31": 1,
"-7_-32": 1,
"-7_-33": 1,
"-7_-34": 1,
"-7_-35": 1,
"-7_-36": 1,
"-7_-37": 1,
"-7_-38": 1,
"-7_-39": 1,
"-7_-40": 1,
"-8_-11": 1,
"-8_-12": 1,
"-8_-13": 1,
"-8_-14": 1,
"-8_-15": 1,
"-8_-16": 1,
"-8_-17": 1,
"-8_-18": 1,
"-8_-19": 1,
"-8_-20": 1,
"-8_-21": 1,
"-8_-22": 1,
"-8_-23": 1,
"-8_-24": 1,
"-8_-25": 1,
"-8_-26": 1,
"-8_-27": 1,
"-8_-28": 1,
"-8_-29": 1,
"-8_-30": 1,
"-8_-31": 1,
"-8_-32": 1,
"-8_-33": 1,
"-8_-34": 1,
"-8_-35": 1,
"-8_-36": 1,
"-8_-37": 1,
"-8_-38": 1,
"-8_-39": 1,
"-8_-40": 1,
"-9_-11": 1,
"-9_-12": 1,
"-9_-13": 1,
"-9_-14": 1,
"-9_-15": 1,
"-9_-16": 1,
"-9_-17": 1,
"-9_-18": 1,
"-9_-19": 1,
"-9_-20": 1,
"-9_-21": 1,
"-9_-22": 1,
"-9_-23": 1,
"-9_-24": 1,
"-9_-25": 1,
"-9_-26": 1,
"-9_-27": 1,
"-9_-28": 1,
"-9_-29": 1,
"-9_-30": 1,
"-9_-31": 1,
"-9_-32": 1,
"-9_-33": 1,
"-9_-34": 1,
"-9_-35": 1,
"-9_-36": 1,
"-9_-37": 1,
"-9_-38": 1,
"-9_-39": 1,
"-9_-40": 1,
"0_-11": 1,
"0_-12": 1,
"0_-13": 1,
"0_-14": 1,
"0_-15": 1,
"0_-16": 1,
"0_-17": 1,
"0_-18": 1,
"0_-19": 1,
"0_-20": 1,
"0_-21": 1,
"0_-22": 1,
"0_-23": 1,
"0_-24": 1,
"0_-25": 1,
"0_-26": 1,
"0_-27": 1,
"0_-28": 1,
"0_-29": 1,
"0_-30": 1,
"0_-31": 1,
"0_-32": 1,
"0_-33": 1,
"0_-34": 1,
"0_-35": 1,
"0_-36": 1,
"0_-37": 1,
"0_-38": 1,
"0_-39": 1,
"10_-11": 1,
"10_-12": 1,
"10_-13": 1,
"10_-14": 1,
"10_-15": 1,
"10_-16": 1,
"10_-17": 1,
"10_-18": 1,
"10_-19": 1,
"10_-20": 1,
"10_-21": 1,
"10_-22": 1,
"10_-23": 1,
"10_-24": 1,
"10_-25": 1,
"10_-26": 1,
"10_-27": 1,
"10_-28": 1,
"10_-29": 1,
"10_-30": 1,
"10_-31": 1,
"10_-32": 1,
"10_-33": 1,
"10_-34": 1,
"10_-35": 1,
"10_-36": 1,
"10_-37": 1,
"10_-38": 1,
"10_-39": 1,
"10_-40": 1,
"10_-41": 1,
"10_-42": 1,
"10_-43": 1,
"10_-44": 1,
"10_-45": 1,
"10_-46": 1,
"10_-47": 1,
"10_-48": 1,
"10_-49": 1,
"11_-11": 1,
"11_-12": 1,
"11_-13": 1,
"11_-14": 1,
"11_-15": 1,
"11_-16": 1,
"11_-17": 1,
"11_-18": 1,
"11_-19": 1,
"11_-20": 1,
"11_-21": 1,
"11_-22": 1,
"11_-23": 1,
"11_-24": 1,
"11_-25": 1,
"11_-26": 1,
"11_-27": 1,
"11_-28": 1,
"11_-29": 1,
"11_-30": 1,
"11_-31": 1,
"11_-32": 1,
"11_-33": 1,
"11_-34": 1,
"11_-35": 1,
"11_-36": 1,
"11_-37": 1,
"11_-38": 1,
"11_-39": 1,
"11_-40": 1,
"11_-41": 1,
"11_-42": 1,
"11_-43": 1,
"11_-44": 1,
"11_-45": 1,
"11_-46": 1,
"11_-47": 1,
"11_-48": 1,
"11_-49": 1,
"12_-11": 1,
"12_-12": 1,
"12_-13": 1,
"12_-14": 1,
"12_-15": 1,
"12_-16": 1,
"12_-17": 1,
"12_-18": 1,
"12_-19": 1,
"12_-20": 1,
"12_-21": 1,
"12_-22": 1,
"12_-23": 1,
"12_-24": 1,
"12_-25": 1,
"12_-26": 1,
"12_-27": 1,
"12_-28": 1,
"12_-29": 1,
"12_-30": 1,
"12_-31": 1,
"12_-32": 1,
"12_-33": 1,
"12_-34": 1,
"12_-35": 1,
"12_-36": 1,
"12_-37": 1,
"12_-38": 1,
"12_-39": 1,
"12_-40": 1,
"12_-41": 1,
"12_-42": 1,
"12_-43": 1,
"12_-44": 1,
"12_-45": 1,
"12_-46": 1,
"12_-47": 1,
"12_-48": 1,
"12_-49": 1,
"13_-12": 1,
"13_-13": 1,
"13_-14": 1,
"13_-15": 1,
"13_-16": 1,
"13_-17": 1,
"13_-18": 1,
"13_-19": 1,
"13_-20": 1,
"13_-21": 1,
"13_-22": 1,
"13_-23": 1,
"13_-24": 1,
"13_-25": 1,
"13_-26": 1,
"13_-27": 1,
"13_-28": 1,
"13_-29": 1,
"13_-30": 1,
"13_-31": 1,
"13_-32": 1,
"13_-33": 1,
"13_-34": 1,
"13_-35": 1,
"13_-36": 1,
"13_-37": 1,
"13_-38": 1,
"13_-39": 1,
"13_-40": 1,
"13_-41": 1,
"13_-42": 1,
"13_-43": 1,
"13_-44": 1,
"13_-45": 1,
"13_-46": 1,
"13_-47": 1,
"13_-48": 1,
"13_-49": 1,
"14_-12": 1,
"14_-13": 1,
"14_-14": 1,
"14_-15": 1,
"14_-16": 1,
"14_-17": 1,
"14_-18": 1,
"14_-19": 1,
"14_-20": 1,
"14_-21": 1,
"14_-22": 1,
"14_-23": 1,
"14_-24": 1,
"14_-25": 1,
"14_-26": 1,
"14_-27": 1,
"14_-28": 1,
"14_-29": 1,
"14_-30": 1,
"14_-31": 1,
"14_-32": 1,
"14_-33": 1,
"14_-34": 1,
"14_-35": 1,
"14_-36": 1,
"14_-37": 1,
"14_-38": 1,
"14_-39": 1,
"14_-40": 1,
"14_-41": 1,
"14_-42": 1,
"14_-43": 1,
"14_-44": 1,
"14_-45": 1,
"14_-46": 1,
"14_-47": 1,
"14_-48": 1,
"14_-49": 1,
"14_-50": 1,
"15_-12": 1,
"15_-13": 1,
"15_-14": 1,
"15_-15": 1,
"15_-16": 1,
"15_-17": 1,
"15_-18": 1,
"15_-19": 1,
"15_-20": 1,
"15_-21": 1,
"15_-22": 1,
"15_-23": 1,
"15_-24": 1,
"15_-25": 1,
"15_-26": 1,
"15_-27": 1,
"15_-28": 1,
"15_-29": 1,
"15_-30": 1,
"15_-31": 1,
"15_-32": 1,
"15_-33": 1,
"15_-34": 1,
"15_-35": 1,
"15_-36": 1,
"15_-37": 1,
"15_-38": 1,
"15_-39": 1,
"15_-40": 1,
"15_-41": 1,
"15_-42": 1,
"15_-43": 1,
"15_-44": 1,
"15_-45": 1,
"15_-46": 1,
"15_-47": 1,
"15_-48": 1,
"15_-49": 1,
"16_-12": 1,
"16_-13": 1,
"16_-14": 1,
"16_-15": 1,
"16_-16": 1,
"16_-17": 1,
"16_-18": 1,
"16_-19": 1,
"16_-20": 1,
"16_-21": 1,
"16_-22": 1,
"16_-23": 1,
"16_-24": 1,
"16_-25": 1,
"16_-26": 1,
"16_-27": 1,
"16_-28": 1,
"16_-29": 1,
"16_-30": 1,
"16_-31": 1,
"16_-32": 1,
"16_-33": 1,
"16_-34": 1,
"16_-35": 1,
"16_-36": 1,
"16_-37": 1,
"16_-38": 1,
"16_-39": 1,
"16_-40": 1,
"16_-41": 1,
"16_-42": 1,
"16_-43": 1,
"16_-44": 1,
"16_-45": 1,
"16_-46": 1,
"17_-12": 1,
"17_-13": 1,
"17_-16": 1,
"17_-17": 1,
"17_-18": 1,
"17_-19": 1,
"17_-20": 1,
"17_-21": 1,
"17_-22": 1,
"17_-23": 1,
"17_-24": 1,
"17_-25": 1,
"17_-26": 1,
"17_-27": 1,
"17_-28": 1,
"17_-29": 1,
"17_-30": 1,
"17_-31": 1,
"17_-32": 1,
"17_-33": 1,
"17_-34": 1,
"17_-35": 1,
"17_-36": 1,
"17_-37": 1,
"17_-38": 1,
"17_-39": 1,
"17_-40": 1,
"17_-41": 1,
"17_-42": 1,
"17_-43": 1,
"17_-44": 1,
"18_-12": 1,
"18_-13": 1,
"18_-18": 1,
"18_-19": 1,
"18_-20": 1,
"18_-21": 1,
"18_-22": 1,
"18_-23": 1,
"18_-24": 1,
"18_-25": 1,
"18_-26": 1,
"18_-27": 1,
"18_-28": 1,
"18_-29": 1,
"18_-30": 1,
"18_-31": 1,
"18_-32": 1,
"18_-33": 1,
"18_-34": 1,
"18_-35": 1,
"18_-36": 1,
"18_-37": 1,
"18_-38": 1,
"18_-39": 1,
"18_-40": 1,
"18_-41": 1,
"18_-42": 1,
"19_-12": 1,
"19_-13": 1,
"19_-14": 1,
"19_-15": 1,
"19_-18": 1,
"19_-19": 1,
"19_-20": 1,
"19_-21": 1,
"19_-22": 1,
"19_-23": 1,
"19_-24": 1,
"19_-25": 1,
"19_-26": 1,
"19_-27": 1,
"19_-28": 1,
"19_-29": 1,
"19_-30": 1,
"19_-31": 1,
"19_-32": 1,
"19_-33": 1,
"19_-34": 1,
"19_-35": 1,
"19_-36": 1,
"19_-37": 1,
"19_-38": 1,
"19_-39": 1,
"19_-40": 1,
"1_-11": 1,
"1_-12": 1,
"1_-13": 1,
"1_-14": 1,
"1_-15": 1,
"1_-16": 1,
"1_-17": 1,
"1_-18": 1,
"1_-19": 1,
"1_-20": 1,
"1_-21": 1,
"1_-22": 1,
"1_-23": 1,
"1_-24": 1,
"1_-25": 1,
"1_-26": 1,
"1_-27": 1,
"1_-28": 1,
"1_-29": 1,
"1_-30": 1,
"1_-31": 1,
"1_-32": 1,
"1_-33": 1,
"1_-34": 1,
"1_-35": 1,
"1_-36": 1,
"1_-37": 1,
"1_-38": 1,
"1_-39": 1,
"1_-40": 1,
"1_-41": 1,
"20_-12": 1,
"20_-13": 1,
"20_-14": 1,
"20_-15": 1,
"20_-18": 1,
"20_-19": 1,
"20_-20": 1,
"20_-21": 1,
"20_-22": 1,
"20_-23": 1,
"20_-24": 1,
"20_-25": 1,
"20_-26": 1,
"20_-27": 1,
"20_-28": 1,
"20_-29": 1,
"20_-30": 1,
"20_-31": 1,
"20_-32": 1,
"20_-33": 1,
"20_-34": 1,
"20_-35": 1,
"20_-36": 1,
"20_-37": 1,
"20_-38": 1,
"20_-39": 1,
"20_-40": 1,
"21_-13": 1,
"21_-14": 1,
"21_-15": 1,
"21_-20": 1,
"21_-21": 1,
"21_-22": 1,
"21_-23": 1,
"21_-24": 1,
"21_-25": 1,
"21_-26": 1,
"21_-27": 1,
"21_-28": 1,
"21_-29": 1,
"21_-30": 1,
"21_-31": 1,
"21_-32": 1,
"21_-33": 1,
"21_-34": 1,
"22_-13": 1,
"22_-14": 1,
"22_-15": 1,
"22_-16": 1,
"22_-20": 1,
"22_-21": 1,
"22_-22": 1,
"22_-23": 1,
"22_-25": 1,
"22_-26": 1,
"22_-27": 1,
"22_-28": 1,
"22_-29": 1,
"23_-13": 1,
"23_-14": 1,
"23_-15": 1,
"23_-16": 1,
"23_-20": 1,
"24_-13": 1,
"24_-14": 1,
"24_-15": 1,
"24_-16": 1,
"2_-12": 1,
"2_-13": 1,
"2_-14": 1,
"2_-15": 1,
"2_-16": 1,
"2_-17": 1,
"2_-18": 1,
"2_-19": 1,
"2_-20": 1,
"2_-21": 1,
"2_-22": 1,
"2_-23": 1,
"2_-24": 1,
"2_-25": 1,
"2_-26": 1,
"2_-27": 1,
"2_-28": 1,
"2_-29": 1,
"2_-30": 1,
"2_-31": 1,
"2_-32": 1,
"2_-33": 1,
"2_-34": 1,
"2_-35": 1,
"2_-36": 1,
"2_-37": 1,
"2_-38": 1,
"2_-39": 1,
"2_-40": 1,
"2_-41": 1,
"3_-12": 1,
"3_-13": 1,
"3_-14": 1,
"3_-15": 1,
"3_-16": 1,
"3_-17": 1,
"3_-18": 1,
"3_-19": 1,
"3_-20": 1,
"3_-21": 1,
"3_-22": 1,
"3_-23": 1,
"3_-24": 1,
"3_-25": 1,
"3_-26": 1,
"3_-27": 1,
"3_-28": 1,
"3_-29": 1,
"3_-30": 1,
"3_-31": 1,
"3_-32": 1,
"3_-33": 1,
"3_-34": 1,
"3_-35": 1,
"3_-36": 1,
"3_-37": 1,
"3_-38": 1,
"3_-39": 1,
"3_-40": 1,
"3_-41": 1,
"4_-11": 1,
"4_-12": 1,
"4_-13": 1,
"4_-14": 1,
"4_-15": 1,
"4_-16": 1,
"4_-17": 1,
"4_-18": 1,
"4_-19": 1,
"4_-20": 1,
"4_-21": 1,
"4_-22": 1,
"4_-23": 1,
"4_-24": 1,
"4_-25": 1,
"4_-26": 1,
"4_-27": 1,
"4_-28": 1,
"4_-29": 1,
"4_-30": 1,
"4_-31": 1,
"4_-32": 1,
"4_-33": 1,
"4_-34": 1,
"4_-35": 1,
"4_-36": 1,
"4_-37": 1,
"4_-38": 1,
"4_-39": 1,
"4_-40": 1,
"4_-41": 1,
"5_-11": 1,
"5_-12": 1,
"5_-13": 1,
"5_-14": 1,
"5_-15": 1,
"5_-16": 1,
"5_-17": 1,
"5_-18": 1,
"5_-19": 1,
"5_-20": 1,
"5_-21": 1,
"5_-22": 1,
"5_-23": 1,
"5_-24": 1,
"5_-25": 1,
"5_-26": 1,
"5_-27": 1,
"5_-28": 1,
"5_-29": 1,
"5_-30": 1,
"5_-31": 1,
"5_-32": 1,
"5_-33": 1,
"5_-34": 1,
"5_-35": 1,
"5_-36": 1,
"5_-37": 1,
"5_-38": 1,
"5_-39": 1,
"5_-40": 1,
"5_-41": 1,
"5_-42": 1,
"6_-12": 1,
"6_-13": 1,
"6_-14": 1,
"6_-15": 1,
"6_-16": 1,
"6_-17": 1,
"6_-18": 1,
"6_-19": 1,
"6_-20": 1,
"6_-21": 1,
"6_-22": 1,
"6_-23": 1,
"6_-24": 1,
"6_-25": 1,
"6_-26": 1,
"6_-27": 1,
"6_-28": 1,
"6_-29": 1,
"6_-30": 1,
"6_-31": 1,
"6_-32": 1,
"6_-33": 1,
"6_-34": 1,
"6_-35": 1,
"6_-36": 1,
"6_-37": 1,
"6_-38": 1,
"6_-39": 1,
"6_-40": 1,
"6_-41": 1,
"6_-42": 1,
"6_-43": 1,
"6_-44": 1,
"7_-15": 1,
"7_-16": 1,
"7_-17": 1,
"7_-18": 1,
"7_-19": 1,
"7_-20": 1,
"7_-21": 1,
"7_-22": 1,
"7_-23": 1,
"7_-24": 1,
"7_-25": 1,
"7_-26": 1,
"7_-27": 1,
"7_-28": 1,
"7_-29": 1,
"7_-30": 1,
"7_-31": 1,
"7_-32": 1,
"7_-33": 1,
"7_-34": 1,
"7_-35": 1,
"7_-36": 1,
"7_-37": 1,
"7_-38": 1,
"7_-39": 1,
"7_-40": 1,
"7_-41": 1,
"7_-42": 1,
"7_-43": 1,
"7_-44": 1,
"8_-14": 1,
"8_-15": 1,
"8_-16": 1,
"8_-17": 1,
"8_-18": 1,
"8_-19": 1,
"8_-20": 1,
"8_-21": 1,
"8_-22": 1,
"8_-23": 1,
"8_-24": 1,
"8_-25": 1,
"8_-26": 1,
"8_-27": 1,
"8_-28": 1,
"8_-29": 1,
"8_-30": 1,
"8_-31": 1,
"8_-32": 1,
"8_-33": 1,
"8_-34": 1,
"8_-35": 1,
"8_-36": 1,
"8_-37": 1,
"8_-38": 1,
"8_-39": 1,
"8_-40": 1,
"8_-41": 1,
"8_-42": 1,
"8_-43": 1,
"8_-44": 1,
"8_-45": 1,
"8_-46": 1,
"8_-48": 1,
"8_-49": 1,
"9_-11": 1,
"9_-12": 1,
"9_-13": 1,
"9_-14": 1,
"9_-15": 1,
"9_-16": 1,
"9_-17": 1,
"9_-18": 1,
"9_-19": 1,
"9_-20": 1,
"9_-21": 1,
"9_-22": 1,
"9_-23": 1,
"9_-24": 1,
"9_-25": 1,
"9_-26": 1,
"9_-27": 1,
"9_-28": 1,
"9_-29": 1,
"9_-30": 1,
"9_-31": 1,
"9_-32": 1,
"9_-33": 1,
"9_-34": 1,
"9_-35": 1,
"9_-36": 1,
"9_-37": 1,
"9_-38": 1,
"9_-39": 1,
"9_-40": 1,
"9_-41": 1,
"9_-42": 1,
"9_-43": 1,
"9_-44": 1,
"9_-45": 1,
"9_-46": 1,
"9_-47": 1,
"9_-48": 1,
"9_-49": 1,
}
),
timeline_period="day",
time_range=Range(
begin=datetime(
1970, 1, 1, 0, 30, tzinfo=FixedOffsetTimezone(offset=600, name=None)
),
end=datetime(
1970, 2, 1, 0, 30, tzinfo=FixedOffsetTimezone(offset=600, name=None)
),
),
footprint_geometry=(
shapely.wkt.load(
Path(__file__)
.parent.joinpath("data/wofs-summary-odd-footprint.wkt")
.open("r")
)
),
footprint_crs="EPSG:3577",
footprint_count=1244,
newest_dataset_creation_time=datetime(
def calculate_summary(self, product_name: str,
time: Range) -> TimePeriodOverview:
"""
Create a summary of the given product/time range.
"""
log = self.log.bind(product_name=product_name, time=time)
log.debug("summary.query")
begin_time, end_time, where_clause = self._where(product_name, time)
select_by_srid = (select((
func.ST_SRID(DATASET_SPATIAL.c.footprint).label("srid"),
func.count().label("dataset_count"),
func.ST_Transform(
func.ST_Union(DATASET_SPATIAL.c.footprint),
self._target_srid(),
type_=Geometry(),
).label("footprint_geometry"),
func.sum(DATASET_SPATIAL.c.size_bytes).label("size_bytes"),
func.max(DATASET_SPATIAL.c.creation_time).label(
"newest_dataset_creation_time"),
)).where(where_clause).group_by("srid").alias("srid_summaries"))
# Union all srid groups into one summary.
result = self._engine.execute(
select((
func.sum(
select_by_srid.c.dataset_count).label("dataset_count"),
func.array_agg(select_by_srid.c.srid).label("srids"),
func.sum(select_by_srid.c.size_bytes).label("size_bytes"),
func.ST_Union(
func.ST_Buffer(select_by_srid.c.footprint_geometry, 0),
type_=Geometry(srid=self._target_srid()),
).label("footprint_geometry"),
func.max(select_by_srid.c.newest_dataset_creation_time).label(
"newest_dataset_creation_time"),
func.now().label("summary_gen_time"),
)))
rows = result.fetchall()
log.debug("summary.query.done", srid_rows=len(rows))
assert len(rows) == 1
row = dict(rows[0])
row["dataset_count"] = int(
row["dataset_count"]) if row["dataset_count"] else 0
if row["footprint_geometry"] is not None:
row["footprint_crs"] = self._get_srid_name(
row["footprint_geometry"].srid)
row["footprint_geometry"] = geo_shape.to_shape(
row["footprint_geometry"])
else:
row["footprint_crs"] = None
row["crses"] = None
if row["srids"] is not None:
row["crses"] = {self._get_srid_name(s) for s in row["srids"]}
del row["srids"]
# Convert from Python Decimal
if row["size_bytes"] is not None:
row["size_bytes"] = int(row["size_bytes"])
has_data = row["dataset_count"] > 0
log.debug("counter.calc")
# Initialise all requested days as zero
day_counts = Counter({
d.date(): 0
for d in pd.date_range(begin_time, end_time, closed="left")
})
region_counts = Counter()
if has_data:
day_counts.update(
Counter({
day.date(): count
for day, count in self._engine.execute(
select([
func.date_trunc(
"day",
DATASET_SPATIAL.c.center_time.op(
"AT TIME ZONE")(self.grouping_time_zone),
).label("day"),
func.count(),
]).where(where_clause).group_by("day"))
}))
region_counts = Counter({
item: count
for item, count in self._engine.execute(
select([
DATASET_SPATIAL.c.region_code.label("region_code"),
func.count(),
]).where(where_clause).group_by("region_code"))
})
summary = TimePeriodOverview(
**row,
timeline_period="day",
time_range=Range(begin_time, end_time),
timeline_dataset_counts=day_counts,
region_dataset_counts=region_counts,
# TODO: filter invalid from the counts?
footprint_count=row["dataset_count"] or 0,
)
log.debug(
"summary.calc.done",
dataset_count=summary.dataset_count,
footprints_missing=summary.dataset_count - summary.footprint_count,
)
return summary
