def test_generate_incremental_archivals(run_generate,
summary_store: SummaryStore):
run_generate("ls8_nbar_scene")
index = summary_store.index
# When we have a summarised product...
original_summary = summary_store.get("ls8_nbar_scene")
original_dataset_count = original_summary.dataset_count
# ... and we archive one dataset ...
product_name = "ls8_nbar_scene"
dataset_id = _one_dataset(index, product_name)
try:
index.datasets.archive([dataset_id])
# ... the next generation should catch it and update with one less dataset....
run_generate("ls8_nbar_scene")
assert (summary_store.get(
"ls8_nbar_scene").dataset_count == original_dataset_count -
1), "Expected dataset count to decrease after archival"
finally:
# Now let's restore the dataset!
index.datasets.restore([dataset_id])
# It should be in the count again.
# (this change should work because the new 'updated' column will be bumped on restore)
run_generate("ls8_nbar_scene")
assert (
summary_store.get(
"ls8_nbar_scene").dataset_count == original_dataset_count
), "A dataset that was restored from archival was not refreshed by Explorer"
def test_calc_albers_summary_with_storage(summary_store: SummaryStore):
summary_store.refresh_all_products()
# Should not exist yet.
summary = summary_store.get(
'ls8_nbar_albers',
year=None,
month=None,
day=None,
)
assert summary is None
summary = summary_store.get(
'ls8_nbar_albers',
year=2017,
month=None,
day=None,
)
assert summary is None
# Calculate overall summary
summary = summary_store.get_or_update(
'ls8_nbar_albers',
year=2017,
month=None,
day=None,
)
_expect_values(
summary,
dataset_count=918,
footprint_count=918,
time_range=Range(begin=datetime(2017, 4, 1, 0, 0, tzinfo=DEFAULT_TZ),
end=datetime(2017, 6, 1, 0, 0, tzinfo=DEFAULT_TZ)),
newest_creation_time=datetime(2017,
10,
25,
23,
9,
2,
486851,
tzinfo=tzutc()),
timeline_period='day',
# Data spans 61 days in 2017
timeline_count=61,
crses={'EPSG:3577'},
# Ingested tiles don't store their size.
# TODO: probably should represent this as None instead of zero?
size_bytes=0)
# get_or_update should now return the cached copy.
cached_s = summary_store.get_or_update(
'ls8_nbar_albers',
year=2017,
month=None,
day=None,
)
assert cached_s.summary_gen_time is not None
assert cached_s.summary_gen_time == summary.summary_gen_time, \
"A new, rather than cached, summary was returned"
assert cached_s.dataset_count == summary.dataset_count
def test_calc_albers_summary_with_storage(summary_store: SummaryStore):
# Should not exist yet.
summary = summary_store.get("ls8_nbar_albers",
year=None,
month=None,
day=None)
assert summary is None
summary = summary_store.get("ls8_nbar_albers",
year=2017,
month=None,
day=None)
assert summary is None
# We don't want it to add a few minutes overlap buffer,
# as we add datasets and refresh immediately.
summary_store.dataset_overlap_carefulness = timedelta(seconds=0)
# Calculate overall summary
_, summary = summary_store.refresh("ls8_nbar_albers")
_expect_values(
summary,
dataset_count=918,
footprint_count=918,
time_range=Range(
begin=datetime(2017, 4, 1, 0, 0, tzinfo=DEFAULT_TZ),
end=datetime(2017, 6, 1, 0, 0, tzinfo=DEFAULT_TZ),
),
newest_creation_time=datetime(2017,
10,
25,
23,
9,
2,
486_851,
tzinfo=tzutc()),
timeline_period="day",
# Data spans 61 days in 2017
timeline_count=61,
crses={"EPSG:3577"},
# Ingested tiles don't store their size.
# TODO: probably should represent this as None instead of zero?
size_bytes=0,
)
original = summary_store.get("ls8_nbar_albers", 2017)
# It should now return the same copy, not rebuild it.
summary_store.refresh("ls8_nbar_albers")
cached_s = summary_store.get("ls8_nbar_albers", 2017)
assert original is not cached_s
assert cached_s.dataset_count == original.dataset_count
assert cached_s.summary_gen_time is not None
assert (cached_s.summary_gen_time == original.summary_gen_time
), "A new, rather than cached, summary was returned"
def test_put_get_summaries(summary_store: SummaryStore):
"""
Test the serialisation/deserialisation from postgres
"""
o = _overview()
assert o.summary_gen_time is None, "Generation time should be set by server"
product_name = "some_product"
summary_store._set_product_extent(
ProductSummary(
product_name, 4321, datetime(2017, 1, 1), datetime(2017, 4, 1), [], [], {}
)
)
summary_store._put(product_name, 2017, None, None, o)
loaded = summary_store.get(product_name, 2017, None, None)
assert o is not loaded, (
"Store should not return the original objects " "(they may change)"
)
assert (
o.summary_gen_time is not None
), "Summary-gen-time should have been added by the server"
original_gen_time = o.summary_gen_time
assert o.footprint_geometry.area == pytest.approx(4.857_924_619_872)
assert loaded.dataset_count == 4
assert (
sum(loaded.region_dataset_counts.values()) == 4
), "Region dataset counts don't match total count"
assert sorted(loaded.region_dataset_counts.keys()) == [
"1_2",
"3_4",
"4_5",
], "Incorrect set of regions"
assert o.footprint_crs == loaded.footprint_crs
assert loaded.footprint_crs == "EPSG:3577"
assert loaded.footprint_srid == 3577
assert loaded.footprint_geometry.area == pytest.approx(o.footprint_geometry.area)
o.dataset_count = 4321
o.newest_dataset_creation_time = datetime(2018, 2, 2, 2, 2, 2, tzinfo=tz.tzutc())
time.sleep(1)
summary_store._put(product_name, 2017, None, None, o)
assert o.summary_gen_time != original_gen_time
loaded = summary_store.get(product_name, 2017, None, None)
assert loaded.dataset_count == 4321
assert loaded.newest_dataset_creation_time == datetime(
2018, 2, 2, 2, 2, 2, tzinfo=tz.tzutc()
)
assert (
loaded.summary_gen_time != original_gen_time
), "An update should update the generation time"
def test_generate_empty_time(run_generate, summary_store: SummaryStore):
run_generate("ls8_nbar_albers")
# No datasets in 2018
assert (summary_store.get("ls8_nbar_albers", year=2018) is
None), "There should be no datasets in 2018"
# Year that does not exist for LS8
summary = summary_store.get("ls8_nbar_albers",
year=2006,
month=None,
day=None)
assert summary is None
def test_add_no_periods(summary_store: SummaryStore):
"""
All the get/update methods should work on products with no datasets.
"""
result, summary = summary_store.refresh("ga_ls8c_level1_3")
assert result == GenerateResult.CREATED
assert summary.dataset_count == 0
assert summary_store.get("ga_ls8c_level1_3", 2015, 7, 4).dataset_count == 0
result, summary = summary_store.refresh("ga_ls8c_level1_3")
assert result == GenerateResult.NO_CHANGES
assert summary.dataset_count == 0
assert summary_store.get("ga_ls8c_level1_3").dataset_count == 0
assert summary_store.get("ga_ls8c_level1_3", 2015, 7, None) is None
def test_generate_scene_all_time(run_generate, summary_store: SummaryStore):
run_generate('ls8_nbar_scene')
# All time
_expect_values(
summary_store.get(
'ls8_nbar_scene',
year=None,
month=None,
day=None,
),
dataset_count=3036,
footprint_count=3036,
time_range=Range(begin=datetime(2016, 1, 1, 0, 0, tzinfo=DEFAULT_TZ),
end=datetime(2018, 1, 1, 0, 0, tzinfo=DEFAULT_TZ)),
newest_creation_time=datetime(2018, 1, 10, 3, 11, 56, tzinfo=tzutc()),
timeline_period='month',
timeline_count=24,
crses={
'EPSG:28355', 'EPSG:28349', 'EPSG:28352', 'EPSG:28357',
'EPSG:28350', 'EPSG:28351', 'EPSG:28353', 'EPSG:28356',
'EPSG:28354'
},
size_bytes=1805759242975,
)
def test_generate_month(run_generate, summary_store: SummaryStore):
run_generate("ls8_nbar_scene")
# One Month
_expect_values(
summary_store.get("ls8_nbar_scene", 2017, 4, None),
dataset_count=408,
footprint_count=408,
time_range=Range(
begin=datetime(2017, 4, 1, 0, 0, tzinfo=DEFAULT_TZ),
end=datetime(2017, 5, 1, 0, 0, tzinfo=DEFAULT_TZ),
),
newest_creation_time=datetime(2017, 7, 4, 11, 18, 20, tzinfo=tzutc()),
timeline_period="day",
timeline_count=30,
crses={
"EPSG:28355",
"EPSG:28349",
"EPSG:28352",
"EPSG:28350",
"EPSG:28351",
"EPSG:28353",
"EPSG:28356",
"EPSG:28354",
},
size_bytes=245_344_352_585,
)
def test_generate_scene_all_time(run_generate, summary_store: SummaryStore):
run_generate("ls8_nbar_scene")
# All time
_expect_values(
summary_store.get("ls8_nbar_scene", year=None, month=None, day=None),
dataset_count=3036,
footprint_count=3036,
time_range=Range(
begin=datetime(2016, 1, 1, 0, 0, tzinfo=DEFAULT_TZ),
end=datetime(2018, 1, 1, 0, 0, tzinfo=DEFAULT_TZ),
),
newest_creation_time=datetime(2018, 1, 10, 3, 11, 56, tzinfo=tzutc()),
timeline_period="month",
timeline_count=24,
crses={
"EPSG:28355",
"EPSG:28349",
"EPSG:28352",
"EPSG:28357",
"EPSG:28350",
"EPSG:28351",
"EPSG:28353",
"EPSG:28356",
"EPSG:28354",
},
size_bytes=1_805_759_242_975,
)
def test_generate_scene_year(run_generate, summary_store: SummaryStore):
run_generate()
# One year
_expect_values(
summary_store.get("ls8_nbar_scene", year=2017, month=None, day=None),
dataset_count=1792,
footprint_count=1792,
time_range=Range(
begin=datetime(2017, 1, 1, 0, 0, tzinfo=DEFAULT_TZ),
end=datetime(2018, 1, 1, 0, 0, tzinfo=DEFAULT_TZ),
),
newest_creation_time=datetime(2018, 1, 10, 3, 11, 56, tzinfo=tzutc()),
timeline_period="day",
timeline_count=365,
crses={
"EPSG:28355",
"EPSG:28349",
"EPSG:28352",
"EPSG:28350",
"EPSG:28351",
"EPSG:28353",
"EPSG:28356",
"EPSG:28354",
},
size_bytes=1_060_669_242_142,
)
def test_dataset_changing_product(run_generate, summary_store: SummaryStore):
"""
If a dataset it updated to be in a different product, Explorer should
correctly update its summaries.
(this really happened at NCI previously)
This is a trickier case than regular updates because everything in Explorer
is product-specific. Summarising one product at a time, etc.
"""
run_generate("ls8_nbar_scene")
index = summary_store.index
dataset_id = _one_dataset(index, "ls8_nbar_scene")
our_product = index.products.get_by_name("ls8_nbar_scene")
other_product = index.products.get_by_name("ls8_nbar_albers")
# When we have a summarised product...
original_summary = summary_store.get("ls8_nbar_scene")
original_dataset_count = original_summary.dataset_count
try:
# Move the dataset to another product
_change_dataset_product(index, dataset_id, other_product)
assert index.datasets.get(dataset_id).type.name == "ls8_nbar_albers"
# Explorer should remove it too.
print(f"Test dataset: {dataset_id}")
# TODO: Make this work without a force-refresh.
# It's hard because we're scanning for updated datasets in the product...
# but it's not in the product. And the incremental updater misses it.
# So we have to force the non-incremental updater.
run_generate("ls8_nbar_albers", "ls8_nbar_scene", "--force-refresh")
assert (summary_store.get(
"ls8_nbar_scene").dataset_count == original_dataset_count -
1), "Expected dataset to be removed after product change"
finally:
# Now change it back
_change_dataset_product(index, dataset_id, our_product)
run_generate("ls8_nbar_albers", "ls8_nbar_scene", "--force-refresh")
assert (
summary_store.get(
"ls8_nbar_scene").dataset_count == original_dataset_count
), "Expected dataset to be added again after the product changed back"
def test_get_null(summary_store: SummaryStore):
"""
An area with nothing generated should come back as null.
(It's important for us to distinguish between an area with zero datasets
and an area where the summary/extent has not been generated.)
"""
loaded = summary_store.get("some_product", 2019, 4, None)
assert loaded is None
def test_calc_empty(summary_store: SummaryStore):
summary_store.refresh_all_products()
# Should not exist.
summary = summary_store.get("ls8_fake_product",
year=2006,
month=None,
day=None)
assert summary is None
def test_generate_telemetry(run_generate, summary_store: SummaryStore):
"""
Telemetry data polygons can be synthesized from the path/row values
"""
run_generate("ls8_satellite_telemetry_data")
_expect_values(
summary_store.get("ls8_satellite_telemetry_data"),
dataset_count=1199,
footprint_count=1199,
time_range=Range(
begin=datetime(2016, 1, 1, 0, 0, tzinfo=DEFAULT_TZ),
end=datetime(2018, 1, 1, 0, 0, tzinfo=DEFAULT_TZ),
),
region_dataset_counts={
"91": 56,
"92": 56,
"93": 56,
"90": 51,
"95": 47,
"94": 45,
"96": 44,
"101": 43,
"98": 43,
"100": 42,
"105": 42,
"111": 42,
"99": 42,
"104": 41,
"110": 41,
"112": 41,
"103": 40,
"107": 40,
"108": 40,
"109": 40,
"89": 40,
"97": 40,
"113": 39,
"102": 37,
"106": 36,
"114": 32,
"116": 29,
"115": 27,
"88": 27,
},
newest_creation_time=datetime(2017, 12, 31, 3, 38, 43, tzinfo=tzutc()),
timeline_period="month",
timeline_count=24,
crses={"EPSG:4326"},
size_bytes=10333203380934,
)
def test_s2_l2a_summary(run_generate, summary_store: SummaryStore):
run_generate("s2_l2a")
expect_values(
summary_store.get("s2_l2a"),
dataset_count=4,
footprint_count=4,
time_range=Range(
begin=datetime(2016, 10, 31, 14, 30, tzinfo=tzutc()),
end=datetime(2019, 6, 30, 14, 30, tzinfo=tzutc()),
),
newest_creation_time=datetime(2019, 6, 20, 11, 57, 34, tzinfo=tzutc()),
timeline_period="day",
timeline_count=91,
crses={"EPSG:32632", "EPSG:32630", "EPSG:32627"},
size_bytes=0,
)
def test_s2a_l1_summary(run_generate, summary_store: SummaryStore):
run_generate("s2a_level1c_granule")
expect_values(
summary_store.get("s2a_level1c_granule"),
dataset_count=8,
footprint_count=8,
time_range=Range(
begin=datetime(2017, 9, 30, 14, 30, tzinfo=tzutc()),
end=datetime(2017, 10, 31, 14, 30, tzinfo=tzutc()),
),
newest_creation_time=datetime(2017, 10, 23, 1, 13, 7, tzinfo=tzutc()),
timeline_period="day",
timeline_count=31,
crses={"EPSG:32753"},
size_bytes=3_442_177_050,
)
def test_generate_empty_time(run_generate, summary_store: SummaryStore):
run_generate("ls8_nbar_albers")
# No datasets in 2018
summary = summary_store.get_or_update("ls8_nbar_albers",
year=2018,
month=None,
day=None)
assert summary.dataset_count == 0, "There should be no datasets in 2018"
# assert len(summary.timeline_dataset_counts) == 365, "Empty regions should still show up in timeline histogram"
# Year that does not exist for LS8
summary = summary_store.get("ls8_nbar_albers",
year=2006,
month=None,
day=None)
assert summary is None
def test_generate_scene_year(run_generate, summary_store: SummaryStore):
run_generate()
# One year
_expect_values(
summary_store.get(
'ls8_nbar_scene',
year=2017,
month=None,
day=None,
),
dataset_count=1792,
footprint_count=1792,
time_range=Range(begin=datetime(2017, 1, 1, 0, 0, tzinfo=DEFAULT_TZ),
end=datetime(2018, 1, 1, 0, 0, tzinfo=DEFAULT_TZ)),
newest_creation_time=datetime(2018, 1, 10, 3, 11, 56, tzinfo=tzutc()),
timeline_period='day',
timeline_count=365,
crses={
'EPSG:28355', 'EPSG:28349', 'EPSG:28352', 'EPSG:28350',
'EPSG:28351', 'EPSG:28353', 'EPSG:28356', 'EPSG:28354'
},
size_bytes=1060669242142,
)
def test_generate_day(run_generate, summary_store: SummaryStore):
run_generate("ls8_nbar_albers")
_expect_values(
summary_store.get("ls8_nbar_albers", year=2017, month=5, day=2),
dataset_count=29,
footprint_count=29,
time_range=Range(
begin=datetime(2017, 5, 2, 0, 0, tzinfo=DEFAULT_TZ),
end=datetime(2017, 5, 3, 0, 0, tzinfo=DEFAULT_TZ),
),
newest_creation_time=datetime(2017,
10,
20,
8,
53,
26,
475_609,
tzinfo=tzutc()),
timeline_period="day",
timeline_count=1,
crses={"EPSG:3577"},
size_bytes=None,
)
