def test_generate_incremental_archivals(run_generate, summary_store: SummaryStore): run_generate("ls8_nbar_scene") index = summary_store.index # When we have a summarised product... original_summary = summary_store.get("ls8_nbar_scene") original_dataset_count = original_summary.dataset_count # ... and we archive one dataset ... product_name = "ls8_nbar_scene" dataset_id = _one_dataset(index, product_name) try: index.datasets.archive([dataset_id]) # ... the next generation should catch it and update with one less dataset.... run_generate("ls8_nbar_scene") assert (summary_store.get( "ls8_nbar_scene").dataset_count == original_dataset_count - 1), "Expected dataset count to decrease after archival" finally: # Now let's restore the dataset! index.datasets.restore([dataset_id]) # It should be in the count again. # (this change should work because the new 'updated' column will be bumped on restore) run_generate("ls8_nbar_scene") assert ( summary_store.get( "ls8_nbar_scene").dataset_count == original_dataset_count ), "A dataset that was restored from archival was not refreshed by Explorer"
def test_calc_albers_summary_with_storage(summary_store: SummaryStore): summary_store.refresh_all_products() # Should not exist yet. summary = summary_store.get( 'ls8_nbar_albers', year=None, month=None, day=None, ) assert summary is None summary = summary_store.get( 'ls8_nbar_albers', year=2017, month=None, day=None, ) assert summary is None # Calculate overall summary summary = summary_store.get_or_update( 'ls8_nbar_albers', year=2017, month=None, day=None, ) _expect_values( summary, dataset_count=918, footprint_count=918, time_range=Range(begin=datetime(2017, 4, 1, 0, 0, tzinfo=DEFAULT_TZ), end=datetime(2017, 6, 1, 0, 0, tzinfo=DEFAULT_TZ)), newest_creation_time=datetime(2017, 10, 25, 23, 9, 2, 486851, tzinfo=tzutc()), timeline_period='day', # Data spans 61 days in 2017 timeline_count=61, crses={'EPSG:3577'}, # Ingested tiles don't store their size. # TODO: probably should represent this as None instead of zero? size_bytes=0) # get_or_update should now return the cached copy. cached_s = summary_store.get_or_update( 'ls8_nbar_albers', year=2017, month=None, day=None, ) assert cached_s.summary_gen_time is not None assert cached_s.summary_gen_time == summary.summary_gen_time, \ "A new, rather than cached, summary was returned" assert cached_s.dataset_count == summary.dataset_count
def test_calc_albers_summary_with_storage(summary_store: SummaryStore): # Should not exist yet. summary = summary_store.get("ls8_nbar_albers", year=None, month=None, day=None) assert summary is None summary = summary_store.get("ls8_nbar_albers", year=2017, month=None, day=None) assert summary is None # We don't want it to add a few minutes overlap buffer, # as we add datasets and refresh immediately. summary_store.dataset_overlap_carefulness = timedelta(seconds=0) # Calculate overall summary _, summary = summary_store.refresh("ls8_nbar_albers") _expect_values( summary, dataset_count=918, footprint_count=918, time_range=Range( begin=datetime(2017, 4, 1, 0, 0, tzinfo=DEFAULT_TZ), end=datetime(2017, 6, 1, 0, 0, tzinfo=DEFAULT_TZ), ), newest_creation_time=datetime(2017, 10, 25, 23, 9, 2, 486_851, tzinfo=tzutc()), timeline_period="day", # Data spans 61 days in 2017 timeline_count=61, crses={"EPSG:3577"}, # Ingested tiles don't store their size. # TODO: probably should represent this as None instead of zero? size_bytes=0, ) original = summary_store.get("ls8_nbar_albers", 2017) # It should now return the same copy, not rebuild it. summary_store.refresh("ls8_nbar_albers") cached_s = summary_store.get("ls8_nbar_albers", 2017) assert original is not cached_s assert cached_s.dataset_count == original.dataset_count assert cached_s.summary_gen_time is not None assert (cached_s.summary_gen_time == original.summary_gen_time ), "A new, rather than cached, summary was returned"
def test_put_get_summaries(summary_store: SummaryStore): """ Test the serialisation/deserialisation from postgres """ o = _overview() assert o.summary_gen_time is None, "Generation time should be set by server" product_name = "some_product" summary_store._set_product_extent( ProductSummary( product_name, 4321, datetime(2017, 1, 1), datetime(2017, 4, 1), [], [], {} ) ) summary_store._put(product_name, 2017, None, None, o) loaded = summary_store.get(product_name, 2017, None, None) assert o is not loaded, ( "Store should not return the original objects " "(they may change)" ) assert ( o.summary_gen_time is not None ), "Summary-gen-time should have been added by the server" original_gen_time = o.summary_gen_time assert o.footprint_geometry.area == pytest.approx(4.857_924_619_872) assert loaded.dataset_count == 4 assert ( sum(loaded.region_dataset_counts.values()) == 4 ), "Region dataset counts don't match total count" assert sorted(loaded.region_dataset_counts.keys()) == [ "1_2", "3_4", "4_5", ], "Incorrect set of regions" assert o.footprint_crs == loaded.footprint_crs assert loaded.footprint_crs == "EPSG:3577" assert loaded.footprint_srid == 3577 assert loaded.footprint_geometry.area == pytest.approx(o.footprint_geometry.area) o.dataset_count = 4321 o.newest_dataset_creation_time = datetime(2018, 2, 2, 2, 2, 2, tzinfo=tz.tzutc()) time.sleep(1) summary_store._put(product_name, 2017, None, None, o) assert o.summary_gen_time != original_gen_time loaded = summary_store.get(product_name, 2017, None, None) assert loaded.dataset_count == 4321 assert loaded.newest_dataset_creation_time == datetime( 2018, 2, 2, 2, 2, 2, tzinfo=tz.tzutc() ) assert ( loaded.summary_gen_time != original_gen_time ), "An update should update the generation time"
def test_generate_empty_time(run_generate, summary_store: SummaryStore): run_generate("ls8_nbar_albers") # No datasets in 2018 assert (summary_store.get("ls8_nbar_albers", year=2018) is None), "There should be no datasets in 2018" # Year that does not exist for LS8 summary = summary_store.get("ls8_nbar_albers", year=2006, month=None, day=None) assert summary is None
def test_add_no_periods(summary_store: SummaryStore): """ All the get/update methods should work on products with no datasets. """ result, summary = summary_store.refresh("ga_ls8c_level1_3") assert result == GenerateResult.CREATED assert summary.dataset_count == 0 assert summary_store.get("ga_ls8c_level1_3", 2015, 7, 4).dataset_count == 0 result, summary = summary_store.refresh("ga_ls8c_level1_3") assert result == GenerateResult.NO_CHANGES assert summary.dataset_count == 0 assert summary_store.get("ga_ls8c_level1_3").dataset_count == 0 assert summary_store.get("ga_ls8c_level1_3", 2015, 7, None) is None
def test_generate_scene_all_time(run_generate, summary_store: SummaryStore): run_generate('ls8_nbar_scene') # All time _expect_values( summary_store.get( 'ls8_nbar_scene', year=None, month=None, day=None, ), dataset_count=3036, footprint_count=3036, time_range=Range(begin=datetime(2016, 1, 1, 0, 0, tzinfo=DEFAULT_TZ), end=datetime(2018, 1, 1, 0, 0, tzinfo=DEFAULT_TZ)), newest_creation_time=datetime(2018, 1, 10, 3, 11, 56, tzinfo=tzutc()), timeline_period='month', timeline_count=24, crses={ 'EPSG:28355', 'EPSG:28349', 'EPSG:28352', 'EPSG:28357', 'EPSG:28350', 'EPSG:28351', 'EPSG:28353', 'EPSG:28356', 'EPSG:28354' }, size_bytes=1805759242975, )
def test_generate_month(run_generate, summary_store: SummaryStore): run_generate("ls8_nbar_scene") # One Month _expect_values( summary_store.get("ls8_nbar_scene", 2017, 4, None), dataset_count=408, footprint_count=408, time_range=Range( begin=datetime(2017, 4, 1, 0, 0, tzinfo=DEFAULT_TZ), end=datetime(2017, 5, 1, 0, 0, tzinfo=DEFAULT_TZ), ), newest_creation_time=datetime(2017, 7, 4, 11, 18, 20, tzinfo=tzutc()), timeline_period="day", timeline_count=30, crses={ "EPSG:28355", "EPSG:28349", "EPSG:28352", "EPSG:28350", "EPSG:28351", "EPSG:28353", "EPSG:28356", "EPSG:28354", }, size_bytes=245_344_352_585, )
def test_generate_scene_all_time(run_generate, summary_store: SummaryStore): run_generate("ls8_nbar_scene") # All time _expect_values( summary_store.get("ls8_nbar_scene", year=None, month=None, day=None), dataset_count=3036, footprint_count=3036, time_range=Range( begin=datetime(2016, 1, 1, 0, 0, tzinfo=DEFAULT_TZ), end=datetime(2018, 1, 1, 0, 0, tzinfo=DEFAULT_TZ), ), newest_creation_time=datetime(2018, 1, 10, 3, 11, 56, tzinfo=tzutc()), timeline_period="month", timeline_count=24, crses={ "EPSG:28355", "EPSG:28349", "EPSG:28352", "EPSG:28357", "EPSG:28350", "EPSG:28351", "EPSG:28353", "EPSG:28356", "EPSG:28354", }, size_bytes=1_805_759_242_975, )
def test_generate_scene_year(run_generate, summary_store: SummaryStore): run_generate() # One year _expect_values( summary_store.get("ls8_nbar_scene", year=2017, month=None, day=None), dataset_count=1792, footprint_count=1792, time_range=Range( begin=datetime(2017, 1, 1, 0, 0, tzinfo=DEFAULT_TZ), end=datetime(2018, 1, 1, 0, 0, tzinfo=DEFAULT_TZ), ), newest_creation_time=datetime(2018, 1, 10, 3, 11, 56, tzinfo=tzutc()), timeline_period="day", timeline_count=365, crses={ "EPSG:28355", "EPSG:28349", "EPSG:28352", "EPSG:28350", "EPSG:28351", "EPSG:28353", "EPSG:28356", "EPSG:28354", }, size_bytes=1_060_669_242_142, )
def test_dataset_changing_product(run_generate, summary_store: SummaryStore): """ If a dataset it updated to be in a different product, Explorer should correctly update its summaries. (this really happened at NCI previously) This is a trickier case than regular updates because everything in Explorer is product-specific. Summarising one product at a time, etc. """ run_generate("ls8_nbar_scene") index = summary_store.index dataset_id = _one_dataset(index, "ls8_nbar_scene") our_product = index.products.get_by_name("ls8_nbar_scene") other_product = index.products.get_by_name("ls8_nbar_albers") # When we have a summarised product... original_summary = summary_store.get("ls8_nbar_scene") original_dataset_count = original_summary.dataset_count try: # Move the dataset to another product _change_dataset_product(index, dataset_id, other_product) assert index.datasets.get(dataset_id).type.name == "ls8_nbar_albers" # Explorer should remove it too. print(f"Test dataset: {dataset_id}") # TODO: Make this work without a force-refresh. # It's hard because we're scanning for updated datasets in the product... # but it's not in the product. And the incremental updater misses it. # So we have to force the non-incremental updater. run_generate("ls8_nbar_albers", "ls8_nbar_scene", "--force-refresh") assert (summary_store.get( "ls8_nbar_scene").dataset_count == original_dataset_count - 1), "Expected dataset to be removed after product change" finally: # Now change it back _change_dataset_product(index, dataset_id, our_product) run_generate("ls8_nbar_albers", "ls8_nbar_scene", "--force-refresh") assert ( summary_store.get( "ls8_nbar_scene").dataset_count == original_dataset_count ), "Expected dataset to be added again after the product changed back"
def test_get_null(summary_store: SummaryStore): """ An area with nothing generated should come back as null. (It's important for us to distinguish between an area with zero datasets and an area where the summary/extent has not been generated.) """ loaded = summary_store.get("some_product", 2019, 4, None) assert loaded is None
def test_calc_empty(summary_store: SummaryStore): summary_store.refresh_all_products() # Should not exist. summary = summary_store.get("ls8_fake_product", year=2006, month=None, day=None) assert summary is None
def test_generate_telemetry(run_generate, summary_store: SummaryStore): """ Telemetry data polygons can be synthesized from the path/row values """ run_generate("ls8_satellite_telemetry_data") _expect_values( summary_store.get("ls8_satellite_telemetry_data"), dataset_count=1199, footprint_count=1199, time_range=Range( begin=datetime(2016, 1, 1, 0, 0, tzinfo=DEFAULT_TZ), end=datetime(2018, 1, 1, 0, 0, tzinfo=DEFAULT_TZ), ), region_dataset_counts={ "91": 56, "92": 56, "93": 56, "90": 51, "95": 47, "94": 45, "96": 44, "101": 43, "98": 43, "100": 42, "105": 42, "111": 42, "99": 42, "104": 41, "110": 41, "112": 41, "103": 40, "107": 40, "108": 40, "109": 40, "89": 40, "97": 40, "113": 39, "102": 37, "106": 36, "114": 32, "116": 29, "115": 27, "88": 27, }, newest_creation_time=datetime(2017, 12, 31, 3, 38, 43, tzinfo=tzutc()), timeline_period="month", timeline_count=24, crses={"EPSG:4326"}, size_bytes=10333203380934, )
def test_s2_l2a_summary(run_generate, summary_store: SummaryStore): run_generate("s2_l2a") expect_values( summary_store.get("s2_l2a"), dataset_count=4, footprint_count=4, time_range=Range( begin=datetime(2016, 10, 31, 14, 30, tzinfo=tzutc()), end=datetime(2019, 6, 30, 14, 30, tzinfo=tzutc()), ), newest_creation_time=datetime(2019, 6, 20, 11, 57, 34, tzinfo=tzutc()), timeline_period="day", timeline_count=91, crses={"EPSG:32632", "EPSG:32630", "EPSG:32627"}, size_bytes=0, )
def test_s2a_l1_summary(run_generate, summary_store: SummaryStore): run_generate("s2a_level1c_granule") expect_values( summary_store.get("s2a_level1c_granule"), dataset_count=8, footprint_count=8, time_range=Range( begin=datetime(2017, 9, 30, 14, 30, tzinfo=tzutc()), end=datetime(2017, 10, 31, 14, 30, tzinfo=tzutc()), ), newest_creation_time=datetime(2017, 10, 23, 1, 13, 7, tzinfo=tzutc()), timeline_period="day", timeline_count=31, crses={"EPSG:32753"}, size_bytes=3_442_177_050, )
def test_generate_empty_time(run_generate, summary_store: SummaryStore): run_generate("ls8_nbar_albers") # No datasets in 2018 summary = summary_store.get_or_update("ls8_nbar_albers", year=2018, month=None, day=None) assert summary.dataset_count == 0, "There should be no datasets in 2018" # assert len(summary.timeline_dataset_counts) == 365, "Empty regions should still show up in timeline histogram" # Year that does not exist for LS8 summary = summary_store.get("ls8_nbar_albers", year=2006, month=None, day=None) assert summary is None
def test_generate_scene_year(run_generate, summary_store: SummaryStore): run_generate() # One year _expect_values( summary_store.get( 'ls8_nbar_scene', year=2017, month=None, day=None, ), dataset_count=1792, footprint_count=1792, time_range=Range(begin=datetime(2017, 1, 1, 0, 0, tzinfo=DEFAULT_TZ), end=datetime(2018, 1, 1, 0, 0, tzinfo=DEFAULT_TZ)), newest_creation_time=datetime(2018, 1, 10, 3, 11, 56, tzinfo=tzutc()), timeline_period='day', timeline_count=365, crses={ 'EPSG:28355', 'EPSG:28349', 'EPSG:28352', 'EPSG:28350', 'EPSG:28351', 'EPSG:28353', 'EPSG:28356', 'EPSG:28354' }, size_bytes=1060669242142, )
def test_generate_day(run_generate, summary_store: SummaryStore): run_generate("ls8_nbar_albers") _expect_values( summary_store.get("ls8_nbar_albers", year=2017, month=5, day=2), dataset_count=29, footprint_count=29, time_range=Range( begin=datetime(2017, 5, 2, 0, 0, tzinfo=DEFAULT_TZ), end=datetime(2017, 5, 3, 0, 0, tzinfo=DEFAULT_TZ), ), newest_creation_time=datetime(2017, 10, 20, 8, 53, 26, 475_609, tzinfo=tzutc()), timeline_period="day", timeline_count=1, crses={"EPSG:3577"}, size_bytes=None, )