def test_get_overview_date_selector(client: FlaskClient):
# [1] = year, [2] = month, [3] = day
# check when no year, month, day has been selected
html = get_html(client, "/s2_l2a")
menu = html.find("#product-headers .header-option")
assert len(menu[1].find(".option-menu ul li")) == 3
# check only year has been selected
html = get_html(client, "/s2_l2a/2016")
menu = html.find("#product-headers .header-option")
assert len(menu[1].find(".option-menu ul li")) == 3
assert len(menu[2].find(".option-menu ul li")) == 3
# check month has been selected
html = get_html(client, "/s2_l2a/2016/11")
menu = html.find("#product-headers .header-option")
assert len(menu[1].find(".option-menu ul li")) == 3
assert len(menu[2].find(".option-menu ul li")) == 3
assert len(menu[3].find(".option-menu ul li")) == 3
# checking when day is selected
html = get_html(client, "/s2_l2a/2016/11/9")
menu = html.find("#product-headers .header-option")
assert len(menu[1].find(".option-menu ul li")) == 3
assert len(menu[2].find(".option-menu ul li")) == 3
assert len(menu[3].find(".option-menu ul li")) == 3
def test_search_page(client: FlaskClient):
html = get_html(client, "/datasets/ls7_nbar_scene")
search_results = html.find(".search-result a")
assert len(search_results) == 4
html = get_html(client, "/datasets/ls7_nbar_scene/2017/05")
search_results = html.find(".search-result a")
assert len(search_results) == 3
def test_get_day_overviews(client: FlaskClient):
# Individual days are computed on-the-fly rather than from summaries, so can
# have their own issues.
# With a dataset
html = get_html(client, "/ls7_nbar_scene/2017/4/20")
check_dataset_count(html, 1)
assert "ls7_nbar_scene on 20th April 2017" in _h1_text(html)
# Empty day
html = get_html(client, "/ls7_nbar_scene/2017/4/22")
check_dataset_count(html, 0)
def test_empty_product_page(client: FlaskClient):
"""
A product page is displayable when summarised, but with 0 datasets.
"""
html = get_html(client, "/products/ls5_nbar_scene")
assert "No datasets" in one_element(html, ".header-stat-information").text
# ... yet a normal product doesn't show the message:
html = get_html(client, "/products/ls7_nbar_scene")
assert "No datasets" not in one_element(html,
".header-stat-information").text
assert "4 datasets" in one_element(html, ".header-stat-information").text
def test_product_audit(unpopulated_client: FlaskClient, run_generate):
run_generate()
client = unpopulated_client
res = get_html(client, "/product-audit/?timings")
# print(res.html)
largest_footprint_size = res.find(".footprint-size .search-result")
assert len(largest_footprint_size) == 2
largest_product_footprint = (largest_footprint_size[0].find(
".product-name", first=True).text)
largest_val = largest_footprint_size[0].find(".size-value",
first=True).text
# They're both the same :/
assert largest_product_footprint in ("s2a_ard_granule",
"s2a_level1c_granule")
assert largest_val == "181.6B"
assert len(res.find(".unavailable-metadata .search-result")) == 2
res: Response = client.get("/audit/day-query-times.txt")
plain_timing_results = res.data.decode("utf-8")
print(plain_timing_results)
assert '"s2a_ard_granule"\t8\t' in plain_timing_results
def test_product_audit(unpopulated_client: FlaskClient, run_generate):
run_generate()
client = unpopulated_client
res = get_html(client, '/product-audit/?timings')
# print(res.html)
largest_footprint_size = res.find('.footprint-size .search-result')
assert len(largest_footprint_size) == 2
largest_product_footprint = largest_footprint_size[0].find('.product-name',
first=True).text
largest_val = largest_footprint_size[0].find('.size-value',
first=True).text
# They're both the same :/
assert largest_product_footprint in ('s2a_ard_granule',
's2a_level1c_granule')
assert largest_val == '181.6B'
assert len(res.find('.unavailable-metadata .search-result')) == 2
res: Response = client.get('/product-audit/day-times.txt')
plain_timing_results = res.data.decode('utf-8')
print(plain_timing_results)
assert '"s2a_ard_granule"\t8\t' in plain_timing_results
def test_search_time_completion(client: FlaskClient):
# They only specified a begin time, so the end time should be filled in with the product extent.
html = get_html(client, "/datasets/ls7_nbar_scene?time-begin=1999-05-28")
assert one_element(html, "#search-time-before").attrs["value"] == "1999-05-28"
# One day after the product extent end (range is exclusive)
assert one_element(html, "#search-time-after").attrs["value"] == "2017-05-04"
search_results = html.find(".search-result a")
assert len(search_results) == 4
def test_storage_page(client: FlaskClient, populated_index: Index):
html: HTML = get_html(client, "/audit/storage")
assert html.find(".product-name", containing="wofs_albers")
product_count = len(list(populated_index.products.get_all()))
assert f"{product_count} products" in html.text
assert len(html.find(".data-table tbody tr")) == product_count
def test_invalid_footprint_wofs_summary_load(client: FlaskClient):
# This all-time overview has a valid footprint that becomes invalid
# when reprojected to wgs84 by shapely.
from .data_wofs_summary import wofs_time_summary
_model.STORE._put(wofs_time_summary)
html = get_html(client, "/wofs_summary")
check_dataset_count(html, 1244)
def test_uninitialised_overview(unpopulated_client: FlaskClient,
summary_store: SummaryStore):
# Populate one product, so they don't get the usage error message ("run cubedash generate")
# Then load an unpopulated product.
summary_store.get_or_update("ls7_nbar_albers")
html = get_html(unpopulated_client, "/ls7_nbar_scene/2017")
assert html.find(".coverage-region-count",
first=True).text == "0 unique scenes"
def test_search_time_completion(client: FlaskClient):
# They only specified a begin time, so the end time should be filled in with the product extent.
html = get_html(client, '/datasets/ls7_nbar_scene?time-begin=1999-05-28')
assert html.find('#search-time-before', first=True).attrs['value'] == '1999-05-28'
# One day after the product extent end (range is exclusive)
assert html.find('#search-time-after', first=True).attrs['value'] == '2017-05-04'
search_results = html.find('.search-result a')
assert len(search_results) == 4
def test_view_metadata_type(client: FlaskClient):
# Does it load without error?
html: HTML = get_html(client, "/metadata-type/eo")
assert "eo Metadata Type" in _text(html, "h2")
# Does the page list products using the type?
products_using_it = [t.text for t in html.find(".type-usage-item")]
assert "ls8_nbar_albers" in products_using_it
def test_no_data_pages(client: FlaskClient):
"""
Fetch products that exist but have no summaries generated.
(these should load with "empty" messages: not throw exceptions)
"""
html = get_html(client, "/ls8_nbar_albers/2017")
assert "No data: not yet generated" in html.text
assert "Unknown number of datasets" in html.text
html = get_html(client, "/ls8_nbar_albers/2017/5")
assert "No data: not yet generated" in html.text
assert "Unknown number of datasets" in html.text
# Days are generated on demand: it should query and see that there are no datasets.
html = get_html(client, "/ls8_nbar_albers/2017/5/2")
check_dataset_count(html, 0)
def test_get_overview(client: FlaskClient):
html = get_html(client, '/wofs_albers')
check_dataset_count(html, 11)
check_last_processed(html, '2018-05-20T11:25:35')
assert 'wofs_albers whole collection' in _h1_text(html)
check_area('61,...km2', html)
html = get_html(client, '/wofs_albers/2017')
check_dataset_count(html, 11)
check_last_processed(html, '2018-05-20T11:25:35')
assert 'wofs_albers across 2017' in _h1_text(html)
html = get_html(client, '/wofs_albers/2017/04')
check_dataset_count(html, 4)
check_last_processed(html, '2018-05-20T09:36:57')
assert 'wofs_albers across April 2017' in _h1_text(html)
check_area('30,...km2', html)
def test_uninitialised_search_page(empty_client: FlaskClient, summary_store: SummaryStore):
# Populate one product, so they don't get the usage error message ("run cubedash generate")
summary_store.refresh_product(summary_store.index.products.get_by_name('ls7_nbar_albers'))
summary_store.get_or_update('ls7_nbar_albers')
# Then load a completely uninitialised product.
html = get_html(empty_client, '/datasets/ls7_nbar_scene')
search_results = html.find('.search-result a')
assert len(search_results) == 4
def test_uninitialised_search_page(empty_client: FlaskClient,
summary_store: SummaryStore):
# Populate one product, so they don't get the usage error message ("run cubedash generate")
summary_store.refresh("ls7_nbar_albers")
# Then load a completely uninitialised product.
html = get_html(empty_client, "/datasets/ls7_nbar_scene")
search_results = html.find(".search-result a")
assert len(search_results) == 4
def test_view_metadata_type(client: FlaskClient):
# Does it load without error?
html: HTML = get_html(client, "/metadata-type/eo")
assert html.find("h2", first=True).text == "eo"
assert html.find(".sub-header", first=True).text == "metadata type of 44 products"
# Does the page list products using the type?
products_using_it = [t.text for t in html.find(".type-usage-item")]
assert "ls8_nbar_albers" in products_using_it
def test_get_overview(client: FlaskClient):
html = get_html(client, "/wofs_albers")
check_dataset_count(html, 11)
check_last_processed(html, "2018-05-20T11:25:35")
assert "wofs_albers whole collection" in _h1_text(html)
check_area("61,...km2", html)
html = get_html(client, "/wofs_albers/2017")
check_dataset_count(html, 11)
check_last_processed(html, "2018-05-20T11:25:35")
assert "wofs_albers across 2017" in _h1_text(html)
html = get_html(client, "/wofs_albers/2017/04")
check_dataset_count(html, 4)
check_last_processed(html, "2018-05-20T09:36:57")
assert "wofs_albers across April 2017" in _h1_text(html)
check_area("30,...km2", html)
def test_undisplayable_product(client: FlaskClient):
"""
Telemetry products have no footprint available at all.
"""
html = get_html(client, "/ls7_satellite_telemetry_data")
check_dataset_count(html, 4)
assert "36.6GiB" in html.find(".coverage-filesize", first=True).text
assert "(None displayable)" in html.text
assert "No CRSes defined" in html.text
def test_out_of_date_range(client: FlaskClient):
"""
We have generated summaries for this product, but the date is out of the product's date range.
"""
html = get_html(client, "/wofs_albers/2010")
# The common error here is to say "No data: not yet generated" rather than "0 datasets"
assert check_dataset_count(html, 0)
assert b"Historic Flood Mapping Water Observations from Space" in html.text
def test_sentry(sentry_client: FlaskClient):
"""Ensure Sentry Client gets initialized correctly
Args:
sentry_client (FlaskClient): Client for Flask app with Sentry enabled
"""
html: HTML = get_html(sentry_client, "/ls7_nbar_scene")
# Ensure rendered page has a SENTRY link
assert "raven.min.js" in str(_script(html))
def test_loading_high_low_tide_comp(client: FlaskClient):
html = get_html(client, "/high_tide_comp_20p/2008")
assert html.search('High Tide 20 percentage composites for entire coastline') is not None
check_dataset_count(html, 306)
# Footprint is not exact due to shapely.simplify()
check_area('2,984,...km2', html)
assert html.find('.last-processed time', first=True).attrs['datetime'] == '2017-06-08T20:58:07.014314+00:00'
def test_all_products_are_shown(client: FlaskClient):
"""
After all the complicated grouping logic, there should still be one header link for each product.
"""
html = get_html(client, '/ls7_nbar_scene')
# We use a sorted array instead of a Set to detect duplicates too.
found_product_names = sorted([a.text.strip() for a in html.find('.product-selection-header .option-menu-link')])
indexed_product_names = sorted(p.name for p in _model.STORE.all_dataset_types())
assert found_product_names == indexed_product_names, "Product shown in menu don't match the indexed products"
def test_empty_product_overview(client: FlaskClient):
"""
A page is still displayable without error when it has no datasets.
"""
html = get_html(client, "/ls5_nbar_scene")
assert_is_text(html, ".dataset-count", "0 datasets")
assert_is_text(html, ".query-param.key-platform .value", "LANDSAT_5")
assert_is_text(html, ".query-param.key-instrument .value", "TM")
assert_is_text(html, ".query-param.key-product_type .value", "nbar")
def test_invalid_footprint_wofs_summary_load(client: FlaskClient):
# This all-time overview has a valid footprint that becomes invalid
# when reprojected to wrs84 by shapely.
from .data_wofs_summary import wofs_time_summary
_model.STORE._do_put("wofs_summary", None, None, None, wofs_time_summary)
html = get_html(client, "/wofs_summary")
check_dataset_count(html, 1244)
d = get_geojson(client, "/api/regions/wofs_summary")
assert len(d["features"]) == 1244
def test_product_audit(unpopulated_client: FlaskClient, run_generate):
run_generate()
client = unpopulated_client
res = get_html(client, "/product-audit/")
# print(res.html)
assert (res.find(".unavailable-metadata .search-result .product-name",
first=True).text == "s2_l2a")
assert (res.find(".unavailable-metadata .search-result .missing-footprint",
first=True).attrs["title"] == "0 of 5 missing footprint")
def test_view_dataset(client: FlaskClient):
# ls7_level1_scene dataset
html = get_html(client, "/dataset/57848615-2421-4d25-bfef-73f57de0574d")
# Label of dataset is header
assert 'LS7_ETM_OTH_P51_GALPGS01-002_105_074_20170501' in _h1_text(html)
# wofs_albers dataset (has no label or location)
rv: Response = client.get("/dataset/20c024b5-6623-4b06-b00c-6b5789f81eeb")
assert b'-20.502 to -19.6' in rv.data
assert b'132.0 to 132.924' in rv.data
def test_view_dataset(client: FlaskClient):
# ls7_level1_scene dataset
html = get_html(client, "/dataset/57848615-2421-4d25-bfef-73f57de0574d")
# Label of dataset is header
assert ("LS7_ETM_OTH_P51_GALPGS01-002_105_074_20170501"
in html.find("h2", first=True).text)
# wofs_albers dataset (has no label or location)
rv: HTML = get_html(client,
"/dataset/20c024b5-6623-4b06-b00c-6b5789f81eeb")
assert "-20.502 to -19.6" in rv.text
assert "132.0 to 132.924" in rv.text
# No dataset found: should return 404, not a server error.
rv: Response = client.get("/dataset/de071517-af92-4dd7-bf91-12b4e7c9a435",
follow_redirects=True)
assert rv.status_code == 404
assert b"No dataset found" in rv.data, rv.data.decode("utf-8")
def test_uninitialised_product(empty_client: FlaskClient,
summary_store: SummaryStore):
"""
An unsummarised product should still be viewable on the product page.
(but should be described as not summarised)
"""
# Populate one product, so they don't get the usage error message ("run cubedash generate")
# Then load an unpopulated product.
summary_store.refresh("ls7_nbar_albers")
html = get_html(empty_client, "/products/ls7_nbar_scene")
# The page should load without error, but will mention its lack of information
assert html.find("h2", first=True).text == "ls7_nbar_scene"
assert ("Product not summarised"
in one_element(html, ".header-stat-information").text)
# ... and a product that we populated does not have the message:
html = get_html(empty_client, "/products/ls7_nbar_albers")
assert ("Product not summarised"
not in one_element(html, ".header-stat-information").text)
def test_loading_high_low_tide_comp(client: FlaskClient):
html = get_html(client, "/high_tide_comp_20p/2008")
assert (
html.search("High Tide 20 percentage composites for entire coastline")
is not None)
check_dataset_count(html, 306)
# Footprint is not exact due to shapely.simplify()
check_area("2,984,...km2", html)
assert (one_element(html, ".last-processed time").attrs["datetime"] ==
"2017-06-08T20:58:07.014314+00:00")