def setUp(self):
super(TestSearchStacStatic, self).setUp()
self.dag = EODataAccessGateway()
self.cat_dir_path = self.root_cat = os.path.join(
TEST_RESOURCES_PATH, "stac")
# paths are relative in this test catalog, chdir needed
self._currentdir = os.getcwd()
os.chdir(self.cat_dir_path)
self.root_cat = os.path.join(self.cat_dir_path, "catalog.json")
self.root_cat_len = 5
self.child_cat = os.path.join(self.cat_dir_path, "country", "FRA",
"year", "2018", "2018.json")
self.child_cat_len = 2
self.stac_provider = "astraea_eod"
self.product_type = "S2_MSI_L1C"
self.extent_big = {
"lonmin": -55,
"lonmax": -53,
"latmin": 2,
"latmax": 5
}
self.extent_small = {
"lonmin": -55,
"lonmax": -54.5,
"latmin": 2,
"latmax": 2.5
}
def setUpClass(cls):
# use tests/resources/user_conf.yml if exists else default file ~/.config/eodag/eodag.yml
tests_user_conf = os.path.join(TEST_RESOURCES_PATH, "user_conf.yml")
if os.path.isfile(tests_user_conf):
cls.eodag = EODataAccessGateway(user_conf_file_path=os.path.join(
TEST_RESOURCES_PATH, "user_conf.yml"))
else:
cls.eodag = EODataAccessGateway()
# create TESTS_DOWNLOAD_PATH is not exists
if not os.path.exists(TESTS_DOWNLOAD_PATH):
os.makedirs(TESTS_DOWNLOAD_PATH)
for provider, conf in cls.eodag.providers_config.items():
# Change download directory to TESTS_DOWNLOAD_PATH for tests
if hasattr(conf, "download") and hasattr(conf.download,
"outputs_prefix"):
conf.download.outputs_prefix = TESTS_DOWNLOAD_PATH
elif hasattr(conf, "api") and hasattr(conf.api, "outputs_prefix"):
conf.api.outputs_prefix = TESTS_DOWNLOAD_PATH
else:
# no outputs_prefix found for provider
pass
# Force all providers implementing RestoSearch and defining how to retrieve
# products by specifying the
# location scheme to use https, enabling actual downloading of the product
if (getattr(getattr(conf, "search", {}), "product_location_scheme",
"https") == "file"):
conf.search.product_location_scheme = "https"
# Disable extraction
try: # Case HTTPDownload plugin
conf.download.extract = False
except (KeyError, AttributeError): # case api plugin
conf.api.extract = False
def test_end_to_end_good_apikey_wrong_credentials_aws_eos(self):
# Setup
# We retrieve correct credentials from the user_conf.yml file
tests_user_conf = os.path.join(TEST_RESOURCES_PATH, "user_conf.yml")
if not os.path.isfile(tests_user_conf):
self.skipTest("user_conf.yml file with credentials not found.")
# But we set the access key id and the secret to wrong values
try:
os.environ[
"EODAG__AWS_EOS__AUTH__CREDENTIALS__AWS_ACCESS_KEY_ID"] = "badaccessid"
os.environ[
"EODAG__AWS_EOS__AUTH__CREDENTIALS__AWS_SECRET_ACCESS_KEY"] = "badsecret"
eodag = EODataAccessGateway(user_conf_file_path=os.path.join(
TEST_RESOURCES_PATH, "user_conf.yml"))
eodag.set_preferred_provider(AWSEOS_SEARCH_ARGS[0])
results, nb_results = eodag.search(
raise_errors=True,
**dict(
zip(["productType", "start", "end", "geom"],
AWSEOS_SEARCH_ARGS[1:])))
self.assertGreater(len(results), 0)
one_product = results[0]
self.assertEqual(one_product.provider, AWSEOS_SEARCH_ARGS[0])
with self.assertRaises(AuthenticationError):
self.eodag.download(one_product)
# Teardown
finally:
os.environ.pop(
"EODAG__AWS_EOS__AUTH__CREDENTIALS__AWS_ACCESS_KEY_ID")
os.environ.pop(
"EODAG__AWS_EOS__AUTH__CREDENTIALS__AWS_SECRET_ACCESS_KEY")
def test_core_object_prioritize_config_file_in_envvar(self):
"""The core object must use the config file pointed to by the EODAG_CFG_FILE env var""" # noqa
os.environ["EODAG_CFG_FILE"] = os.path.join(
TEST_RESOURCES_PATH, "file_config_override.yml")
dag = EODataAccessGateway()
# usgs priority is set to 5 in the test config overrides
self.assertEqual(dag.get_preferred_provider(), ("usgs", 5))
# peps outputs prefix is set to /data
self.assertEqual(dag.providers_config["peps"].download.outputs_prefix,
"/data")
def test_update_providers_from_ext_plugin(self):
"""Load fake external plugin and check if it updates providers config"""
default_providers_count = len(self.dag.providers_config)
fakeplugin_location = os.path.join(TEST_RESOURCES_PATH,
"fake_ext_plugin")
# Create the fake entry point definition
ep = pkg_resources.EntryPoint.parse(
"FakePluginAPI = eodag_fakeplugin:FakePluginAPI")
# Create a fake distribution to insert into the global working_set
d = pkg_resources.Distribution(
location=fakeplugin_location,
project_name="eodag_fakeplugin",
version="0.1",
)
ep.dist = d
# Add the mapping to the fake EntryPoint
d._ep_map = {"eodag.plugins.api": {"FakePluginAPI": ep}}
# Add the fake distribution to the global working_set
pkg_resources.working_set.add(d, fakeplugin_location)
# New EODataAccessGateway instance, check if new conf has been loaded
self.dag = EODataAccessGateway()
self.assertEqual(len(self.dag.providers_config),
default_providers_count + 1)
def test_group_by_extent(self):
geom_coords_1 = [[[89, 2], [90, 2], [90, 3], [89, 3], [89, 2]]]
geom_coords_2 = [[[90, 3], [91, 3], [91, 4], [90, 4], [90, 3]]]
geom_coords_3 = [[[92, 4], [92, 4], [92, 5], [91, 5], [91, 4]]]
eo_geom1 = EOProduct.from_geojson(
self._minimal_eoproduct_geojson_repr("1", geom_coords_1)
)
eo_geom2 = EOProduct.from_geojson(
self._minimal_eoproduct_geojson_repr("2", geom_coords_2)
)
eo_geom3 = EOProduct.from_geojson(
self._minimal_eoproduct_geojson_repr("3", geom_coords_3)
)
first_search = SearchResult([eo_geom1])
second_search = SearchResult([eo_geom1, eo_geom2])
third_search = SearchResult([eo_geom1, eo_geom2, eo_geom3])
grouped_searches = EODataAccessGateway.group_by_extent(
[first_search, second_search, third_search]
)
# The returned value is a List[SearchResult]
self.assertIsInstance(grouped_searches, list)
self.assertTrue(all(isinstance(sr, SearchResult) for sr in grouped_searches))
# We expect three groups because we have given products that have
# three different geometry bounds.
self.assertEqual(len(grouped_searches), 3)
# Given how the search results were constructed the resulting groups
# must have these 3 different lengths.
ss_len = [len(sr) for sr in grouped_searches]
self.assertIn(1, ss_len)
self.assertIn(2, ss_len)
self.assertIn(3, ss_len)
def execution_involving_conf_dir(self, inspect=None):
"""Check that the path(s) inspected (str, list) are created after the instantation
of EODataAccessGateway. If they were already there, rename them (.old), instantiate,
check, delete the new files, and restore the existing files to there previous name."""
if inspect is not None:
if isinstance(inspect, str):
inspect = [inspect]
conf_dir = os.path.join(os.path.expanduser("~"), ".config",
"eodag")
olds = []
currents = []
for inspected in inspect:
old = current = os.path.join(conf_dir, inspected)
if os.path.exists(current):
old = os.path.join(conf_dir, "{}.old".format(inspected))
shutil.move(current, old)
olds.append(old)
currents.append(current)
EODataAccessGateway()
for old, current in zip(olds, currents):
self.assertTrue(os.path.exists(current))
if old != current:
try:
shutil.rmtree(current)
except OSError:
os.unlink(current)
shutil.move(old, current)
def test_core_object_open_index_if_exists(self, exists_in_mock,
open_dir_mock):
"""The core object must use the existing index dir if any"""
index_dir = os.path.join(self.conf_dir, ".index")
if not os.path.exists(index_dir):
makedirs(index_dir)
EODataAccessGateway()
open_dir_mock.assert_called_with(index_dir)
def test_core_object_prioritize_locations_file_in_envvar(self):
"""The core object must use the locations file pointed to by the EODAG_LOCS_CFG_FILE env var""" # noqa
os.environ["EODAG_LOCS_CFG_FILE"] = os.path.join(
TEST_RESOURCES_PATH, "file_locations_override.yml")
dag = EODataAccessGateway()
self.assertEqual(
dag.locations_config,
[dict(attr="dummyattr", name="dummyname", path="dummypath.shp")],
)
def setUp(self):
super(TestSearchStacStatic, self).setUp()
self.dag = EODataAccessGateway()
self.cat_dir_path = self.root_cat = os.path.join(TEST_RESOURCES_PATH, "stac")
self.root_cat = os.path.join(self.cat_dir_path, "catalog.json")
self.root_cat_len = 5
self.child_cat = os.path.join(
self.cat_dir_path, "country", "FRA", "year", "2018", "2018.json"
)
self.child_cat_len = 2
self.item = os.path.join(
os.path.dirname(self.child_cat),
"items",
"S2A_MSIL1C_20181231T141041_N0207_R110_T21NYF_20181231T155050",
"S2A_MSIL1C_20181231T141041_N0207_R110_T21NYF_20181231T155050.json",
)
self.singlefile_cat = os.path.join(TEST_RESOURCES_PATH, "stac_singlefile.json")
self.singlefile_cat_len = 5
self.stac_provider = "astraea_eod"
self.product_type = "S2_MSI_L1C"
self.extent_big = {"lonmin": -55, "lonmax": -53, "latmin": 2, "latmax": 5}
self.extent_small = {"lonmin": -55, "lonmax": -54.5, "latmin": 2, "latmax": 2.5}
self.static_stac_provider = "foo_static"
self.dag.update_providers_config(
f"""
{self.static_stac_provider}:
search:
type: StaticStacSearch
api_endpoint: {self.root_cat}
products:
GENERIC_PRODUCT_TYPE:
productType: '{{productType}}'
download:
type: HTTPDownload
base_uri: https://fake-endpoint
flatten_top_dirs: True
"""
)
self.dag.set_preferred_provider(self.static_stac_provider)
def test_core_object_set_default_locations_config(self):
"""The core object must set the default locations config on instantiation""" # noqa
dag = EODataAccessGateway()
conf_dir = os.path.join(os.path.expanduser("~"), ".config", "eodag")
default_shpfile = os.path.join(conf_dir, "shp",
"ne_110m_admin_0_map_units.shp")
self.assertIsInstance(dag.locations_config, list)
self.assertEqual(
dag.locations_config,
[dict(attr="ADM0_A3_US", name="country", path=default_shpfile)],
)
def setUpClass(cls):
cls.dag = EODataAccessGateway()
# Get a SearchResult obj with 2 S2_MSI_L1C peps products
search_results_file = os.path.join(TEST_RESOURCES_PATH,
"eodag_search_result_peps.geojson")
with open(search_results_file, encoding="utf-8") as f:
search_results_geojson = json.load(f)
cls.search_results = SearchResult.from_geojson(search_results_geojson)
cls.search_results_size = len(cls.search_results)
# Change the id of these products, to emulate different products
search_results_data_2 = deepcopy(cls.search_results.data)
search_results_data_2[0].properties["id"] = "a"
search_results_data_2[1].properties["id"] = "b"
cls.search_results_2 = SearchResult(search_results_data_2)
cls.search_results_size_2 = len(cls.search_results_2)
def execution_involving_conf_dir(self, inspect=None):
if inspect is not None:
conf_dir = os.path.join(os.path.expanduser("~"), ".config",
"eodag")
old = current = os.path.join(conf_dir, inspect)
if os.path.exists(current):
old = os.path.join(conf_dir, "{}.old".format(inspect))
shutil.move(current, old)
EODataAccessGateway()
self.assertTrue(os.path.exists(current))
if old != current:
try:
shutil.rmtree(current)
except OSError:
os.unlink(current)
shutil.move(old, current)
def test_search_all_user_items_per_page(self, mocked_search_iter_page):
"""search_all must use the value of items_per_page provided by the user"""
dag = EODataAccessGateway()
dummy_provider_config = """
dummy_provider:
search:
type: QueryStringSearch
api_endpoint: https://api.my_new_provider/search
metadata_mapping:
dummy: 'dummy'
products:
S2_MSI_L1C:
productType: '{productType}'
"""
mocked_search_iter_page.return_value = (self.search_results
for _ in range(1))
dag.update_providers_config(dummy_provider_config)
dag.set_preferred_provider("dummy_provider")
dag.search_all(productType="S2_MSI_L1C", items_per_page=7)
self.assertEqual(
mocked_search_iter_page.call_args[1]["items_per_page"], 7)
def test_search_iter_page_must_reset_next_attrs_if_next_mechanism(
self, normalize_results, _request):
"""search_iter_page must reset the search plugin if the next mechanism is used"""
# More specifically: next_page_url must be None and
# config.pagination["next_page_url_tpl"] must be equal to its original value.
_request.return_value.json.return_value = {
"features": [],
"links": [{
"rel": "next",
"href": "url/to/next/page"
}],
}
p1 = EOProduct("dummy", dict(geometry="POINT (0 0)", id="1"))
p1.search_intersection = None
p2 = EOProduct("dummy", dict(geometry="POINT (0 0)", id="2"))
p2.search_intersection = None
normalize_results.side_effect = [[p1], [p2]]
dag = EODataAccessGateway()
dummy_provider_config = """
dummy_provider:
search:
type: QueryStringSearch
api_endpoint: https://api.my_new_provider/search
pagination:
next_page_url_tpl: 'dummy_next_page_url_tpl'
next_page_url_key_path: '$.links[?(@.rel="next")].href'
metadata_mapping:
dummy: 'dummy'
products:
S2_MSI_L1C:
productType: '{productType}'
"""
dag.update_providers_config(dummy_provider_config)
dag.set_preferred_provider("dummy_provider")
page_iterator = dag.search_iter_page(productType="S2_MSI_L1C")
next(page_iterator)
search_plugin = next(
dag._plugins_manager.get_search_plugins(product_type="S2_MSI_L1C"))
self.assertIsNone(search_plugin.next_page_url)
self.assertEqual(
search_plugin.config.pagination["next_page_url_tpl"],
"dummy_next_page_url_tpl",
)
def test_search_all_use_max_items_per_page(self, mocked_search_iter_page):
"""search_all must use the configured parameter max_items_per_page if available""" # noqa
dag = EODataAccessGateway()
dummy_provider_config = """
dummy_provider:
search:
type: QueryStringSearch
api_endpoint: https://api.my_new_provider/search
pagination:
max_items_per_page: 2
metadata_mapping:
dummy: 'dummy'
products:
S2_MSI_L1C:
productType: '{productType}'
"""
mocked_search_iter_page.return_value = (self.search_results
for _ in range(1))
dag.update_providers_config(dummy_provider_config)
dag.set_preferred_provider("dummy_provider")
dag.search_all(productType="S2_MSI_L1C")
self.assertEqual(
mocked_search_iter_page.call_args[1]["items_per_page"], 2)
def test_search_all_use_default_value(self, mocked_search_iter_page):
"""search_all must use the DEFAULT_MAX_ITEMS_PER_PAGE if the provider's one wasn't configured""" # noqa
dag = EODataAccessGateway()
dummy_provider_config = """
dummy_provider:
search:
type: QueryStringSearch
api_endpoint: https://api.my_new_provider/search
metadata_mapping:
dummy: 'dummy'
products:
S2_MSI_L1C:
productType: '{productType}'
"""
mocked_search_iter_page.return_value = (self.search_results
for _ in range(1))
dag.update_providers_config(dummy_provider_config)
dag.set_preferred_provider("dummy_provider")
dag.search_all(productType="S2_MSI_L1C")
self.assertEqual(
mocked_search_iter_page.call_args[1]["items_per_page"],
DEFAULT_MAX_ITEMS_PER_PAGE,
)
def test_core_object_locations_file_not_found(self):
"""The core object must set the locations to an empty list when the file is not found""" # noqa
dag = EODataAccessGateway(locations_conf_path="no_locations.yml")
self.assertEqual(dag.locations_config, [])
def setUpClass(cls):
tests_wrong_conf = os.path.join(TEST_RESOURCES_PATH,
"wrong_credentials_conf.yml")
cls.eodag = EODataAccessGateway(user_conf_file_path=tests_wrong_conf)
class TestCore(unittest.TestCase):
SUPPORTED_PRODUCT_TYPES = {
"L8_REFLECTANCE": ["theia"],
"L57_REFLECTANCE": ["theia"],
"L5_L1T": [],
"L5_L1G": [],
"L5_L1GT": [],
"L7_L1G": [],
"L7_L1T": [],
"L7_L1GT": [],
"L8_OLI_TIRS_C1L1": ["onda", "usgs", "aws_eos"],
"S1_SAR_GRD": [
"peps",
"sobloo",
"onda",
"wekeo",
"mundi",
"creodias",
"aws_eos",
],
"S1_SAR_OCN": ["peps", "sobloo", "onda", "creodias"],
"S1_SAR_RAW": ["sobloo", "onda", "creodias"],
"S1_SAR_SLC": ["peps", "sobloo", "onda", "wekeo", "mundi", "creodias"],
"S2_MSI_L2A": ["onda", "mundi", "creodias", "peps", "aws_eos"],
"S2_MSI_L2A_MAJA": ["theia"],
"S2_MSI_L2B_MAJA_SNOW": ["theia"],
"S2_MSI_L2B_MAJA_WATER": ["theia"],
"S2_MSI_L3A_WASP": ["theia"],
"S2_MSI_L1C": [
"aws_eos",
"peps",
"sobloo",
"onda",
"wekeo",
"mundi",
"creodias",
],
"S3_ERR": ["peps", "onda", "wekeo", "creodias"],
"S3_EFR": ["peps", "onda", "wekeo", "creodias"],
"S3_LAN": ["peps", "sobloo", "onda", "wekeo", "creodias"],
"S3_SRA": ["sobloo", "onda", "wekeo", "creodias"],
"S3_SRA_BS": ["sobloo", "onda", "wekeo", "creodias"],
"S3_SRA_A_BS": ["sobloo", "onda", "creodias"],
"S3_WAT": ["onda", "wekeo", "creodias"],
"S3_OLCI_L2LFR": ["peps", "onda", "wekeo", "creodias", "mundi"],
"S3_OLCI_L2LRR": ["peps", "onda", "wekeo", "creodias"],
"S3_SLSTR_L1RBT": ["peps", "onda", "wekeo", "creodias"],
"S3_SLSTR_L2LST": ["peps", "onda", "wekeo", "creodias"],
"PLD_PAN": ["theia"],
"PLD_XS": ["theia"],
"PLD_BUNDLE": ["theia"],
"PLD_PANSHARPENED": ["theia"],
"ES_FRS": [],
"CBERS4_PAN10M_L2": ["aws_eos"],
"CBERS4_PAN10M_L4": ["aws_eos"],
"CBERS4_PAN5M_L2": ["aws_eos"],
"CBERS4_PAN5M_L4": ["aws_eos"],
"CBERS4_MUX_L2": ["aws_eos"],
"CBERS4_MUX_L4": ["aws_eos"],
"CBERS4_AWFI_L2": ["aws_eos"],
"CBERS4_AWFI_L4": ["aws_eos"],
"MODIS_MCD43A4": ["aws_eos"],
"NAIP": ["aws_eos"],
"SPOT_SWH": ["theia"],
"SPOT_SWH_OLD": ["theia"],
"SPOT5_SPIRIT": ["theia"],
"VENUS_L1C": ["theia"],
"VENUS_L2A_MAJA": ["theia"],
"VENUS_L3A_MAJA": ["theia"],
"OSO": ["theia"],
}
SUPPORTED_PROVIDERS = [
"peps",
"usgs",
"theia",
"sobloo",
"creodias",
"mundi",
"onda",
"wekeo",
"aws_eos",
]
def setUp(self):
super(TestCore, self).setUp()
self.dag = EODataAccessGateway()
def tearDown(self):
super(TestCore, self).tearDown()
for old in glob.glob1(self.dag.conf_dir, "*.old") + glob.glob1(
self.dag.conf_dir, ".*.old"):
old_path = os.path.join(self.dag.conf_dir, old)
if os.path.exists(old_path):
try:
os.remove(old_path)
except OSError:
shutil.rmtree(old_path)
if os.getenv("EODAG_CFG_FILE") is not None:
os.environ.pop("EODAG_CFG_FILE")
def test_supported_providers_in_unit_test(self):
"""Every provider must be referenced in the core unittest SUPPORTED_PROVIDERS class attribute""" # noqa
for provider in self.dag.available_providers():
self.assertIn(provider, self.SUPPORTED_PROVIDERS)
def test_supported_product_types_in_unit_test(self):
"""Every product type must be referenced in the core unit test SUPPORTED_PRODUCT_TYPES class attribute""" # noqa
for product_type in self.dag.list_product_types():
self.assertIn(product_type["ID"],
self.SUPPORTED_PRODUCT_TYPES.keys())
def test_list_product_types_ok(self):
"""Core api must correctly return the list of supported product types"""
product_types = self.dag.list_product_types()
self.assertIsInstance(product_types, list)
for product_type in product_types:
self.assertListProductTypesRightStructure(product_type)
# There should be no repeated product type in the output
self.assertEqual(len(product_types),
len(set(pt["ID"] for pt in product_types)))
def test_list_product_types_for_provider_ok(self):
"""Core api must correctly return the list of supported product types for a given provider""" # noqa
for provider in self.SUPPORTED_PROVIDERS:
product_types = self.dag.list_product_types(provider=provider)
self.assertIsInstance(product_types, list)
for product_type in product_types:
self.assertListProductTypesRightStructure(product_type)
self.assertIn(provider,
self.SUPPORTED_PRODUCT_TYPES[product_type["ID"]])
def test_list_product_types_for_unsupported_provider(self):
"""Core api must raise UnsupportedProvider error for list_product_types with unsupported provider""" # noqa
unsupported_provider = "a"
self.assertRaises(
UnsupportedProvider,
self.dag.list_product_types,
provider=unsupported_provider,
)
def assertListProductTypesRightStructure(self, structure):
"""Helper method to verify that the structure given is a good result of
EODataAccessGateway.list_product_types
"""
self.assertIsInstance(structure, dict)
self.assertIn("ID", structure)
self.assertIn("abstract", structure)
self.assertIn("instrument", structure)
self.assertIn("platform", structure)
self.assertIn("platformSerialIdentifier", structure)
self.assertIn("processingLevel", structure)
self.assertIn("sensorType", structure)
self.assertIn(structure["ID"], self.SUPPORTED_PRODUCT_TYPES)
def test_core_object_creates_config_standard_location(self):
"""The core object must create a user config file in standard user config location on instantiation""" # noqa
self.execution_involving_conf_dir(inspect="eodag.yml")
def test_core_object_creates_index_if_not_exist(self):
"""The core object must create an index in user config directory"""
self.execution_involving_conf_dir(inspect=".index")
@mock.patch("eodag.api.core.open_dir", autospec=True)
@mock.patch("eodag.api.core.exists_in", autospec=True, return_value=True)
def test_core_object_open_index_if_exists(self, exists_in_mock,
open_dir_mock):
"""The core object must use the existing index dir if any"""
conf_dir = os.path.join(os.path.expanduser("~"), ".config", "eodag")
index_dir = os.path.join(conf_dir, ".index")
if not os.path.exists(index_dir):
makedirs(index_dir)
EODataAccessGateway()
open_dir_mock.assert_called_with(index_dir)
def test_core_object_prioritize_config_file_in_envvar(self):
"""The core object must use the config file pointed to by the EODAG_CFG_FILE env var""" # noqa
os.environ["EODAG_CFG_FILE"] = os.path.join(
TEST_RESOURCES_PATH, "file_config_override.yml")
dag = EODataAccessGateway()
# usgs priority is set to 5 in the test config overrides
self.assertEqual(dag.get_preferred_provider(), ("usgs", 5))
# peps outputs prefix is set to /data
self.assertEqual(dag.providers_config["peps"].download.outputs_prefix,
"/data")
def execution_involving_conf_dir(self, inspect=None):
if inspect is not None:
conf_dir = os.path.join(os.path.expanduser("~"), ".config",
"eodag")
old = current = os.path.join(conf_dir, inspect)
if os.path.exists(current):
old = os.path.join(conf_dir, "{}.old".format(inspect))
shutil.move(current, old)
EODataAccessGateway()
self.assertTrue(os.path.exists(current))
if old != current:
try:
shutil.rmtree(current)
except OSError:
os.unlink(current)
shutil.move(old, current)
class TestSearchStacStatic(unittest.TestCase):
def setUp(self):
super(TestSearchStacStatic, self).setUp()
self.dag = EODataAccessGateway()
self.cat_dir_path = self.root_cat = os.path.join(TEST_RESOURCES_PATH, "stac")
self.root_cat = os.path.join(self.cat_dir_path, "catalog.json")
self.root_cat_len = 5
self.child_cat = os.path.join(
self.cat_dir_path, "country", "FRA", "year", "2018", "2018.json"
)
self.child_cat_len = 2
self.item = os.path.join(
os.path.dirname(self.child_cat),
"items",
"S2A_MSIL1C_20181231T141041_N0207_R110_T21NYF_20181231T155050",
"S2A_MSIL1C_20181231T141041_N0207_R110_T21NYF_20181231T155050.json",
)
self.singlefile_cat = os.path.join(TEST_RESOURCES_PATH, "stac_singlefile.json")
self.singlefile_cat_len = 5
self.stac_provider = "astraea_eod"
self.product_type = "S2_MSI_L1C"
self.extent_big = {"lonmin": -55, "lonmax": -53, "latmin": 2, "latmax": 5}
self.extent_small = {"lonmin": -55, "lonmax": -54.5, "latmin": 2, "latmax": 2.5}
self.static_stac_provider = "foo_static"
self.dag.update_providers_config(
f"""
{self.static_stac_provider}:
search:
type: StaticStacSearch
api_endpoint: {self.root_cat}
products:
GENERIC_PRODUCT_TYPE:
productType: '{{productType}}'
download:
type: HTTPDownload
base_uri: https://fake-endpoint
flatten_top_dirs: True
"""
)
self.dag.set_preferred_provider(self.static_stac_provider)
def test_search_stac_static_load_child(self):
"""load_stac_items from child catalog must provide items"""
items = self.dag.load_stac_items(
self.child_cat, recursive=True, provider=self.stac_provider
)
self.assertIsInstance(items, SearchResult)
self.assertEqual(len(items), self.child_cat_len)
self.assertEqual(items[0].provider, self.stac_provider)
# if no product_type is provided, product_type is None
self.assertIsNone(items[0].product_type)
def test_search_stac_static_load_root_not_recursive(self):
"""load_stac_items from root must provide an empty list when no recursive"""
items = self.dag.load_stac_items(
self.root_cat, recursive=False, provider=self.stac_provider
)
self.assertEqual(len(items), 0)
def test_search_stac_static_load_root_recursive(self):
"""load_stac_items from root must provide items when recursive"""
items = self.dag.load_stac_items(
self.root_cat,
recursive=True,
provider=self.stac_provider,
productType=self.product_type,
)
self.assertEqual(len(items), self.root_cat_len)
for item in items:
self.assertEqual(item.provider, self.stac_provider)
self.assertEqual(item.product_type, self.product_type)
def test_search_stac_static(self):
"""Use StaticStacSearch plugin to search all items"""
items, nb = self.dag.search()
self.assertEqual(len(items), self.root_cat_len)
self.assertEqual(nb, self.root_cat_len)
for item in items:
self.assertEqual(item.provider, self.static_stac_provider)
def test_search_stac_static_load_item(self):
"""load_stac_items from a single item must provide it"""
item = self.dag.load_stac_items(self.item, provider=self.stac_provider)
self.assertIsInstance(item, SearchResult)
self.assertEqual(len(item), 1)
self.assertEqual(item[0].provider, self.stac_provider)
# if no product_type is provided, product_type is None
self.assertIsNone(item[0].product_type)
def test_search_stac_static_load_item_updated_provider(self):
"""load_stac_items from a single item using updated provider"""
item = self.dag.load_stac_items(self.item, provider=self.stac_provider)
self.assertEqual(item[0].properties["license"], "proprietary")
self.assertEqual(item[0].properties["platform"], "S2ST")
self.assertEqual(item[0].properties["orbitDirection"], "descending")
self.assertNotIn("foo", item[0].properties)
# fake provider with mixed metadata_mapping
self.dag.update_providers_config(
"""
fake_provider:
search:
type: StacSearch
api_endpoint: 'https://fake-endpoint'
metadata_mapping:
license: '{platform}'
foo: '{orbitDirection}'
products:
GENERIC_PRODUCT_TYPE:
productType: '{productType}'
download:
type: HTTPDownload
base_uri: 'https://fake-uri'
"""
)
item = self.dag.load_stac_items(
self.item, provider="fake_provider", raise_errors=True
)
self.assertEqual(item[0].properties["platform"], "S2ST")
self.assertEqual(item[0].properties["license"], "S2ST")
self.assertEqual(item[0].properties["orbitDirection"], "descending")
self.assertIn("foo", item[0].properties)
self.assertEqual(item[0].properties["foo"], "descending")
def test_search_stac_static_load_singlefile_catalog(self):
"""load_stac_items from child catalog must provide items"""
items = self.dag.load_stac_items(
self.singlefile_cat, provider=self.stac_provider
)
self.assertIsInstance(items, SearchResult)
self.assertEqual(len(items), self.singlefile_cat_len)
self.assertEqual(items[0].provider, self.stac_provider)
# if no product_type is provided, product_type is None
self.assertIsNone(items[0].product_type)
def test_search_stac_static_crunch_filter_date(self):
"""load_stac_items from root and filter by date"""
items = self.dag.load_stac_items(
self.root_cat,
recursive=True,
provider=self.stac_provider,
productType=self.product_type,
)
filtered_items = items.crunch(
FilterDate({"start": "2018-01-01", "end": "2019-01-01"})
)
self.assertEqual(len(filtered_items), self.child_cat_len)
for item in filtered_items:
self.assertIn("2018", item.properties["startTimeFromAscendingNode"])
def test_search_stac_static_by_date(self):
"""Use StaticStacSearch plugin to search by date"""
filtered_items, nb = self.dag.search(start="2018-01-01", end="2019-01-01")
self.assertEqual(len(filtered_items), self.child_cat_len)
self.assertEqual(nb, self.child_cat_len)
for item in filtered_items:
self.assertIn("2018", item.properties["startTimeFromAscendingNode"])
def test_search_stac_static_crunch_filter_overlap(self):
"""load_stac_items from root and filter by overlap"""
# tests over extent_big search geometry
items = self.dag.load_stac_items(
self.root_cat,
recursive=True,
provider=self.stac_provider,
productType=self.product_type,
geom=self.extent_big,
)
self.assertEqual(len(items), self.root_cat_len)
filtered_items = items.crunch(
FilterOverlap({"minimum_overlap": 10}), geometry=self.extent_big
)
self.assertEqual(len(filtered_items), 3)
filtered_items = items.crunch(
FilterOverlap({"minimum_overlap": 100}), geometry=self.extent_big
)
self.assertEqual(len(filtered_items), 1)
filtered_items = items.crunch(
FilterOverlap({"within": True}), geometry=self.extent_big
)
self.assertEqual(len(filtered_items), 1)
filtered_items = items.crunch(
FilterOverlap({"contains": True}), geometry=self.extent_big
)
self.assertEqual(len(filtered_items), 0)
filtered_items = items.crunch(
FilterOverlap({"intersects": True}), geometry=self.extent_big
)
self.assertEqual(len(filtered_items), 3)
# tests over extent_small search geometry
items = self.dag.load_stac_items(
self.root_cat,
recursive=True,
provider=self.stac_provider,
productType=self.product_type,
geom=self.extent_small,
)
self.assertEqual(len(items), self.root_cat_len)
filtered_items = items.crunch(
FilterOverlap({"contains": True}), geometry=self.extent_small
)
self.assertEqual(len(filtered_items), 1)
def test_search_stac_static_by_geom(self):
"""Use StaticStacSearch plugin to search by geometry"""
items, nb = self.dag.search(
geom=self.extent_big,
)
self.assertEqual(len(items), 3)
self.assertEqual(nb, 3)
def test_search_stac_static_crunch_filter_property(self):
"""load_stac_items from root and filter by property"""
items = self.dag.load_stac_items(
self.root_cat,
recursive=True,
provider=self.stac_provider,
productType=self.product_type,
)
self.assertEqual(len(items), self.root_cat_len)
filtered_items = items.crunch(FilterProperty({"orbitNumber": 110}))
self.assertEqual(len(filtered_items), 3)
filtered_items = items.crunch(
FilterProperty({"platformSerialIdentifier": "S2A", "operator": "eq"})
)
self.assertEqual(len(filtered_items), 4)
filtered_items = items.crunch(
FilterProperty({"cloudCover": 10, "operator": "lt"})
)
self.assertEqual(len(filtered_items), 1)
def test_search_stac_static_by_property(self):
"""Use StaticStacSearch plugin to search by property"""
items, nb = self.dag.search(orbitNumber=110)
self.assertEqual(len(items), 3)
self.assertEqual(nb, 3)
def test_search_stac_static_by_cloudcover(self):
"""Use StaticStacSearch plugin to search by cloud cover"""
items, nb = self.dag.search(cloudCover=10)
self.assertEqual(len(items), 1)
self.assertEqual(nb, 1)
def test_search_stac_static_crunch_filter_lastest_by_name(self):
"""load_stac_items from root and filter by name"""
items = self.dag.load_stac_items(
self.root_cat,
recursive=True,
provider=self.stac_provider,
productType=self.product_type,
)
self.assertEqual(len(items), self.root_cat_len)
filtered_items = items.crunch(
FilterLatestByName(
{"name_pattern": r"S2[AB]_MSIL1C_20(?P<tileid>\d{6}).*T21NY.*"}
)
)
self.assertEqual(len(filtered_items), 2)
def setUpClass(cls):
cls.dag = EODataAccessGateway()
def setUp(self):
super(TestCore, self).setUp()
self.dag = EODataAccessGateway()
def setUp(self):
super(TestCoreInvolvingConfDir, self).setUp()
self.dag = EODataAccessGateway()
def setUp(self):
super(TestExternalPluginConfig, self).setUp()
self.dag = EODataAccessGateway()
class TestCore(unittest.TestCase):
SUPPORTED_PRODUCT_TYPES = {
"L8_REFLECTANCE": ["theia"],
"L57_REFLECTANCE": ["theia"],
"L8_OLI_TIRS_C1L1": ["onda", "usgs", "aws_eos", "astraea_eod"],
"S1_SAR_GRD": [
"peps",
"sobloo",
"onda",
"wekeo",
"mundi",
"creodias",
"aws_eos",
],
"S1_SAR_OCN": ["peps", "sobloo", "onda", "creodias"],
"S1_SAR_RAW": ["sobloo", "onda", "creodias"],
"S1_SAR_SLC": ["peps", "sobloo", "onda", "wekeo", "mundi", "creodias"],
"S2_MSI_L2A": [
"onda",
"mundi",
"creodias",
"peps",
"aws_eos",
"sobloo",
"astraea_eod",
],
"S2_MSI_L2A_MAJA": ["theia"],
"S2_MSI_L2B_MAJA_SNOW": ["theia"],
"S2_MSI_L2B_MAJA_WATER": ["theia"],
"S2_MSI_L3A_WASP": ["theia"],
"S2_MSI_L1C": [
"aws_eos",
"peps",
"sobloo",
"onda",
"wekeo",
"mundi",
"creodias",
"astraea_eod",
],
"S3_ERR": ["peps", "onda", "wekeo", "creodias"],
"S3_EFR": ["peps", "onda", "wekeo", "creodias"],
"S3_LAN": ["peps", "onda", "wekeo", "creodias"],
"S3_SRA": ["onda", "wekeo", "creodias"],
"S3_SRA_BS": ["onda", "wekeo", "creodias"],
"S3_SRA_A_BS": ["onda", "creodias"],
"S3_WAT": ["onda", "wekeo", "creodias"],
"S3_OLCI_L2LFR": ["peps", "onda", "wekeo", "creodias", "mundi"],
"S3_OLCI_L2LRR": ["peps", "onda", "wekeo", "creodias"],
"S3_SLSTR_L1RBT": ["peps", "onda", "wekeo", "creodias"],
"S3_SLSTR_L2LST": ["peps", "onda", "wekeo", "creodias"],
"PLD_PAN": ["theia"],
"PLD_XS": ["theia"],
"PLD_BUNDLE": ["theia"],
"PLD_PANSHARPENED": ["theia"],
"CBERS4_PAN10M_L2": ["aws_eos"],
"CBERS4_PAN10M_L4": ["aws_eos"],
"CBERS4_PAN5M_L2": ["aws_eos"],
"CBERS4_PAN5M_L4": ["aws_eos"],
"CBERS4_MUX_L2": ["aws_eos"],
"CBERS4_MUX_L4": ["aws_eos"],
"CBERS4_AWFI_L2": ["aws_eos"],
"CBERS4_AWFI_L4": ["aws_eos"],
"MODIS_MCD43A4": ["aws_eos", "astraea_eod"],
"NAIP": ["aws_eos", "astraea_eod"],
"SPOT_SWH": ["theia"],
"SPOT_SWH_OLD": ["theia"],
"SPOT5_SPIRIT": ["theia"],
"VENUS_L1C": ["theia"],
"VENUS_L2A_MAJA": ["theia"],
"VENUS_L3A_MAJA": ["theia"],
"OSO": ["theia"],
GENERIC_PRODUCT_TYPE: [
"theia",
"peps",
"sobloo",
"onda",
"mundi",
"creodias",
"astraea_eod",
],
}
SUPPORTED_PROVIDERS = [
"peps",
"usgs",
"theia",
"sobloo",
"creodias",
"mundi",
"onda",
"wekeo",
"aws_eos",
"astraea_eod",
]
def setUp(self):
super(TestCore, self).setUp()
self.dag = EODataAccessGateway()
def tearDown(self):
super(TestCore, self).tearDown()
for old in glob.glob1(self.dag.conf_dir, "*.old") + glob.glob1(
self.dag.conf_dir, ".*.old"):
old_path = os.path.join(self.dag.conf_dir, old)
if os.path.exists(old_path):
try:
os.remove(old_path)
except OSError:
shutil.rmtree(old_path)
if os.getenv("EODAG_CFG_FILE") is not None:
os.environ.pop("EODAG_CFG_FILE")
if os.getenv("EODAG_LOCS_CFG_FILE") is not None:
os.environ.pop("EODAG_LOCS_CFG_FILE")
def test_supported_providers_in_unit_test(self):
"""Every provider must be referenced in the core unittest SUPPORTED_PROVIDERS class attribute""" # noqa
for provider in self.dag.available_providers():
self.assertIn(provider, self.SUPPORTED_PROVIDERS)
def test_supported_product_types_in_unit_test(self):
"""Every product type must be referenced in the core unit test SUPPORTED_PRODUCT_TYPES class attribute""" # noqa
for product_type in self.dag.list_product_types():
self.assertIn(product_type["ID"],
self.SUPPORTED_PRODUCT_TYPES.keys())
def test_list_product_types_ok(self):
"""Core api must correctly return the list of supported product types"""
product_types = self.dag.list_product_types()
self.assertIsInstance(product_types, list)
for product_type in product_types:
self.assertListProductTypesRightStructure(product_type)
# There should be no repeated product type in the output
self.assertEqual(len(product_types),
len(set(pt["ID"] for pt in product_types)))
def test_list_product_types_for_provider_ok(self):
"""Core api must correctly return the list of supported product types for a given provider""" # noqa
for provider in self.SUPPORTED_PROVIDERS:
product_types = self.dag.list_product_types(provider=provider)
self.assertIsInstance(product_types, list)
for product_type in product_types:
self.assertListProductTypesRightStructure(product_type)
self.assertIn(provider,
self.SUPPORTED_PRODUCT_TYPES[product_type["ID"]])
def test_list_product_types_for_unsupported_provider(self):
"""Core api must raise UnsupportedProvider error for list_product_types with unsupported provider""" # noqa
unsupported_provider = "a"
self.assertRaises(
UnsupportedProvider,
self.dag.list_product_types,
provider=unsupported_provider,
)
def assertListProductTypesRightStructure(self, structure):
"""Helper method to verify that the structure given is a good result of
EODataAccessGateway.list_product_types
"""
self.assertIsInstance(structure, dict)
self.assertIn("ID", structure)
self.assertIn("abstract", structure)
self.assertIn("instrument", structure)
self.assertIn("platform", structure)
self.assertIn("platformSerialIdentifier", structure)
self.assertIn("processingLevel", structure)
self.assertIn("sensorType", structure)
self.assertIn(structure["ID"], self.SUPPORTED_PRODUCT_TYPES)
def test_core_object_creates_config_standard_location(self):
"""The core object must create a user config file in standard user config location on instantiation""" # noqa
self.execution_involving_conf_dir(inspect="eodag.yml")
def test_core_object_creates_index_if_not_exist(self):
"""The core object must create an index in user config directory"""
self.execution_involving_conf_dir(inspect=".index")
@mock.patch("eodag.api.core.open_dir", autospec=True)
@mock.patch("eodag.api.core.exists_in", autospec=True, return_value=True)
def test_core_object_open_index_if_exists(self, exists_in_mock,
open_dir_mock):
"""The core object must use the existing index dir if any"""
conf_dir = os.path.join(os.path.expanduser("~"), ".config", "eodag")
index_dir = os.path.join(conf_dir, ".index")
if not os.path.exists(index_dir):
makedirs(index_dir)
EODataAccessGateway()
open_dir_mock.assert_called_with(index_dir)
def test_core_object_prioritize_config_file_in_envvar(self):
"""The core object must use the config file pointed to by the EODAG_CFG_FILE env var""" # noqa
os.environ["EODAG_CFG_FILE"] = os.path.join(
TEST_RESOURCES_PATH, "file_config_override.yml")
dag = EODataAccessGateway()
# usgs priority is set to 5 in the test config overrides
self.assertEqual(dag.get_preferred_provider(), ("usgs", 5))
# peps outputs prefix is set to /data
self.assertEqual(dag.providers_config["peps"].download.outputs_prefix,
"/data")
def execution_involving_conf_dir(self, inspect=None):
"""Check that the path(s) inspected (str, list) are created after the instantation
of EODataAccessGateway. If they were already there, rename them (.old), instantiate,
check, delete the new files, and restore the existing files to there previous name."""
if inspect is not None:
if isinstance(inspect, str):
inspect = [inspect]
conf_dir = os.path.join(os.path.expanduser("~"), ".config",
"eodag")
olds = []
currents = []
for inspected in inspect:
old = current = os.path.join(conf_dir, inspected)
if os.path.exists(current):
old = os.path.join(conf_dir, "{}.old".format(inspected))
shutil.move(current, old)
olds.append(old)
currents.append(current)
EODataAccessGateway()
for old, current in zip(olds, currents):
self.assertTrue(os.path.exists(current))
if old != current:
try:
shutil.rmtree(current)
except OSError:
os.unlink(current)
shutil.move(old, current)
def test_core_object_creates_locations_standard_location(self):
"""The core object must create a locations config file and a shp dir in standard user config location on instantiation""" # noqa
self.execution_involving_conf_dir(inspect=["locations.yml", "shp"])
def test_core_object_set_default_locations_config(self):
"""The core object must set the default locations config on instantiation""" # noqa
dag = EODataAccessGateway()
conf_dir = os.path.join(os.path.expanduser("~"), ".config", "eodag")
default_shpfile = os.path.join(conf_dir, "shp",
"ne_110m_admin_0_map_units.shp")
self.assertIsInstance(dag.locations_config, list)
self.assertEqual(
dag.locations_config,
[dict(attr="ADM0_A3_US", name="country", path=default_shpfile)],
)
def test_core_object_locations_file_not_found(self):
"""The core object must set the locations to an empty list when the file is not found""" # noqa
dag = EODataAccessGateway(locations_conf_path="no_locations.yml")
self.assertEqual(dag.locations_config, [])
def test_core_object_prioritize_locations_file_in_envvar(self):
"""The core object must use the locations file pointed to by the EODEODAG_LOCS_CFG_FILEAG_CFG_FILE env var""" # noqa
os.environ["EODAG_LOCS_CFG_FILE"] = os.path.join(
TEST_RESOURCES_PATH, "file_locations_override.yml")
dag = EODataAccessGateway()
self.assertEqual(
dag.locations_config,
[dict(attr="dummyattr", name="dummyname", path="dummypath.shp")],
)
def test_get_geometry_from_various_no_locations(self):
"""The search geometry can be set from a dict, list, tuple, WKT string or shapely geom"""
ref_geom_as_wkt = "POLYGON ((0 50, 0 52, 2 52, 2 50, 0 50))"
ref_geom = wkt.loads(ref_geom_as_wkt)
# Good dict
geometry = {
"lonmin": 0,
"latmin": 50,
"lonmax": 2,
"latmax": 52,
}
self.assertEquals(get_geometry_from_various([], geometry=geometry),
ref_geom)
# Bad dict with a missing key
del geometry["lonmin"]
self.assertRaises(
TypeError,
get_geometry_from_various,
[],
geometry=geometry,
)
# Tuple
geometry = (0, 50, 2, 52)
self.assertEquals(get_geometry_from_various([], geometry=geometry),
ref_geom)
# List
geometry = list(geometry)
self.assertEquals(get_geometry_from_various([], geometry=geometry),
ref_geom)
# List without 4 items
geometry.pop()
self.assertRaises(
TypeError,
get_geometry_from_various,
[],
geometry=geometry,
)
# WKT
geometry = ref_geom_as_wkt
self.assertEquals(get_geometry_from_various([], geometry=geometry),
ref_geom)
# Some other shapely geom
geometry = LineString([[0, 0], [1, 1]])
self.assertIsInstance(get_geometry_from_various([], geometry=geometry),
LineString)
def test_get_geometry_from_various_only_locations(self):
"""The search geometry can be set from a locations config file query"""
locations_config = self.dag.locations_config
# No query args
self.assertIsNone(get_geometry_from_various(locations_config))
# Bad query arg
# 'country' is the expected name here
self.assertIsNone(
get_geometry_from_various(locations_config, bad_query_arg="dummy"))
# France
geom_france = get_geometry_from_various(locations_config,
country="FRA")
self.assertIsInstance(geom_france, MultiPolygon)
self.assertEquals(len(geom_france), 3) # France + Guyana + Corsica
# Not defined
self.assertIsNone(
get_geometry_from_various(locations_config,
country="bad_query_value"))
def test_get_geometry_from_various_only_locations_regex(self):
"""The search geometry can be set from a locations config file query and a regex"""
locations_config = self.dag.locations_config
# Pakistan + Panama (each has a unique polygon) => Multypolygon of len 2
geom_regex_pa = get_geometry_from_various(locations_config,
country="PA[A-Z]")
self.assertIsInstance(geom_regex_pa, MultiPolygon)
self.assertEquals(len(geom_regex_pa), 2)
def test_get_geometry_from_various_geometry_and_locations(self):
"""The search geometry can be set from a given geometry and a locations config file query"""
geometry = {
"lonmin": 20,
"latmin": 50,
"lonmax": 22,
"latmax": 52,
}
locations_config = self.dag.locations_config
geom_combined = get_geometry_from_various(locations_config,
country="FRA",
geometry=geometry)
self.assertIsInstance(geom_combined, MultiPolygon)
self.assertEquals(
len(geom_combined),
4) # France + Guyana + Corsica + somewhere over Poland
geometry = {
"lonmin": 0,
"latmin": 50,
"lonmax": 2,
"latmax": 52,
}
geom_combined = get_geometry_from_various(locations_config,
country="FRA",
geometry=geometry)
self.assertIsInstance(geom_combined, MultiPolygon)
self.assertEquals(len(geom_combined),
3) # The bounding box overlaps with France inland
def test_rebuild_index(self):
"""Change eodag version and check that whoosh index is rebuilt"""
index_dir = os.path.join(self.dag.conf_dir, ".index")
index_dir_mtime = os.path.getmtime(index_dir)
self.assertNotEqual(self.dag.get_version(), "fake-version")
with mock.patch(
"eodag.api.core.EODataAccessGateway.get_version",
autospec=True,
return_value="fake-version",
):
self.assertEqual(self.dag.get_version(), "fake-version")
self.dag.build_index()
# check that index_dir has beeh re-created
self.assertNotEqual(os.path.getmtime(index_dir), index_dir_mtime)
class TestSearchStacStatic(unittest.TestCase):
def setUp(self):
super(TestSearchStacStatic, self).setUp()
self.dag = EODataAccessGateway()
self.cat_dir_path = self.root_cat = os.path.join(
TEST_RESOURCES_PATH, "stac")
# paths are relative in this test catalog, chdir needed
self._currentdir = os.getcwd()
os.chdir(self.cat_dir_path)
self.root_cat = os.path.join(self.cat_dir_path, "catalog.json")
self.root_cat_len = 5
self.child_cat = os.path.join(self.cat_dir_path, "country", "FRA",
"year", "2018", "2018.json")
self.child_cat_len = 2
self.stac_provider = "astraea_eod"
self.product_type = "S2_MSI_L1C"
self.extent_big = {
"lonmin": -55,
"lonmax": -53,
"latmin": 2,
"latmax": 5
}
self.extent_small = {
"lonmin": -55,
"lonmax": -54.5,
"latmin": 2,
"latmax": 2.5
}
def tearDown(self):
os.chdir(self._currentdir)
def test_search_stac_static_load_child(self):
"""load_stac_items from child catalog must provide items"""
items = self.dag.load_stac_items(self.child_cat,
recursive=True,
provider=self.stac_provider)
self.assertIsInstance(items, SearchResult)
self.assertEqual(len(items), self.child_cat_len)
self.assertEqual(items[0].provider, self.stac_provider)
# if no product_type is provided, product_type is None
self.assertIsNone(items[0].product_type)
def test_search_stac_static_load_root_not_recursive(self):
"""load_stac_items from root must provide an empty list when no recursive"""
items = self.dag.load_stac_items(self.root_cat,
recursive=False,
provider=self.stac_provider)
self.assertEqual(len(items), 0)
def test_search_stac_static_load_root_recursive(self):
"""load_stac_items from root must provide items when recursive"""
items = self.dag.load_stac_items(
self.root_cat,
recursive=True,
provider=self.stac_provider,
productType=self.product_type,
)
self.assertEqual(len(items), self.root_cat_len)
for item in items:
self.assertEqual(item.provider, self.stac_provider)
self.assertEqual(item.product_type, self.product_type)
def test_search_stac_static_crunch_filter_date(self):
"""load_stac_items from root and filter by date"""
items = self.dag.load_stac_items(
self.root_cat,
recursive=True,
provider=self.stac_provider,
productType=self.product_type,
)
filtered_items = items.crunch(
FilterDate({
"start": "2018-01-01",
"end": "2019-01-01"
}))
self.assertEqual(len(filtered_items), self.child_cat_len)
for item in filtered_items:
self.assertIn("2018",
item.properties["startTimeFromAscendingNode"])
def test_search_stac_static_crunch_filter_overlap(self):
"""load_stac_items from root and filter by overlap"""
# tests over extent_big search geometry
items = self.dag.load_stac_items(
self.root_cat,
recursive=True,
provider=self.stac_provider,
productType=self.product_type,
geom=self.extent_big,
)
self.assertEqual(len(items), self.root_cat_len)
filtered_items = items.crunch(FilterOverlap({"minimum_overlap": 10}),
geometry=self.extent_big)
self.assertEqual(len(filtered_items), 3)
filtered_items = items.crunch(FilterOverlap({"minimum_overlap": 100}),
geometry=self.extent_big)
self.assertEqual(len(filtered_items), 1)
filtered_items = items.crunch(FilterOverlap({"within": True}),
geometry=self.extent_big)
self.assertEqual(len(filtered_items), 1)
filtered_items = items.crunch(FilterOverlap({"contains": True}),
geometry=self.extent_big)
self.assertEqual(len(filtered_items), 0)
# tests over extent_small search geometry
items = self.dag.load_stac_items(
self.root_cat,
recursive=True,
provider=self.stac_provider,
productType=self.product_type,
geom=self.extent_small,
)
self.assertEqual(len(items), self.root_cat_len)
filtered_items = items.crunch(FilterOverlap({"contains": True}),
geometry=self.extent_small)
self.assertEqual(len(filtered_items), 1)
def test_search_stac_static_crunch_filter_property(self):
"""load_stac_items from root and filter by property"""
items = self.dag.load_stac_items(
self.root_cat,
recursive=True,
provider=self.stac_provider,
productType=self.product_type,
)
self.assertEqual(len(items), self.root_cat_len)
filtered_items = items.crunch(FilterProperty({"orbitNumber": 110}))
self.assertEqual(len(filtered_items), 3)
filtered_items = items.crunch(
FilterProperty({
"platformSerialIdentifier": "S2A",
"operator": "eq"
}))
self.assertEqual(len(filtered_items), 4)
filtered_items = items.crunch(
FilterProperty({
"cloudCover": 10,
"operator": "lt"
}))
self.assertEqual(len(filtered_items), 1)
def test_search_stac_static_crunch_filter_lastest_by_name(self):
"""load_stac_items from root and filter by name"""
items = self.dag.load_stac_items(
self.root_cat,
recursive=True,
provider=self.stac_provider,
productType=self.product_type,
)
self.assertEqual(len(items), self.root_cat_len)
filtered_items = items.crunch(
FilterLatestByName({
"name_pattern":
r"S2[AB]_MSIL1C_20(?P<tileid>\d{6}).*T21NY.*"
}))
self.assertEqual(len(filtered_items), 2)
class TestEODagDownloadCredentialsNotSet(unittest.TestCase):
"""MisconfiguredError must be raised when downloading one or several products
if the credentials are not set."""
# USGS not tested here because when the credentials are not set it already fails during the search.
# Given a GeoJSON Search Result it can however download the products.
@classmethod
def setUpClass(self):
default_conf_file = resource_filename(
"eodag", os.path.join("resources", "user_conf_template.yml"))
self.eodag = EODataAccessGateway(user_conf_file_path=default_conf_file)
def test_eodag_download_missing_credentials_theia(self):
search_resuls = os.path.join(TEST_RESOURCES_PATH,
"eodag_search_result_theia.geojson")
products = self.eodag.deserialize_and_register(search_resuls)
with self.assertRaises(MisconfiguredError):
self.eodag.download(products[0])
with self.assertRaises(MisconfiguredError):
self.eodag.download_all(products)
def test_eodag_download_missing_credentials_sobloo(self):
search_resuls = os.path.join(TEST_RESOURCES_PATH,
"eodag_search_result_sobloo.geojson")
products = self.eodag.deserialize_and_register(search_resuls)
with self.assertRaises(MisconfiguredError):
self.eodag.download(products[0])
with self.assertRaises(MisconfiguredError):
self.eodag.download_all(products)
def test_eodag_download_missing_credentials_peps(self):
search_resuls = os.path.join(TEST_RESOURCES_PATH,
"eodag_search_result_peps.geojson")
products = self.eodag.deserialize_and_register(search_resuls)
with self.assertRaises(MisconfiguredError):
self.eodag.download(products[0])
with self.assertRaises(MisconfiguredError):
self.eodag.download_all(products)
def test_eodag_download_missing_credentials_aws_eos(self):
search_resuls = os.path.join(TEST_RESOURCES_PATH,
"eodag_search_result_awseos.geojson")
products = self.eodag.deserialize_and_register(search_resuls)
with self.assertRaises(MisconfiguredError):
self.eodag.download(products[0])
with self.assertRaises(MisconfiguredError):
self.eodag.download_all(products)
def test_eodag_download_missing_credentials_creodias(self):
search_resuls = os.path.join(TEST_RESOURCES_PATH,
"eodag_search_result_creodias.geojson")
products = self.eodag.deserialize_and_register(search_resuls)
with self.assertRaises(MisconfiguredError):
self.eodag.download(products[0])
with self.assertRaises(MisconfiguredError):
self.eodag.download_all(products)
def test_eodag_download_missing_credentials_mundi(self):
search_resuls = os.path.join(TEST_RESOURCES_PATH,
"eodag_search_result_mundi.geojson")
products = self.eodag.deserialize_and_register(search_resuls)
with self.assertRaises(MisconfiguredError):
self.eodag.download(products[0])
with self.assertRaises(MisconfiguredError):
self.eodag.download_all(products)
def test_eodag_download_missing_credentials_onda(self):
search_resuls = os.path.join(TEST_RESOURCES_PATH,
"eodag_search_result_onda.geojson")
products = self.eodag.deserialize_and_register(search_resuls)
with self.assertRaises(MisconfiguredError):
self.eodag.download(products[0])
with self.assertRaises(MisconfiguredError):
self.eodag.download_all(products)
def setUpClass(self):
default_conf_file = resource_filename(
"eodag", os.path.join("resources", "user_conf_template.yml"))
self.eodag = EODataAccessGateway(user_conf_file_path=default_conf_file)
def setUpClass(cls):
cls.dag = EODataAccessGateway()
cls.conf_dir = os.path.join(os.path.expanduser("~"), ".config",
"eodag")
