def setUp(self): super(TestSearchStacStatic, self).setUp() self.dag = EODataAccessGateway() self.cat_dir_path = self.root_cat = os.path.join( TEST_RESOURCES_PATH, "stac") # paths are relative in this test catalog, chdir needed self._currentdir = os.getcwd() os.chdir(self.cat_dir_path) self.root_cat = os.path.join(self.cat_dir_path, "catalog.json") self.root_cat_len = 5 self.child_cat = os.path.join(self.cat_dir_path, "country", "FRA", "year", "2018", "2018.json") self.child_cat_len = 2 self.stac_provider = "astraea_eod" self.product_type = "S2_MSI_L1C" self.extent_big = { "lonmin": -55, "lonmax": -53, "latmin": 2, "latmax": 5 } self.extent_small = { "lonmin": -55, "lonmax": -54.5, "latmin": 2, "latmax": 2.5 }
def setUpClass(cls): # use tests/resources/user_conf.yml if exists else default file ~/.config/eodag/eodag.yml tests_user_conf = os.path.join(TEST_RESOURCES_PATH, "user_conf.yml") if os.path.isfile(tests_user_conf): cls.eodag = EODataAccessGateway(user_conf_file_path=os.path.join( TEST_RESOURCES_PATH, "user_conf.yml")) else: cls.eodag = EODataAccessGateway() # create TESTS_DOWNLOAD_PATH is not exists if not os.path.exists(TESTS_DOWNLOAD_PATH): os.makedirs(TESTS_DOWNLOAD_PATH) for provider, conf in cls.eodag.providers_config.items(): # Change download directory to TESTS_DOWNLOAD_PATH for tests if hasattr(conf, "download") and hasattr(conf.download, "outputs_prefix"): conf.download.outputs_prefix = TESTS_DOWNLOAD_PATH elif hasattr(conf, "api") and hasattr(conf.api, "outputs_prefix"): conf.api.outputs_prefix = TESTS_DOWNLOAD_PATH else: # no outputs_prefix found for provider pass # Force all providers implementing RestoSearch and defining how to retrieve # products by specifying the # location scheme to use https, enabling actual downloading of the product if (getattr(getattr(conf, "search", {}), "product_location_scheme", "https") == "file"): conf.search.product_location_scheme = "https" # Disable extraction try: # Case HTTPDownload plugin conf.download.extract = False except (KeyError, AttributeError): # case api plugin conf.api.extract = False
def test_end_to_end_good_apikey_wrong_credentials_aws_eos(self): # Setup # We retrieve correct credentials from the user_conf.yml file tests_user_conf = os.path.join(TEST_RESOURCES_PATH, "user_conf.yml") if not os.path.isfile(tests_user_conf): self.skipTest("user_conf.yml file with credentials not found.") # But we set the access key id and the secret to wrong values try: os.environ[ "EODAG__AWS_EOS__AUTH__CREDENTIALS__AWS_ACCESS_KEY_ID"] = "badaccessid" os.environ[ "EODAG__AWS_EOS__AUTH__CREDENTIALS__AWS_SECRET_ACCESS_KEY"] = "badsecret" eodag = EODataAccessGateway(user_conf_file_path=os.path.join( TEST_RESOURCES_PATH, "user_conf.yml")) eodag.set_preferred_provider(AWSEOS_SEARCH_ARGS[0]) results, nb_results = eodag.search( raise_errors=True, **dict( zip(["productType", "start", "end", "geom"], AWSEOS_SEARCH_ARGS[1:]))) self.assertGreater(len(results), 0) one_product = results[0] self.assertEqual(one_product.provider, AWSEOS_SEARCH_ARGS[0]) with self.assertRaises(AuthenticationError): self.eodag.download(one_product) # Teardown finally: os.environ.pop( "EODAG__AWS_EOS__AUTH__CREDENTIALS__AWS_ACCESS_KEY_ID") os.environ.pop( "EODAG__AWS_EOS__AUTH__CREDENTIALS__AWS_SECRET_ACCESS_KEY")
def test_core_object_prioritize_config_file_in_envvar(self): """The core object must use the config file pointed to by the EODAG_CFG_FILE env var""" # noqa os.environ["EODAG_CFG_FILE"] = os.path.join( TEST_RESOURCES_PATH, "file_config_override.yml") dag = EODataAccessGateway() # usgs priority is set to 5 in the test config overrides self.assertEqual(dag.get_preferred_provider(), ("usgs", 5)) # peps outputs prefix is set to /data self.assertEqual(dag.providers_config["peps"].download.outputs_prefix, "/data")
def test_update_providers_from_ext_plugin(self): """Load fake external plugin and check if it updates providers config""" default_providers_count = len(self.dag.providers_config) fakeplugin_location = os.path.join(TEST_RESOURCES_PATH, "fake_ext_plugin") # Create the fake entry point definition ep = pkg_resources.EntryPoint.parse( "FakePluginAPI = eodag_fakeplugin:FakePluginAPI") # Create a fake distribution to insert into the global working_set d = pkg_resources.Distribution( location=fakeplugin_location, project_name="eodag_fakeplugin", version="0.1", ) ep.dist = d # Add the mapping to the fake EntryPoint d._ep_map = {"eodag.plugins.api": {"FakePluginAPI": ep}} # Add the fake distribution to the global working_set pkg_resources.working_set.add(d, fakeplugin_location) # New EODataAccessGateway instance, check if new conf has been loaded self.dag = EODataAccessGateway() self.assertEqual(len(self.dag.providers_config), default_providers_count + 1)
def test_group_by_extent(self): geom_coords_1 = [[[89, 2], [90, 2], [90, 3], [89, 3], [89, 2]]] geom_coords_2 = [[[90, 3], [91, 3], [91, 4], [90, 4], [90, 3]]] geom_coords_3 = [[[92, 4], [92, 4], [92, 5], [91, 5], [91, 4]]] eo_geom1 = EOProduct.from_geojson( self._minimal_eoproduct_geojson_repr("1", geom_coords_1) ) eo_geom2 = EOProduct.from_geojson( self._minimal_eoproduct_geojson_repr("2", geom_coords_2) ) eo_geom3 = EOProduct.from_geojson( self._minimal_eoproduct_geojson_repr("3", geom_coords_3) ) first_search = SearchResult([eo_geom1]) second_search = SearchResult([eo_geom1, eo_geom2]) third_search = SearchResult([eo_geom1, eo_geom2, eo_geom3]) grouped_searches = EODataAccessGateway.group_by_extent( [first_search, second_search, third_search] ) # The returned value is a List[SearchResult] self.assertIsInstance(grouped_searches, list) self.assertTrue(all(isinstance(sr, SearchResult) for sr in grouped_searches)) # We expect three groups because we have given products that have # three different geometry bounds. self.assertEqual(len(grouped_searches), 3) # Given how the search results were constructed the resulting groups # must have these 3 different lengths. ss_len = [len(sr) for sr in grouped_searches] self.assertIn(1, ss_len) self.assertIn(2, ss_len) self.assertIn(3, ss_len)
def execution_involving_conf_dir(self, inspect=None): """Check that the path(s) inspected (str, list) are created after the instantation of EODataAccessGateway. If they were already there, rename them (.old), instantiate, check, delete the new files, and restore the existing files to there previous name.""" if inspect is not None: if isinstance(inspect, str): inspect = [inspect] conf_dir = os.path.join(os.path.expanduser("~"), ".config", "eodag") olds = [] currents = [] for inspected in inspect: old = current = os.path.join(conf_dir, inspected) if os.path.exists(current): old = os.path.join(conf_dir, "{}.old".format(inspected)) shutil.move(current, old) olds.append(old) currents.append(current) EODataAccessGateway() for old, current in zip(olds, currents): self.assertTrue(os.path.exists(current)) if old != current: try: shutil.rmtree(current) except OSError: os.unlink(current) shutil.move(old, current)
def test_core_object_open_index_if_exists(self, exists_in_mock, open_dir_mock): """The core object must use the existing index dir if any""" index_dir = os.path.join(self.conf_dir, ".index") if not os.path.exists(index_dir): makedirs(index_dir) EODataAccessGateway() open_dir_mock.assert_called_with(index_dir)
def test_core_object_prioritize_locations_file_in_envvar(self): """The core object must use the locations file pointed to by the EODAG_LOCS_CFG_FILE env var""" # noqa os.environ["EODAG_LOCS_CFG_FILE"] = os.path.join( TEST_RESOURCES_PATH, "file_locations_override.yml") dag = EODataAccessGateway() self.assertEqual( dag.locations_config, [dict(attr="dummyattr", name="dummyname", path="dummypath.shp")], )
def setUp(self): super(TestSearchStacStatic, self).setUp() self.dag = EODataAccessGateway() self.cat_dir_path = self.root_cat = os.path.join(TEST_RESOURCES_PATH, "stac") self.root_cat = os.path.join(self.cat_dir_path, "catalog.json") self.root_cat_len = 5 self.child_cat = os.path.join( self.cat_dir_path, "country", "FRA", "year", "2018", "2018.json" ) self.child_cat_len = 2 self.item = os.path.join( os.path.dirname(self.child_cat), "items", "S2A_MSIL1C_20181231T141041_N0207_R110_T21NYF_20181231T155050", "S2A_MSIL1C_20181231T141041_N0207_R110_T21NYF_20181231T155050.json", ) self.singlefile_cat = os.path.join(TEST_RESOURCES_PATH, "stac_singlefile.json") self.singlefile_cat_len = 5 self.stac_provider = "astraea_eod" self.product_type = "S2_MSI_L1C" self.extent_big = {"lonmin": -55, "lonmax": -53, "latmin": 2, "latmax": 5} self.extent_small = {"lonmin": -55, "lonmax": -54.5, "latmin": 2, "latmax": 2.5} self.static_stac_provider = "foo_static" self.dag.update_providers_config( f""" {self.static_stac_provider}: search: type: StaticStacSearch api_endpoint: {self.root_cat} products: GENERIC_PRODUCT_TYPE: productType: '{{productType}}' download: type: HTTPDownload base_uri: https://fake-endpoint flatten_top_dirs: True """ ) self.dag.set_preferred_provider(self.static_stac_provider)
def test_core_object_set_default_locations_config(self): """The core object must set the default locations config on instantiation""" # noqa dag = EODataAccessGateway() conf_dir = os.path.join(os.path.expanduser("~"), ".config", "eodag") default_shpfile = os.path.join(conf_dir, "shp", "ne_110m_admin_0_map_units.shp") self.assertIsInstance(dag.locations_config, list) self.assertEqual( dag.locations_config, [dict(attr="ADM0_A3_US", name="country", path=default_shpfile)], )
def setUpClass(cls): cls.dag = EODataAccessGateway() # Get a SearchResult obj with 2 S2_MSI_L1C peps products search_results_file = os.path.join(TEST_RESOURCES_PATH, "eodag_search_result_peps.geojson") with open(search_results_file, encoding="utf-8") as f: search_results_geojson = json.load(f) cls.search_results = SearchResult.from_geojson(search_results_geojson) cls.search_results_size = len(cls.search_results) # Change the id of these products, to emulate different products search_results_data_2 = deepcopy(cls.search_results.data) search_results_data_2[0].properties["id"] = "a" search_results_data_2[1].properties["id"] = "b" cls.search_results_2 = SearchResult(search_results_data_2) cls.search_results_size_2 = len(cls.search_results_2)
def execution_involving_conf_dir(self, inspect=None): if inspect is not None: conf_dir = os.path.join(os.path.expanduser("~"), ".config", "eodag") old = current = os.path.join(conf_dir, inspect) if os.path.exists(current): old = os.path.join(conf_dir, "{}.old".format(inspect)) shutil.move(current, old) EODataAccessGateway() self.assertTrue(os.path.exists(current)) if old != current: try: shutil.rmtree(current) except OSError: os.unlink(current) shutil.move(old, current)
def test_search_all_user_items_per_page(self, mocked_search_iter_page): """search_all must use the value of items_per_page provided by the user""" dag = EODataAccessGateway() dummy_provider_config = """ dummy_provider: search: type: QueryStringSearch api_endpoint: https://api.my_new_provider/search metadata_mapping: dummy: 'dummy' products: S2_MSI_L1C: productType: '{productType}' """ mocked_search_iter_page.return_value = (self.search_results for _ in range(1)) dag.update_providers_config(dummy_provider_config) dag.set_preferred_provider("dummy_provider") dag.search_all(productType="S2_MSI_L1C", items_per_page=7) self.assertEqual( mocked_search_iter_page.call_args[1]["items_per_page"], 7)
def test_search_iter_page_must_reset_next_attrs_if_next_mechanism( self, normalize_results, _request): """search_iter_page must reset the search plugin if the next mechanism is used""" # More specifically: next_page_url must be None and # config.pagination["next_page_url_tpl"] must be equal to its original value. _request.return_value.json.return_value = { "features": [], "links": [{ "rel": "next", "href": "url/to/next/page" }], } p1 = EOProduct("dummy", dict(geometry="POINT (0 0)", id="1")) p1.search_intersection = None p2 = EOProduct("dummy", dict(geometry="POINT (0 0)", id="2")) p2.search_intersection = None normalize_results.side_effect = [[p1], [p2]] dag = EODataAccessGateway() dummy_provider_config = """ dummy_provider: search: type: QueryStringSearch api_endpoint: https://api.my_new_provider/search pagination: next_page_url_tpl: 'dummy_next_page_url_tpl' next_page_url_key_path: '$.links[?(@.rel="next")].href' metadata_mapping: dummy: 'dummy' products: S2_MSI_L1C: productType: '{productType}' """ dag.update_providers_config(dummy_provider_config) dag.set_preferred_provider("dummy_provider") page_iterator = dag.search_iter_page(productType="S2_MSI_L1C") next(page_iterator) search_plugin = next( dag._plugins_manager.get_search_plugins(product_type="S2_MSI_L1C")) self.assertIsNone(search_plugin.next_page_url) self.assertEqual( search_plugin.config.pagination["next_page_url_tpl"], "dummy_next_page_url_tpl", )
def test_search_all_use_max_items_per_page(self, mocked_search_iter_page): """search_all must use the configured parameter max_items_per_page if available""" # noqa dag = EODataAccessGateway() dummy_provider_config = """ dummy_provider: search: type: QueryStringSearch api_endpoint: https://api.my_new_provider/search pagination: max_items_per_page: 2 metadata_mapping: dummy: 'dummy' products: S2_MSI_L1C: productType: '{productType}' """ mocked_search_iter_page.return_value = (self.search_results for _ in range(1)) dag.update_providers_config(dummy_provider_config) dag.set_preferred_provider("dummy_provider") dag.search_all(productType="S2_MSI_L1C") self.assertEqual( mocked_search_iter_page.call_args[1]["items_per_page"], 2)
def test_search_all_use_default_value(self, mocked_search_iter_page): """search_all must use the DEFAULT_MAX_ITEMS_PER_PAGE if the provider's one wasn't configured""" # noqa dag = EODataAccessGateway() dummy_provider_config = """ dummy_provider: search: type: QueryStringSearch api_endpoint: https://api.my_new_provider/search metadata_mapping: dummy: 'dummy' products: S2_MSI_L1C: productType: '{productType}' """ mocked_search_iter_page.return_value = (self.search_results for _ in range(1)) dag.update_providers_config(dummy_provider_config) dag.set_preferred_provider("dummy_provider") dag.search_all(productType="S2_MSI_L1C") self.assertEqual( mocked_search_iter_page.call_args[1]["items_per_page"], DEFAULT_MAX_ITEMS_PER_PAGE, )
def test_core_object_locations_file_not_found(self): """The core object must set the locations to an empty list when the file is not found""" # noqa dag = EODataAccessGateway(locations_conf_path="no_locations.yml") self.assertEqual(dag.locations_config, [])
def setUpClass(cls): tests_wrong_conf = os.path.join(TEST_RESOURCES_PATH, "wrong_credentials_conf.yml") cls.eodag = EODataAccessGateway(user_conf_file_path=tests_wrong_conf)
class TestCore(unittest.TestCase): SUPPORTED_PRODUCT_TYPES = { "L8_REFLECTANCE": ["theia"], "L57_REFLECTANCE": ["theia"], "L5_L1T": [], "L5_L1G": [], "L5_L1GT": [], "L7_L1G": [], "L7_L1T": [], "L7_L1GT": [], "L8_OLI_TIRS_C1L1": ["onda", "usgs", "aws_eos"], "S1_SAR_GRD": [ "peps", "sobloo", "onda", "wekeo", "mundi", "creodias", "aws_eos", ], "S1_SAR_OCN": ["peps", "sobloo", "onda", "creodias"], "S1_SAR_RAW": ["sobloo", "onda", "creodias"], "S1_SAR_SLC": ["peps", "sobloo", "onda", "wekeo", "mundi", "creodias"], "S2_MSI_L2A": ["onda", "mundi", "creodias", "peps", "aws_eos"], "S2_MSI_L2A_MAJA": ["theia"], "S2_MSI_L2B_MAJA_SNOW": ["theia"], "S2_MSI_L2B_MAJA_WATER": ["theia"], "S2_MSI_L3A_WASP": ["theia"], "S2_MSI_L1C": [ "aws_eos", "peps", "sobloo", "onda", "wekeo", "mundi", "creodias", ], "S3_ERR": ["peps", "onda", "wekeo", "creodias"], "S3_EFR": ["peps", "onda", "wekeo", "creodias"], "S3_LAN": ["peps", "sobloo", "onda", "wekeo", "creodias"], "S3_SRA": ["sobloo", "onda", "wekeo", "creodias"], "S3_SRA_BS": ["sobloo", "onda", "wekeo", "creodias"], "S3_SRA_A_BS": ["sobloo", "onda", "creodias"], "S3_WAT": ["onda", "wekeo", "creodias"], "S3_OLCI_L2LFR": ["peps", "onda", "wekeo", "creodias", "mundi"], "S3_OLCI_L2LRR": ["peps", "onda", "wekeo", "creodias"], "S3_SLSTR_L1RBT": ["peps", "onda", "wekeo", "creodias"], "S3_SLSTR_L2LST": ["peps", "onda", "wekeo", "creodias"], "PLD_PAN": ["theia"], "PLD_XS": ["theia"], "PLD_BUNDLE": ["theia"], "PLD_PANSHARPENED": ["theia"], "ES_FRS": [], "CBERS4_PAN10M_L2": ["aws_eos"], "CBERS4_PAN10M_L4": ["aws_eos"], "CBERS4_PAN5M_L2": ["aws_eos"], "CBERS4_PAN5M_L4": ["aws_eos"], "CBERS4_MUX_L2": ["aws_eos"], "CBERS4_MUX_L4": ["aws_eos"], "CBERS4_AWFI_L2": ["aws_eos"], "CBERS4_AWFI_L4": ["aws_eos"], "MODIS_MCD43A4": ["aws_eos"], "NAIP": ["aws_eos"], "SPOT_SWH": ["theia"], "SPOT_SWH_OLD": ["theia"], "SPOT5_SPIRIT": ["theia"], "VENUS_L1C": ["theia"], "VENUS_L2A_MAJA": ["theia"], "VENUS_L3A_MAJA": ["theia"], "OSO": ["theia"], } SUPPORTED_PROVIDERS = [ "peps", "usgs", "theia", "sobloo", "creodias", "mundi", "onda", "wekeo", "aws_eos", ] def setUp(self): super(TestCore, self).setUp() self.dag = EODataAccessGateway() def tearDown(self): super(TestCore, self).tearDown() for old in glob.glob1(self.dag.conf_dir, "*.old") + glob.glob1( self.dag.conf_dir, ".*.old"): old_path = os.path.join(self.dag.conf_dir, old) if os.path.exists(old_path): try: os.remove(old_path) except OSError: shutil.rmtree(old_path) if os.getenv("EODAG_CFG_FILE") is not None: os.environ.pop("EODAG_CFG_FILE") def test_supported_providers_in_unit_test(self): """Every provider must be referenced in the core unittest SUPPORTED_PROVIDERS class attribute""" # noqa for provider in self.dag.available_providers(): self.assertIn(provider, self.SUPPORTED_PROVIDERS) def test_supported_product_types_in_unit_test(self): """Every product type must be referenced in the core unit test SUPPORTED_PRODUCT_TYPES class attribute""" # noqa for product_type in self.dag.list_product_types(): self.assertIn(product_type["ID"], self.SUPPORTED_PRODUCT_TYPES.keys()) def test_list_product_types_ok(self): """Core api must correctly return the list of supported product types""" product_types = self.dag.list_product_types() self.assertIsInstance(product_types, list) for product_type in product_types: self.assertListProductTypesRightStructure(product_type) # There should be no repeated product type in the output self.assertEqual(len(product_types), len(set(pt["ID"] for pt in product_types))) def test_list_product_types_for_provider_ok(self): """Core api must correctly return the list of supported product types for a given provider""" # noqa for provider in self.SUPPORTED_PROVIDERS: product_types = self.dag.list_product_types(provider=provider) self.assertIsInstance(product_types, list) for product_type in product_types: self.assertListProductTypesRightStructure(product_type) self.assertIn(provider, self.SUPPORTED_PRODUCT_TYPES[product_type["ID"]]) def test_list_product_types_for_unsupported_provider(self): """Core api must raise UnsupportedProvider error for list_product_types with unsupported provider""" # noqa unsupported_provider = "a" self.assertRaises( UnsupportedProvider, self.dag.list_product_types, provider=unsupported_provider, ) def assertListProductTypesRightStructure(self, structure): """Helper method to verify that the structure given is a good result of EODataAccessGateway.list_product_types """ self.assertIsInstance(structure, dict) self.assertIn("ID", structure) self.assertIn("abstract", structure) self.assertIn("instrument", structure) self.assertIn("platform", structure) self.assertIn("platformSerialIdentifier", structure) self.assertIn("processingLevel", structure) self.assertIn("sensorType", structure) self.assertIn(structure["ID"], self.SUPPORTED_PRODUCT_TYPES) def test_core_object_creates_config_standard_location(self): """The core object must create a user config file in standard user config location on instantiation""" # noqa self.execution_involving_conf_dir(inspect="eodag.yml") def test_core_object_creates_index_if_not_exist(self): """The core object must create an index in user config directory""" self.execution_involving_conf_dir(inspect=".index") @mock.patch("eodag.api.core.open_dir", autospec=True) @mock.patch("eodag.api.core.exists_in", autospec=True, return_value=True) def test_core_object_open_index_if_exists(self, exists_in_mock, open_dir_mock): """The core object must use the existing index dir if any""" conf_dir = os.path.join(os.path.expanduser("~"), ".config", "eodag") index_dir = os.path.join(conf_dir, ".index") if not os.path.exists(index_dir): makedirs(index_dir) EODataAccessGateway() open_dir_mock.assert_called_with(index_dir) def test_core_object_prioritize_config_file_in_envvar(self): """The core object must use the config file pointed to by the EODAG_CFG_FILE env var""" # noqa os.environ["EODAG_CFG_FILE"] = os.path.join( TEST_RESOURCES_PATH, "file_config_override.yml") dag = EODataAccessGateway() # usgs priority is set to 5 in the test config overrides self.assertEqual(dag.get_preferred_provider(), ("usgs", 5)) # peps outputs prefix is set to /data self.assertEqual(dag.providers_config["peps"].download.outputs_prefix, "/data") def execution_involving_conf_dir(self, inspect=None): if inspect is not None: conf_dir = os.path.join(os.path.expanduser("~"), ".config", "eodag") old = current = os.path.join(conf_dir, inspect) if os.path.exists(current): old = os.path.join(conf_dir, "{}.old".format(inspect)) shutil.move(current, old) EODataAccessGateway() self.assertTrue(os.path.exists(current)) if old != current: try: shutil.rmtree(current) except OSError: os.unlink(current) shutil.move(old, current)
class TestSearchStacStatic(unittest.TestCase): def setUp(self): super(TestSearchStacStatic, self).setUp() self.dag = EODataAccessGateway() self.cat_dir_path = self.root_cat = os.path.join(TEST_RESOURCES_PATH, "stac") self.root_cat = os.path.join(self.cat_dir_path, "catalog.json") self.root_cat_len = 5 self.child_cat = os.path.join( self.cat_dir_path, "country", "FRA", "year", "2018", "2018.json" ) self.child_cat_len = 2 self.item = os.path.join( os.path.dirname(self.child_cat), "items", "S2A_MSIL1C_20181231T141041_N0207_R110_T21NYF_20181231T155050", "S2A_MSIL1C_20181231T141041_N0207_R110_T21NYF_20181231T155050.json", ) self.singlefile_cat = os.path.join(TEST_RESOURCES_PATH, "stac_singlefile.json") self.singlefile_cat_len = 5 self.stac_provider = "astraea_eod" self.product_type = "S2_MSI_L1C" self.extent_big = {"lonmin": -55, "lonmax": -53, "latmin": 2, "latmax": 5} self.extent_small = {"lonmin": -55, "lonmax": -54.5, "latmin": 2, "latmax": 2.5} self.static_stac_provider = "foo_static" self.dag.update_providers_config( f""" {self.static_stac_provider}: search: type: StaticStacSearch api_endpoint: {self.root_cat} products: GENERIC_PRODUCT_TYPE: productType: '{{productType}}' download: type: HTTPDownload base_uri: https://fake-endpoint flatten_top_dirs: True """ ) self.dag.set_preferred_provider(self.static_stac_provider) def test_search_stac_static_load_child(self): """load_stac_items from child catalog must provide items""" items = self.dag.load_stac_items( self.child_cat, recursive=True, provider=self.stac_provider ) self.assertIsInstance(items, SearchResult) self.assertEqual(len(items), self.child_cat_len) self.assertEqual(items[0].provider, self.stac_provider) # if no product_type is provided, product_type is None self.assertIsNone(items[0].product_type) def test_search_stac_static_load_root_not_recursive(self): """load_stac_items from root must provide an empty list when no recursive""" items = self.dag.load_stac_items( self.root_cat, recursive=False, provider=self.stac_provider ) self.assertEqual(len(items), 0) def test_search_stac_static_load_root_recursive(self): """load_stac_items from root must provide items when recursive""" items = self.dag.load_stac_items( self.root_cat, recursive=True, provider=self.stac_provider, productType=self.product_type, ) self.assertEqual(len(items), self.root_cat_len) for item in items: self.assertEqual(item.provider, self.stac_provider) self.assertEqual(item.product_type, self.product_type) def test_search_stac_static(self): """Use StaticStacSearch plugin to search all items""" items, nb = self.dag.search() self.assertEqual(len(items), self.root_cat_len) self.assertEqual(nb, self.root_cat_len) for item in items: self.assertEqual(item.provider, self.static_stac_provider) def test_search_stac_static_load_item(self): """load_stac_items from a single item must provide it""" item = self.dag.load_stac_items(self.item, provider=self.stac_provider) self.assertIsInstance(item, SearchResult) self.assertEqual(len(item), 1) self.assertEqual(item[0].provider, self.stac_provider) # if no product_type is provided, product_type is None self.assertIsNone(item[0].product_type) def test_search_stac_static_load_item_updated_provider(self): """load_stac_items from a single item using updated provider""" item = self.dag.load_stac_items(self.item, provider=self.stac_provider) self.assertEqual(item[0].properties["license"], "proprietary") self.assertEqual(item[0].properties["platform"], "S2ST") self.assertEqual(item[0].properties["orbitDirection"], "descending") self.assertNotIn("foo", item[0].properties) # fake provider with mixed metadata_mapping self.dag.update_providers_config( """ fake_provider: search: type: StacSearch api_endpoint: 'https://fake-endpoint' metadata_mapping: license: '{platform}' foo: '{orbitDirection}' products: GENERIC_PRODUCT_TYPE: productType: '{productType}' download: type: HTTPDownload base_uri: 'https://fake-uri' """ ) item = self.dag.load_stac_items( self.item, provider="fake_provider", raise_errors=True ) self.assertEqual(item[0].properties["platform"], "S2ST") self.assertEqual(item[0].properties["license"], "S2ST") self.assertEqual(item[0].properties["orbitDirection"], "descending") self.assertIn("foo", item[0].properties) self.assertEqual(item[0].properties["foo"], "descending") def test_search_stac_static_load_singlefile_catalog(self): """load_stac_items from child catalog must provide items""" items = self.dag.load_stac_items( self.singlefile_cat, provider=self.stac_provider ) self.assertIsInstance(items, SearchResult) self.assertEqual(len(items), self.singlefile_cat_len) self.assertEqual(items[0].provider, self.stac_provider) # if no product_type is provided, product_type is None self.assertIsNone(items[0].product_type) def test_search_stac_static_crunch_filter_date(self): """load_stac_items from root and filter by date""" items = self.dag.load_stac_items( self.root_cat, recursive=True, provider=self.stac_provider, productType=self.product_type, ) filtered_items = items.crunch( FilterDate({"start": "2018-01-01", "end": "2019-01-01"}) ) self.assertEqual(len(filtered_items), self.child_cat_len) for item in filtered_items: self.assertIn("2018", item.properties["startTimeFromAscendingNode"]) def test_search_stac_static_by_date(self): """Use StaticStacSearch plugin to search by date""" filtered_items, nb = self.dag.search(start="2018-01-01", end="2019-01-01") self.assertEqual(len(filtered_items), self.child_cat_len) self.assertEqual(nb, self.child_cat_len) for item in filtered_items: self.assertIn("2018", item.properties["startTimeFromAscendingNode"]) def test_search_stac_static_crunch_filter_overlap(self): """load_stac_items from root and filter by overlap""" # tests over extent_big search geometry items = self.dag.load_stac_items( self.root_cat, recursive=True, provider=self.stac_provider, productType=self.product_type, geom=self.extent_big, ) self.assertEqual(len(items), self.root_cat_len) filtered_items = items.crunch( FilterOverlap({"minimum_overlap": 10}), geometry=self.extent_big ) self.assertEqual(len(filtered_items), 3) filtered_items = items.crunch( FilterOverlap({"minimum_overlap": 100}), geometry=self.extent_big ) self.assertEqual(len(filtered_items), 1) filtered_items = items.crunch( FilterOverlap({"within": True}), geometry=self.extent_big ) self.assertEqual(len(filtered_items), 1) filtered_items = items.crunch( FilterOverlap({"contains": True}), geometry=self.extent_big ) self.assertEqual(len(filtered_items), 0) filtered_items = items.crunch( FilterOverlap({"intersects": True}), geometry=self.extent_big ) self.assertEqual(len(filtered_items), 3) # tests over extent_small search geometry items = self.dag.load_stac_items( self.root_cat, recursive=True, provider=self.stac_provider, productType=self.product_type, geom=self.extent_small, ) self.assertEqual(len(items), self.root_cat_len) filtered_items = items.crunch( FilterOverlap({"contains": True}), geometry=self.extent_small ) self.assertEqual(len(filtered_items), 1) def test_search_stac_static_by_geom(self): """Use StaticStacSearch plugin to search by geometry""" items, nb = self.dag.search( geom=self.extent_big, ) self.assertEqual(len(items), 3) self.assertEqual(nb, 3) def test_search_stac_static_crunch_filter_property(self): """load_stac_items from root and filter by property""" items = self.dag.load_stac_items( self.root_cat, recursive=True, provider=self.stac_provider, productType=self.product_type, ) self.assertEqual(len(items), self.root_cat_len) filtered_items = items.crunch(FilterProperty({"orbitNumber": 110})) self.assertEqual(len(filtered_items), 3) filtered_items = items.crunch( FilterProperty({"platformSerialIdentifier": "S2A", "operator": "eq"}) ) self.assertEqual(len(filtered_items), 4) filtered_items = items.crunch( FilterProperty({"cloudCover": 10, "operator": "lt"}) ) self.assertEqual(len(filtered_items), 1) def test_search_stac_static_by_property(self): """Use StaticStacSearch plugin to search by property""" items, nb = self.dag.search(orbitNumber=110) self.assertEqual(len(items), 3) self.assertEqual(nb, 3) def test_search_stac_static_by_cloudcover(self): """Use StaticStacSearch plugin to search by cloud cover""" items, nb = self.dag.search(cloudCover=10) self.assertEqual(len(items), 1) self.assertEqual(nb, 1) def test_search_stac_static_crunch_filter_lastest_by_name(self): """load_stac_items from root and filter by name""" items = self.dag.load_stac_items( self.root_cat, recursive=True, provider=self.stac_provider, productType=self.product_type, ) self.assertEqual(len(items), self.root_cat_len) filtered_items = items.crunch( FilterLatestByName( {"name_pattern": r"S2[AB]_MSIL1C_20(?P<tileid>\d{6}).*T21NY.*"} ) ) self.assertEqual(len(filtered_items), 2)
def setUpClass(cls): cls.dag = EODataAccessGateway()
def setUp(self): super(TestCore, self).setUp() self.dag = EODataAccessGateway()
def setUp(self): super(TestCoreInvolvingConfDir, self).setUp() self.dag = EODataAccessGateway()
def setUp(self): super(TestExternalPluginConfig, self).setUp() self.dag = EODataAccessGateway()
class TestCore(unittest.TestCase): SUPPORTED_PRODUCT_TYPES = { "L8_REFLECTANCE": ["theia"], "L57_REFLECTANCE": ["theia"], "L8_OLI_TIRS_C1L1": ["onda", "usgs", "aws_eos", "astraea_eod"], "S1_SAR_GRD": [ "peps", "sobloo", "onda", "wekeo", "mundi", "creodias", "aws_eos", ], "S1_SAR_OCN": ["peps", "sobloo", "onda", "creodias"], "S1_SAR_RAW": ["sobloo", "onda", "creodias"], "S1_SAR_SLC": ["peps", "sobloo", "onda", "wekeo", "mundi", "creodias"], "S2_MSI_L2A": [ "onda", "mundi", "creodias", "peps", "aws_eos", "sobloo", "astraea_eod", ], "S2_MSI_L2A_MAJA": ["theia"], "S2_MSI_L2B_MAJA_SNOW": ["theia"], "S2_MSI_L2B_MAJA_WATER": ["theia"], "S2_MSI_L3A_WASP": ["theia"], "S2_MSI_L1C": [ "aws_eos", "peps", "sobloo", "onda", "wekeo", "mundi", "creodias", "astraea_eod", ], "S3_ERR": ["peps", "onda", "wekeo", "creodias"], "S3_EFR": ["peps", "onda", "wekeo", "creodias"], "S3_LAN": ["peps", "onda", "wekeo", "creodias"], "S3_SRA": ["onda", "wekeo", "creodias"], "S3_SRA_BS": ["onda", "wekeo", "creodias"], "S3_SRA_A_BS": ["onda", "creodias"], "S3_WAT": ["onda", "wekeo", "creodias"], "S3_OLCI_L2LFR": ["peps", "onda", "wekeo", "creodias", "mundi"], "S3_OLCI_L2LRR": ["peps", "onda", "wekeo", "creodias"], "S3_SLSTR_L1RBT": ["peps", "onda", "wekeo", "creodias"], "S3_SLSTR_L2LST": ["peps", "onda", "wekeo", "creodias"], "PLD_PAN": ["theia"], "PLD_XS": ["theia"], "PLD_BUNDLE": ["theia"], "PLD_PANSHARPENED": ["theia"], "CBERS4_PAN10M_L2": ["aws_eos"], "CBERS4_PAN10M_L4": ["aws_eos"], "CBERS4_PAN5M_L2": ["aws_eos"], "CBERS4_PAN5M_L4": ["aws_eos"], "CBERS4_MUX_L2": ["aws_eos"], "CBERS4_MUX_L4": ["aws_eos"], "CBERS4_AWFI_L2": ["aws_eos"], "CBERS4_AWFI_L4": ["aws_eos"], "MODIS_MCD43A4": ["aws_eos", "astraea_eod"], "NAIP": ["aws_eos", "astraea_eod"], "SPOT_SWH": ["theia"], "SPOT_SWH_OLD": ["theia"], "SPOT5_SPIRIT": ["theia"], "VENUS_L1C": ["theia"], "VENUS_L2A_MAJA": ["theia"], "VENUS_L3A_MAJA": ["theia"], "OSO": ["theia"], GENERIC_PRODUCT_TYPE: [ "theia", "peps", "sobloo", "onda", "mundi", "creodias", "astraea_eod", ], } SUPPORTED_PROVIDERS = [ "peps", "usgs", "theia", "sobloo", "creodias", "mundi", "onda", "wekeo", "aws_eos", "astraea_eod", ] def setUp(self): super(TestCore, self).setUp() self.dag = EODataAccessGateway() def tearDown(self): super(TestCore, self).tearDown() for old in glob.glob1(self.dag.conf_dir, "*.old") + glob.glob1( self.dag.conf_dir, ".*.old"): old_path = os.path.join(self.dag.conf_dir, old) if os.path.exists(old_path): try: os.remove(old_path) except OSError: shutil.rmtree(old_path) if os.getenv("EODAG_CFG_FILE") is not None: os.environ.pop("EODAG_CFG_FILE") if os.getenv("EODAG_LOCS_CFG_FILE") is not None: os.environ.pop("EODAG_LOCS_CFG_FILE") def test_supported_providers_in_unit_test(self): """Every provider must be referenced in the core unittest SUPPORTED_PROVIDERS class attribute""" # noqa for provider in self.dag.available_providers(): self.assertIn(provider, self.SUPPORTED_PROVIDERS) def test_supported_product_types_in_unit_test(self): """Every product type must be referenced in the core unit test SUPPORTED_PRODUCT_TYPES class attribute""" # noqa for product_type in self.dag.list_product_types(): self.assertIn(product_type["ID"], self.SUPPORTED_PRODUCT_TYPES.keys()) def test_list_product_types_ok(self): """Core api must correctly return the list of supported product types""" product_types = self.dag.list_product_types() self.assertIsInstance(product_types, list) for product_type in product_types: self.assertListProductTypesRightStructure(product_type) # There should be no repeated product type in the output self.assertEqual(len(product_types), len(set(pt["ID"] for pt in product_types))) def test_list_product_types_for_provider_ok(self): """Core api must correctly return the list of supported product types for a given provider""" # noqa for provider in self.SUPPORTED_PROVIDERS: product_types = self.dag.list_product_types(provider=provider) self.assertIsInstance(product_types, list) for product_type in product_types: self.assertListProductTypesRightStructure(product_type) self.assertIn(provider, self.SUPPORTED_PRODUCT_TYPES[product_type["ID"]]) def test_list_product_types_for_unsupported_provider(self): """Core api must raise UnsupportedProvider error for list_product_types with unsupported provider""" # noqa unsupported_provider = "a" self.assertRaises( UnsupportedProvider, self.dag.list_product_types, provider=unsupported_provider, ) def assertListProductTypesRightStructure(self, structure): """Helper method to verify that the structure given is a good result of EODataAccessGateway.list_product_types """ self.assertIsInstance(structure, dict) self.assertIn("ID", structure) self.assertIn("abstract", structure) self.assertIn("instrument", structure) self.assertIn("platform", structure) self.assertIn("platformSerialIdentifier", structure) self.assertIn("processingLevel", structure) self.assertIn("sensorType", structure) self.assertIn(structure["ID"], self.SUPPORTED_PRODUCT_TYPES) def test_core_object_creates_config_standard_location(self): """The core object must create a user config file in standard user config location on instantiation""" # noqa self.execution_involving_conf_dir(inspect="eodag.yml") def test_core_object_creates_index_if_not_exist(self): """The core object must create an index in user config directory""" self.execution_involving_conf_dir(inspect=".index") @mock.patch("eodag.api.core.open_dir", autospec=True) @mock.patch("eodag.api.core.exists_in", autospec=True, return_value=True) def test_core_object_open_index_if_exists(self, exists_in_mock, open_dir_mock): """The core object must use the existing index dir if any""" conf_dir = os.path.join(os.path.expanduser("~"), ".config", "eodag") index_dir = os.path.join(conf_dir, ".index") if not os.path.exists(index_dir): makedirs(index_dir) EODataAccessGateway() open_dir_mock.assert_called_with(index_dir) def test_core_object_prioritize_config_file_in_envvar(self): """The core object must use the config file pointed to by the EODAG_CFG_FILE env var""" # noqa os.environ["EODAG_CFG_FILE"] = os.path.join( TEST_RESOURCES_PATH, "file_config_override.yml") dag = EODataAccessGateway() # usgs priority is set to 5 in the test config overrides self.assertEqual(dag.get_preferred_provider(), ("usgs", 5)) # peps outputs prefix is set to /data self.assertEqual(dag.providers_config["peps"].download.outputs_prefix, "/data") def execution_involving_conf_dir(self, inspect=None): """Check that the path(s) inspected (str, list) are created after the instantation of EODataAccessGateway. If they were already there, rename them (.old), instantiate, check, delete the new files, and restore the existing files to there previous name.""" if inspect is not None: if isinstance(inspect, str): inspect = [inspect] conf_dir = os.path.join(os.path.expanduser("~"), ".config", "eodag") olds = [] currents = [] for inspected in inspect: old = current = os.path.join(conf_dir, inspected) if os.path.exists(current): old = os.path.join(conf_dir, "{}.old".format(inspected)) shutil.move(current, old) olds.append(old) currents.append(current) EODataAccessGateway() for old, current in zip(olds, currents): self.assertTrue(os.path.exists(current)) if old != current: try: shutil.rmtree(current) except OSError: os.unlink(current) shutil.move(old, current) def test_core_object_creates_locations_standard_location(self): """The core object must create a locations config file and a shp dir in standard user config location on instantiation""" # noqa self.execution_involving_conf_dir(inspect=["locations.yml", "shp"]) def test_core_object_set_default_locations_config(self): """The core object must set the default locations config on instantiation""" # noqa dag = EODataAccessGateway() conf_dir = os.path.join(os.path.expanduser("~"), ".config", "eodag") default_shpfile = os.path.join(conf_dir, "shp", "ne_110m_admin_0_map_units.shp") self.assertIsInstance(dag.locations_config, list) self.assertEqual( dag.locations_config, [dict(attr="ADM0_A3_US", name="country", path=default_shpfile)], ) def test_core_object_locations_file_not_found(self): """The core object must set the locations to an empty list when the file is not found""" # noqa dag = EODataAccessGateway(locations_conf_path="no_locations.yml") self.assertEqual(dag.locations_config, []) def test_core_object_prioritize_locations_file_in_envvar(self): """The core object must use the locations file pointed to by the EODEODAG_LOCS_CFG_FILEAG_CFG_FILE env var""" # noqa os.environ["EODAG_LOCS_CFG_FILE"] = os.path.join( TEST_RESOURCES_PATH, "file_locations_override.yml") dag = EODataAccessGateway() self.assertEqual( dag.locations_config, [dict(attr="dummyattr", name="dummyname", path="dummypath.shp")], ) def test_get_geometry_from_various_no_locations(self): """The search geometry can be set from a dict, list, tuple, WKT string or shapely geom""" ref_geom_as_wkt = "POLYGON ((0 50, 0 52, 2 52, 2 50, 0 50))" ref_geom = wkt.loads(ref_geom_as_wkt) # Good dict geometry = { "lonmin": 0, "latmin": 50, "lonmax": 2, "latmax": 52, } self.assertEquals(get_geometry_from_various([], geometry=geometry), ref_geom) # Bad dict with a missing key del geometry["lonmin"] self.assertRaises( TypeError, get_geometry_from_various, [], geometry=geometry, ) # Tuple geometry = (0, 50, 2, 52) self.assertEquals(get_geometry_from_various([], geometry=geometry), ref_geom) # List geometry = list(geometry) self.assertEquals(get_geometry_from_various([], geometry=geometry), ref_geom) # List without 4 items geometry.pop() self.assertRaises( TypeError, get_geometry_from_various, [], geometry=geometry, ) # WKT geometry = ref_geom_as_wkt self.assertEquals(get_geometry_from_various([], geometry=geometry), ref_geom) # Some other shapely geom geometry = LineString([[0, 0], [1, 1]]) self.assertIsInstance(get_geometry_from_various([], geometry=geometry), LineString) def test_get_geometry_from_various_only_locations(self): """The search geometry can be set from a locations config file query""" locations_config = self.dag.locations_config # No query args self.assertIsNone(get_geometry_from_various(locations_config)) # Bad query arg # 'country' is the expected name here self.assertIsNone( get_geometry_from_various(locations_config, bad_query_arg="dummy")) # France geom_france = get_geometry_from_various(locations_config, country="FRA") self.assertIsInstance(geom_france, MultiPolygon) self.assertEquals(len(geom_france), 3) # France + Guyana + Corsica # Not defined self.assertIsNone( get_geometry_from_various(locations_config, country="bad_query_value")) def test_get_geometry_from_various_only_locations_regex(self): """The search geometry can be set from a locations config file query and a regex""" locations_config = self.dag.locations_config # Pakistan + Panama (each has a unique polygon) => Multypolygon of len 2 geom_regex_pa = get_geometry_from_various(locations_config, country="PA[A-Z]") self.assertIsInstance(geom_regex_pa, MultiPolygon) self.assertEquals(len(geom_regex_pa), 2) def test_get_geometry_from_various_geometry_and_locations(self): """The search geometry can be set from a given geometry and a locations config file query""" geometry = { "lonmin": 20, "latmin": 50, "lonmax": 22, "latmax": 52, } locations_config = self.dag.locations_config geom_combined = get_geometry_from_various(locations_config, country="FRA", geometry=geometry) self.assertIsInstance(geom_combined, MultiPolygon) self.assertEquals( len(geom_combined), 4) # France + Guyana + Corsica + somewhere over Poland geometry = { "lonmin": 0, "latmin": 50, "lonmax": 2, "latmax": 52, } geom_combined = get_geometry_from_various(locations_config, country="FRA", geometry=geometry) self.assertIsInstance(geom_combined, MultiPolygon) self.assertEquals(len(geom_combined), 3) # The bounding box overlaps with France inland def test_rebuild_index(self): """Change eodag version and check that whoosh index is rebuilt""" index_dir = os.path.join(self.dag.conf_dir, ".index") index_dir_mtime = os.path.getmtime(index_dir) self.assertNotEqual(self.dag.get_version(), "fake-version") with mock.patch( "eodag.api.core.EODataAccessGateway.get_version", autospec=True, return_value="fake-version", ): self.assertEqual(self.dag.get_version(), "fake-version") self.dag.build_index() # check that index_dir has beeh re-created self.assertNotEqual(os.path.getmtime(index_dir), index_dir_mtime)
class TestSearchStacStatic(unittest.TestCase): def setUp(self): super(TestSearchStacStatic, self).setUp() self.dag = EODataAccessGateway() self.cat_dir_path = self.root_cat = os.path.join( TEST_RESOURCES_PATH, "stac") # paths are relative in this test catalog, chdir needed self._currentdir = os.getcwd() os.chdir(self.cat_dir_path) self.root_cat = os.path.join(self.cat_dir_path, "catalog.json") self.root_cat_len = 5 self.child_cat = os.path.join(self.cat_dir_path, "country", "FRA", "year", "2018", "2018.json") self.child_cat_len = 2 self.stac_provider = "astraea_eod" self.product_type = "S2_MSI_L1C" self.extent_big = { "lonmin": -55, "lonmax": -53, "latmin": 2, "latmax": 5 } self.extent_small = { "lonmin": -55, "lonmax": -54.5, "latmin": 2, "latmax": 2.5 } def tearDown(self): os.chdir(self._currentdir) def test_search_stac_static_load_child(self): """load_stac_items from child catalog must provide items""" items = self.dag.load_stac_items(self.child_cat, recursive=True, provider=self.stac_provider) self.assertIsInstance(items, SearchResult) self.assertEqual(len(items), self.child_cat_len) self.assertEqual(items[0].provider, self.stac_provider) # if no product_type is provided, product_type is None self.assertIsNone(items[0].product_type) def test_search_stac_static_load_root_not_recursive(self): """load_stac_items from root must provide an empty list when no recursive""" items = self.dag.load_stac_items(self.root_cat, recursive=False, provider=self.stac_provider) self.assertEqual(len(items), 0) def test_search_stac_static_load_root_recursive(self): """load_stac_items from root must provide items when recursive""" items = self.dag.load_stac_items( self.root_cat, recursive=True, provider=self.stac_provider, productType=self.product_type, ) self.assertEqual(len(items), self.root_cat_len) for item in items: self.assertEqual(item.provider, self.stac_provider) self.assertEqual(item.product_type, self.product_type) def test_search_stac_static_crunch_filter_date(self): """load_stac_items from root and filter by date""" items = self.dag.load_stac_items( self.root_cat, recursive=True, provider=self.stac_provider, productType=self.product_type, ) filtered_items = items.crunch( FilterDate({ "start": "2018-01-01", "end": "2019-01-01" })) self.assertEqual(len(filtered_items), self.child_cat_len) for item in filtered_items: self.assertIn("2018", item.properties["startTimeFromAscendingNode"]) def test_search_stac_static_crunch_filter_overlap(self): """load_stac_items from root and filter by overlap""" # tests over extent_big search geometry items = self.dag.load_stac_items( self.root_cat, recursive=True, provider=self.stac_provider, productType=self.product_type, geom=self.extent_big, ) self.assertEqual(len(items), self.root_cat_len) filtered_items = items.crunch(FilterOverlap({"minimum_overlap": 10}), geometry=self.extent_big) self.assertEqual(len(filtered_items), 3) filtered_items = items.crunch(FilterOverlap({"minimum_overlap": 100}), geometry=self.extent_big) self.assertEqual(len(filtered_items), 1) filtered_items = items.crunch(FilterOverlap({"within": True}), geometry=self.extent_big) self.assertEqual(len(filtered_items), 1) filtered_items = items.crunch(FilterOverlap({"contains": True}), geometry=self.extent_big) self.assertEqual(len(filtered_items), 0) # tests over extent_small search geometry items = self.dag.load_stac_items( self.root_cat, recursive=True, provider=self.stac_provider, productType=self.product_type, geom=self.extent_small, ) self.assertEqual(len(items), self.root_cat_len) filtered_items = items.crunch(FilterOverlap({"contains": True}), geometry=self.extent_small) self.assertEqual(len(filtered_items), 1) def test_search_stac_static_crunch_filter_property(self): """load_stac_items from root and filter by property""" items = self.dag.load_stac_items( self.root_cat, recursive=True, provider=self.stac_provider, productType=self.product_type, ) self.assertEqual(len(items), self.root_cat_len) filtered_items = items.crunch(FilterProperty({"orbitNumber": 110})) self.assertEqual(len(filtered_items), 3) filtered_items = items.crunch( FilterProperty({ "platformSerialIdentifier": "S2A", "operator": "eq" })) self.assertEqual(len(filtered_items), 4) filtered_items = items.crunch( FilterProperty({ "cloudCover": 10, "operator": "lt" })) self.assertEqual(len(filtered_items), 1) def test_search_stac_static_crunch_filter_lastest_by_name(self): """load_stac_items from root and filter by name""" items = self.dag.load_stac_items( self.root_cat, recursive=True, provider=self.stac_provider, productType=self.product_type, ) self.assertEqual(len(items), self.root_cat_len) filtered_items = items.crunch( FilterLatestByName({ "name_pattern": r"S2[AB]_MSIL1C_20(?P<tileid>\d{6}).*T21NY.*" })) self.assertEqual(len(filtered_items), 2)
class TestEODagDownloadCredentialsNotSet(unittest.TestCase): """MisconfiguredError must be raised when downloading one or several products if the credentials are not set.""" # USGS not tested here because when the credentials are not set it already fails during the search. # Given a GeoJSON Search Result it can however download the products. @classmethod def setUpClass(self): default_conf_file = resource_filename( "eodag", os.path.join("resources", "user_conf_template.yml")) self.eodag = EODataAccessGateway(user_conf_file_path=default_conf_file) def test_eodag_download_missing_credentials_theia(self): search_resuls = os.path.join(TEST_RESOURCES_PATH, "eodag_search_result_theia.geojson") products = self.eodag.deserialize_and_register(search_resuls) with self.assertRaises(MisconfiguredError): self.eodag.download(products[0]) with self.assertRaises(MisconfiguredError): self.eodag.download_all(products) def test_eodag_download_missing_credentials_sobloo(self): search_resuls = os.path.join(TEST_RESOURCES_PATH, "eodag_search_result_sobloo.geojson") products = self.eodag.deserialize_and_register(search_resuls) with self.assertRaises(MisconfiguredError): self.eodag.download(products[0]) with self.assertRaises(MisconfiguredError): self.eodag.download_all(products) def test_eodag_download_missing_credentials_peps(self): search_resuls = os.path.join(TEST_RESOURCES_PATH, "eodag_search_result_peps.geojson") products = self.eodag.deserialize_and_register(search_resuls) with self.assertRaises(MisconfiguredError): self.eodag.download(products[0]) with self.assertRaises(MisconfiguredError): self.eodag.download_all(products) def test_eodag_download_missing_credentials_aws_eos(self): search_resuls = os.path.join(TEST_RESOURCES_PATH, "eodag_search_result_awseos.geojson") products = self.eodag.deserialize_and_register(search_resuls) with self.assertRaises(MisconfiguredError): self.eodag.download(products[0]) with self.assertRaises(MisconfiguredError): self.eodag.download_all(products) def test_eodag_download_missing_credentials_creodias(self): search_resuls = os.path.join(TEST_RESOURCES_PATH, "eodag_search_result_creodias.geojson") products = self.eodag.deserialize_and_register(search_resuls) with self.assertRaises(MisconfiguredError): self.eodag.download(products[0]) with self.assertRaises(MisconfiguredError): self.eodag.download_all(products) def test_eodag_download_missing_credentials_mundi(self): search_resuls = os.path.join(TEST_RESOURCES_PATH, "eodag_search_result_mundi.geojson") products = self.eodag.deserialize_and_register(search_resuls) with self.assertRaises(MisconfiguredError): self.eodag.download(products[0]) with self.assertRaises(MisconfiguredError): self.eodag.download_all(products) def test_eodag_download_missing_credentials_onda(self): search_resuls = os.path.join(TEST_RESOURCES_PATH, "eodag_search_result_onda.geojson") products = self.eodag.deserialize_and_register(search_resuls) with self.assertRaises(MisconfiguredError): self.eodag.download(products[0]) with self.assertRaises(MisconfiguredError): self.eodag.download_all(products)
def setUpClass(self): default_conf_file = resource_filename( "eodag", os.path.join("resources", "user_conf_template.yml")) self.eodag = EODataAccessGateway(user_conf_file_path=default_conf_file)
def setUpClass(cls): cls.dag = EODataAccessGateway() cls.conf_dir = os.path.join(os.path.expanduser("~"), ".config", "eodag")