class TestCore(unittest.TestCase): SUPPORTED_PRODUCT_TYPES = { "L8_REFLECTANCE": ["theia"], "L57_REFLECTANCE": ["theia"], "L8_OLI_TIRS_C1L1": ["onda", "usgs", "aws_eos", "astraea_eod"], "S1_SAR_GRD": [ "peps", "sobloo", "onda", "wekeo", "mundi", "creodias", "aws_eos", ], "S1_SAR_OCN": ["peps", "sobloo", "onda", "creodias"], "S1_SAR_RAW": ["sobloo", "onda", "creodias"], "S1_SAR_SLC": ["peps", "sobloo", "onda", "wekeo", "mundi", "creodias"], "S2_MSI_L2A": [ "onda", "mundi", "creodias", "peps", "aws_eos", "sobloo", "astraea_eod", ], "S2_MSI_L2A_MAJA": ["theia"], "S2_MSI_L2B_MAJA_SNOW": ["theia"], "S2_MSI_L2B_MAJA_WATER": ["theia"], "S2_MSI_L3A_WASP": ["theia"], "S2_MSI_L1C": [ "aws_eos", "peps", "sobloo", "onda", "wekeo", "mundi", "creodias", "astraea_eod", ], "S3_ERR": ["peps", "onda", "wekeo", "creodias"], "S3_EFR": ["peps", "onda", "wekeo", "creodias"], "S3_LAN": ["peps", "onda", "wekeo", "creodias"], "S3_SRA": ["onda", "wekeo", "creodias"], "S3_SRA_BS": ["onda", "wekeo", "creodias"], "S3_SRA_A_BS": ["onda", "creodias"], "S3_WAT": ["onda", "wekeo", "creodias"], "S3_OLCI_L2LFR": ["peps", "onda", "wekeo", "creodias", "mundi"], "S3_OLCI_L2LRR": ["peps", "onda", "wekeo", "creodias"], "S3_SLSTR_L1RBT": ["peps", "onda", "wekeo", "creodias"], "S3_SLSTR_L2LST": ["peps", "onda", "wekeo", "creodias"], "PLD_PAN": ["theia"], "PLD_XS": ["theia"], "PLD_BUNDLE": ["theia"], "PLD_PANSHARPENED": ["theia"], "CBERS4_PAN10M_L2": ["aws_eos"], "CBERS4_PAN10M_L4": ["aws_eos"], "CBERS4_PAN5M_L2": ["aws_eos"], "CBERS4_PAN5M_L4": ["aws_eos"], "CBERS4_MUX_L2": ["aws_eos"], "CBERS4_MUX_L4": ["aws_eos"], "CBERS4_AWFI_L2": ["aws_eos"], "CBERS4_AWFI_L4": ["aws_eos"], "MODIS_MCD43A4": ["aws_eos", "astraea_eod"], "NAIP": ["aws_eos", "astraea_eod"], "SPOT_SWH": ["theia"], "SPOT_SWH_OLD": ["theia"], "SPOT5_SPIRIT": ["theia"], "VENUS_L1C": ["theia"], "VENUS_L2A_MAJA": ["theia"], "VENUS_L3A_MAJA": ["theia"], "OSO": ["theia"], GENERIC_PRODUCT_TYPE: [ "theia", "peps", "sobloo", "onda", "mundi", "creodias", "astraea_eod", ], } SUPPORTED_PROVIDERS = [ "peps", "usgs", "theia", "sobloo", "creodias", "mundi", "onda", "wekeo", "aws_eos", "astraea_eod", ] def setUp(self): super(TestCore, self).setUp() self.dag = EODataAccessGateway() def tearDown(self): super(TestCore, self).tearDown() for old in glob.glob1(self.dag.conf_dir, "*.old") + glob.glob1( self.dag.conf_dir, ".*.old"): old_path = os.path.join(self.dag.conf_dir, old) if os.path.exists(old_path): try: os.remove(old_path) except OSError: shutil.rmtree(old_path) if os.getenv("EODAG_CFG_FILE") is not None: os.environ.pop("EODAG_CFG_FILE") if os.getenv("EODAG_LOCS_CFG_FILE") is not None: os.environ.pop("EODAG_LOCS_CFG_FILE") def test_supported_providers_in_unit_test(self): """Every provider must be referenced in the core unittest SUPPORTED_PROVIDERS class attribute""" # noqa for provider in self.dag.available_providers(): self.assertIn(provider, self.SUPPORTED_PROVIDERS) def test_supported_product_types_in_unit_test(self): """Every product type must be referenced in the core unit test SUPPORTED_PRODUCT_TYPES class attribute""" # noqa for product_type in self.dag.list_product_types(): self.assertIn(product_type["ID"], self.SUPPORTED_PRODUCT_TYPES.keys()) def test_list_product_types_ok(self): """Core api must correctly return the list of supported product types""" product_types = self.dag.list_product_types() self.assertIsInstance(product_types, list) for product_type in product_types: self.assertListProductTypesRightStructure(product_type) # There should be no repeated product type in the output self.assertEqual(len(product_types), len(set(pt["ID"] for pt in product_types))) def test_list_product_types_for_provider_ok(self): """Core api must correctly return the list of supported product types for a given provider""" # noqa for provider in self.SUPPORTED_PROVIDERS: product_types = self.dag.list_product_types(provider=provider) self.assertIsInstance(product_types, list) for product_type in product_types: self.assertListProductTypesRightStructure(product_type) self.assertIn(provider, self.SUPPORTED_PRODUCT_TYPES[product_type["ID"]]) def test_list_product_types_for_unsupported_provider(self): """Core api must raise UnsupportedProvider error for list_product_types with unsupported provider""" # noqa unsupported_provider = "a" self.assertRaises( UnsupportedProvider, self.dag.list_product_types, provider=unsupported_provider, ) def assertListProductTypesRightStructure(self, structure): """Helper method to verify that the structure given is a good result of EODataAccessGateway.list_product_types """ self.assertIsInstance(structure, dict) self.assertIn("ID", structure) self.assertIn("abstract", structure) self.assertIn("instrument", structure) self.assertIn("platform", structure) self.assertIn("platformSerialIdentifier", structure) self.assertIn("processingLevel", structure) self.assertIn("sensorType", structure) self.assertIn(structure["ID"], self.SUPPORTED_PRODUCT_TYPES) def test_core_object_creates_config_standard_location(self): """The core object must create a user config file in standard user config location on instantiation""" # noqa self.execution_involving_conf_dir(inspect="eodag.yml") def test_core_object_creates_index_if_not_exist(self): """The core object must create an index in user config directory""" self.execution_involving_conf_dir(inspect=".index") @mock.patch("eodag.api.core.open_dir", autospec=True) @mock.patch("eodag.api.core.exists_in", autospec=True, return_value=True) def test_core_object_open_index_if_exists(self, exists_in_mock, open_dir_mock): """The core object must use the existing index dir if any""" conf_dir = os.path.join(os.path.expanduser("~"), ".config", "eodag") index_dir = os.path.join(conf_dir, ".index") if not os.path.exists(index_dir): makedirs(index_dir) EODataAccessGateway() open_dir_mock.assert_called_with(index_dir) def test_core_object_prioritize_config_file_in_envvar(self): """The core object must use the config file pointed to by the EODAG_CFG_FILE env var""" # noqa os.environ["EODAG_CFG_FILE"] = os.path.join( TEST_RESOURCES_PATH, "file_config_override.yml") dag = EODataAccessGateway() # usgs priority is set to 5 in the test config overrides self.assertEqual(dag.get_preferred_provider(), ("usgs", 5)) # peps outputs prefix is set to /data self.assertEqual(dag.providers_config["peps"].download.outputs_prefix, "/data") def execution_involving_conf_dir(self, inspect=None): """Check that the path(s) inspected (str, list) are created after the instantation of EODataAccessGateway. If they were already there, rename them (.old), instantiate, check, delete the new files, and restore the existing files to there previous name.""" if inspect is not None: if isinstance(inspect, str): inspect = [inspect] conf_dir = os.path.join(os.path.expanduser("~"), ".config", "eodag") olds = [] currents = [] for inspected in inspect: old = current = os.path.join(conf_dir, inspected) if os.path.exists(current): old = os.path.join(conf_dir, "{}.old".format(inspected)) shutil.move(current, old) olds.append(old) currents.append(current) EODataAccessGateway() for old, current in zip(olds, currents): self.assertTrue(os.path.exists(current)) if old != current: try: shutil.rmtree(current) except OSError: os.unlink(current) shutil.move(old, current) def test_core_object_creates_locations_standard_location(self): """The core object must create a locations config file and a shp dir in standard user config location on instantiation""" # noqa self.execution_involving_conf_dir(inspect=["locations.yml", "shp"]) def test_core_object_set_default_locations_config(self): """The core object must set the default locations config on instantiation""" # noqa dag = EODataAccessGateway() conf_dir = os.path.join(os.path.expanduser("~"), ".config", "eodag") default_shpfile = os.path.join(conf_dir, "shp", "ne_110m_admin_0_map_units.shp") self.assertIsInstance(dag.locations_config, list) self.assertEqual( dag.locations_config, [dict(attr="ADM0_A3_US", name="country", path=default_shpfile)], ) def test_core_object_locations_file_not_found(self): """The core object must set the locations to an empty list when the file is not found""" # noqa dag = EODataAccessGateway(locations_conf_path="no_locations.yml") self.assertEqual(dag.locations_config, []) def test_core_object_prioritize_locations_file_in_envvar(self): """The core object must use the locations file pointed to by the EODEODAG_LOCS_CFG_FILEAG_CFG_FILE env var""" # noqa os.environ["EODAG_LOCS_CFG_FILE"] = os.path.join( TEST_RESOURCES_PATH, "file_locations_override.yml") dag = EODataAccessGateway() self.assertEqual( dag.locations_config, [dict(attr="dummyattr", name="dummyname", path="dummypath.shp")], ) def test_get_geometry_from_various_no_locations(self): """The search geometry can be set from a dict, list, tuple, WKT string or shapely geom""" ref_geom_as_wkt = "POLYGON ((0 50, 0 52, 2 52, 2 50, 0 50))" ref_geom = wkt.loads(ref_geom_as_wkt) # Good dict geometry = { "lonmin": 0, "latmin": 50, "lonmax": 2, "latmax": 52, } self.assertEquals(get_geometry_from_various([], geometry=geometry), ref_geom) # Bad dict with a missing key del geometry["lonmin"] self.assertRaises( TypeError, get_geometry_from_various, [], geometry=geometry, ) # Tuple geometry = (0, 50, 2, 52) self.assertEquals(get_geometry_from_various([], geometry=geometry), ref_geom) # List geometry = list(geometry) self.assertEquals(get_geometry_from_various([], geometry=geometry), ref_geom) # List without 4 items geometry.pop() self.assertRaises( TypeError, get_geometry_from_various, [], geometry=geometry, ) # WKT geometry = ref_geom_as_wkt self.assertEquals(get_geometry_from_various([], geometry=geometry), ref_geom) # Some other shapely geom geometry = LineString([[0, 0], [1, 1]]) self.assertIsInstance(get_geometry_from_various([], geometry=geometry), LineString) def test_get_geometry_from_various_only_locations(self): """The search geometry can be set from a locations config file query""" locations_config = self.dag.locations_config # No query args self.assertIsNone(get_geometry_from_various(locations_config)) # Bad query arg # 'country' is the expected name here self.assertIsNone( get_geometry_from_various(locations_config, bad_query_arg="dummy")) # France geom_france = get_geometry_from_various(locations_config, country="FRA") self.assertIsInstance(geom_france, MultiPolygon) self.assertEquals(len(geom_france), 3) # France + Guyana + Corsica # Not defined self.assertIsNone( get_geometry_from_various(locations_config, country="bad_query_value")) def test_get_geometry_from_various_only_locations_regex(self): """The search geometry can be set from a locations config file query and a regex""" locations_config = self.dag.locations_config # Pakistan + Panama (each has a unique polygon) => Multypolygon of len 2 geom_regex_pa = get_geometry_from_various(locations_config, country="PA[A-Z]") self.assertIsInstance(geom_regex_pa, MultiPolygon) self.assertEquals(len(geom_regex_pa), 2) def test_get_geometry_from_various_geometry_and_locations(self): """The search geometry can be set from a given geometry and a locations config file query""" geometry = { "lonmin": 20, "latmin": 50, "lonmax": 22, "latmax": 52, } locations_config = self.dag.locations_config geom_combined = get_geometry_from_various(locations_config, country="FRA", geometry=geometry) self.assertIsInstance(geom_combined, MultiPolygon) self.assertEquals( len(geom_combined), 4) # France + Guyana + Corsica + somewhere over Poland geometry = { "lonmin": 0, "latmin": 50, "lonmax": 2, "latmax": 52, } geom_combined = get_geometry_from_various(locations_config, country="FRA", geometry=geometry) self.assertIsInstance(geom_combined, MultiPolygon) self.assertEquals(len(geom_combined), 3) # The bounding box overlaps with France inland def test_rebuild_index(self): """Change eodag version and check that whoosh index is rebuilt""" index_dir = os.path.join(self.dag.conf_dir, ".index") index_dir_mtime = os.path.getmtime(index_dir) self.assertNotEqual(self.dag.get_version(), "fake-version") with mock.patch( "eodag.api.core.EODataAccessGateway.get_version", autospec=True, return_value="fake-version", ): self.assertEqual(self.dag.get_version(), "fake-version") self.dag.build_index() # check that index_dir has beeh re-created self.assertNotEqual(os.path.getmtime(index_dir), index_dir_mtime)
class TestCore(unittest.TestCase): SUPPORTED_PRODUCT_TYPES = { "L8_REFLECTANCE": ["theia"], "L57_REFLECTANCE": ["theia"], "L5_L1T": [], "L5_L1G": [], "L5_L1GT": [], "L7_L1G": [], "L7_L1T": [], "L7_L1GT": [], "L8_OLI_TIRS_C1L1": ["onda", "usgs", "aws_eos"], "S1_SAR_GRD": [ "peps", "sobloo", "onda", "wekeo", "mundi", "creodias", "aws_eos", ], "S1_SAR_OCN": ["peps", "sobloo", "onda", "creodias"], "S1_SAR_RAW": ["sobloo", "onda", "creodias"], "S1_SAR_SLC": ["peps", "sobloo", "onda", "wekeo", "mundi", "creodias"], "S2_MSI_L2A": ["onda", "mundi", "creodias", "peps", "aws_eos"], "S2_MSI_L2A_MAJA": ["theia"], "S2_MSI_L2B_MAJA_SNOW": ["theia"], "S2_MSI_L2B_MAJA_WATER": ["theia"], "S2_MSI_L3A_WASP": ["theia"], "S2_MSI_L1C": [ "aws_eos", "peps", "sobloo", "onda", "wekeo", "mundi", "creodias", ], "S3_ERR": ["peps", "onda", "wekeo", "creodias"], "S3_EFR": ["peps", "onda", "wekeo", "creodias"], "S3_LAN": ["peps", "sobloo", "onda", "wekeo", "creodias"], "S3_SRA": ["sobloo", "onda", "wekeo", "creodias"], "S3_SRA_BS": ["sobloo", "onda", "wekeo", "creodias"], "S3_SRA_A_BS": ["sobloo", "onda", "creodias"], "S3_WAT": ["onda", "wekeo", "creodias"], "S3_OLCI_L2LFR": ["peps", "onda", "wekeo", "creodias", "mundi"], "S3_OLCI_L2LRR": ["peps", "onda", "wekeo", "creodias"], "S3_SLSTR_L1RBT": ["peps", "onda", "wekeo", "creodias"], "S3_SLSTR_L2LST": ["peps", "onda", "wekeo", "creodias"], "PLD_PAN": ["theia"], "PLD_XS": ["theia"], "PLD_BUNDLE": ["theia"], "PLD_PANSHARPENED": ["theia"], "ES_FRS": [], "CBERS4_PAN10M_L2": ["aws_eos"], "CBERS4_PAN10M_L4": ["aws_eos"], "CBERS4_PAN5M_L2": ["aws_eos"], "CBERS4_PAN5M_L4": ["aws_eos"], "CBERS4_MUX_L2": ["aws_eos"], "CBERS4_MUX_L4": ["aws_eos"], "CBERS4_AWFI_L2": ["aws_eos"], "CBERS4_AWFI_L4": ["aws_eos"], "MODIS_MCD43A4": ["aws_eos"], "NAIP": ["aws_eos"], "SPOT_SWH": ["theia"], "SPOT_SWH_OLD": ["theia"], "SPOT5_SPIRIT": ["theia"], "VENUS_L1C": ["theia"], "VENUS_L2A_MAJA": ["theia"], "VENUS_L3A_MAJA": ["theia"], "OSO": ["theia"], } SUPPORTED_PROVIDERS = [ "peps", "usgs", "theia", "sobloo", "creodias", "mundi", "onda", "wekeo", "aws_eos", ] def setUp(self): super(TestCore, self).setUp() self.dag = EODataAccessGateway() def tearDown(self): super(TestCore, self).tearDown() for old in glob.glob1(self.dag.conf_dir, "*.old") + glob.glob1( self.dag.conf_dir, ".*.old"): old_path = os.path.join(self.dag.conf_dir, old) if os.path.exists(old_path): try: os.remove(old_path) except OSError: shutil.rmtree(old_path) if os.getenv("EODAG_CFG_FILE") is not None: os.environ.pop("EODAG_CFG_FILE") def test_supported_providers_in_unit_test(self): """Every provider must be referenced in the core unittest SUPPORTED_PROVIDERS class attribute""" # noqa for provider in self.dag.available_providers(): self.assertIn(provider, self.SUPPORTED_PROVIDERS) def test_supported_product_types_in_unit_test(self): """Every product type must be referenced in the core unit test SUPPORTED_PRODUCT_TYPES class attribute""" # noqa for product_type in self.dag.list_product_types(): self.assertIn(product_type["ID"], self.SUPPORTED_PRODUCT_TYPES.keys()) def test_list_product_types_ok(self): """Core api must correctly return the list of supported product types""" product_types = self.dag.list_product_types() self.assertIsInstance(product_types, list) for product_type in product_types: self.assertListProductTypesRightStructure(product_type) # There should be no repeated product type in the output self.assertEqual(len(product_types), len(set(pt["ID"] for pt in product_types))) def test_list_product_types_for_provider_ok(self): """Core api must correctly return the list of supported product types for a given provider""" # noqa for provider in self.SUPPORTED_PROVIDERS: product_types = self.dag.list_product_types(provider=provider) self.assertIsInstance(product_types, list) for product_type in product_types: self.assertListProductTypesRightStructure(product_type) self.assertIn(provider, self.SUPPORTED_PRODUCT_TYPES[product_type["ID"]]) def test_list_product_types_for_unsupported_provider(self): """Core api must raise UnsupportedProvider error for list_product_types with unsupported provider""" # noqa unsupported_provider = "a" self.assertRaises( UnsupportedProvider, self.dag.list_product_types, provider=unsupported_provider, ) def assertListProductTypesRightStructure(self, structure): """Helper method to verify that the structure given is a good result of EODataAccessGateway.list_product_types """ self.assertIsInstance(structure, dict) self.assertIn("ID", structure) self.assertIn("abstract", structure) self.assertIn("instrument", structure) self.assertIn("platform", structure) self.assertIn("platformSerialIdentifier", structure) self.assertIn("processingLevel", structure) self.assertIn("sensorType", structure) self.assertIn(structure["ID"], self.SUPPORTED_PRODUCT_TYPES) def test_core_object_creates_config_standard_location(self): """The core object must create a user config file in standard user config location on instantiation""" # noqa self.execution_involving_conf_dir(inspect="eodag.yml") def test_core_object_creates_index_if_not_exist(self): """The core object must create an index in user config directory""" self.execution_involving_conf_dir(inspect=".index") @mock.patch("eodag.api.core.open_dir", autospec=True) @mock.patch("eodag.api.core.exists_in", autospec=True, return_value=True) def test_core_object_open_index_if_exists(self, exists_in_mock, open_dir_mock): """The core object must use the existing index dir if any""" conf_dir = os.path.join(os.path.expanduser("~"), ".config", "eodag") index_dir = os.path.join(conf_dir, ".index") if not os.path.exists(index_dir): makedirs(index_dir) EODataAccessGateway() open_dir_mock.assert_called_with(index_dir) def test_core_object_prioritize_config_file_in_envvar(self): """The core object must use the config file pointed to by the EODAG_CFG_FILE env var""" # noqa os.environ["EODAG_CFG_FILE"] = os.path.join( TEST_RESOURCES_PATH, "file_config_override.yml") dag = EODataAccessGateway() # usgs priority is set to 5 in the test config overrides self.assertEqual(dag.get_preferred_provider(), ("usgs", 5)) # peps outputs prefix is set to /data self.assertEqual(dag.providers_config["peps"].download.outputs_prefix, "/data") def execution_involving_conf_dir(self, inspect=None): if inspect is not None: conf_dir = os.path.join(os.path.expanduser("~"), ".config", "eodag") old = current = os.path.join(conf_dir, inspect) if os.path.exists(current): old = os.path.join(conf_dir, "{}.old".format(inspect)) shutil.move(current, old) EODataAccessGateway() self.assertTrue(os.path.exists(current)) if old != current: try: shutil.rmtree(current) except OSError: os.unlink(current) shutil.move(old, current)