def test_two_layers(self): from qc_tool.raster.naming import run_check params = { "unzip_dir": TEST_DATA_DIR.joinpath("raster", "checks", "mmu"), "extensions": [".tif"], "layer_names": { "layer_1": "^test_raster1.tif$", "layer_2": "^mmu_raster_incorrect.tif$", "layer_3": "^mmu_raster_correct.tif" } } status = self.status_class() run_check(params, status) self.assertEqual("ok", status.status) self.assertEqual( "test_raster1.tif", status.params["raster_layer_defs"]["layer_1"]["src_layer_name"]) self.assertEqual( TEST_DATA_DIR.joinpath("raster", "checks", "mmu", "test_raster1.tif"), status.params["raster_layer_defs"]["layer_1"]["src_filepath"]) self.assertEqual( "mmu_raster_incorrect.tif", status.params["raster_layer_defs"]["layer_2"]["src_layer_name"]) self.assertEqual( TEST_DATA_DIR.joinpath("raster", "checks", "mmu", "mmu_raster_incorrect.tif"), status.params["raster_layer_defs"]["layer_2"]["src_filepath"]) self.assertEqual( "mmu_raster_correct.tif", status.params["raster_layer_defs"]["layer_3"]["src_layer_name"]) self.assertEqual( TEST_DATA_DIR.joinpath("raster", "checks", "mmu", "mmu_raster_correct.tif"), status.params["raster_layer_defs"]["layer_3"]["src_filepath"])
def test_vector_aoi_incomplete(self): layer_filepath1 = TEST_DATA_DIR.joinpath( "raster", "checks", "gap", "raster_for_vector_aoi_nok01.tif") layer_defs = { "layer_1": { "src_filepath": layer_filepath1, "src_layer_name": layer_filepath1.name } } self.params.update({ "raster_layer_defs": layer_defs, "aoi_code": "city002" }) from qc_tool.raster.gap import run_check self.params.update({ "layers": ["layer_1"], "outside_area_code": "NODATA", "mask": "aoi_ua_building_heights.gpkg", "du_column_name": "CodeCITY", "boundary_dir": TEST_DATA_DIR.joinpath("boundaries"), "tmp_dir": self.jobdir_manager.tmp_dir, "output_dir": self.jobdir_manager.output_dir, "step_nr": 1 }) status = self.status_class() run_check(self.params, status) self.assertIn("has 136 gap pixels in the mapped area.", status.messages[0]) self.assertIn("s01_raster_for_vector_aoi_nok01_gap_warning.tif", status.attachment_filenames) self.assertTrue(self.params["output_dir"].joinpath( status.attachment_filenames[0]).exists())
def test_aborted(self): # Create an example tiled raster layer width = 2000 height = 256 xmin = 10000 ymax = 10000 pixel_size = 5 # Prepare raster dataset. mem_drv = gdal.GetDriverByName("MEM") mem_ds = mem_drv.Create("memory", width, height, 1, gdal.GDT_Byte) mem_srs = osr.SpatialReference() mem_srs.ImportFromEPSG(3035) mem_ds.SetProjection(mem_srs.ExportToWkt()) mem_ds.SetGeoTransform((xmin, pixel_size, 0, ymax, 0, -pixel_size)) mem_ds.GetRasterBand(1).Fill(0) # Store raster dataset. dest_driver = gdal.GetDriverByName("GTiff") dest_ds = dest_driver.CreateCopy(str( self.params["raster_layer_defs"]["raster_1"]["src_filepath"]), mem_ds, False, options=["TILED=NO"]) # Remove temporary datasource and datasets from memory. mem_dsrc = None mem_ds = None dest_ds = None # Run check on untiled raster from qc_tool.raster.tile import run_check status = self.status_class() run_check(self.params, status) self.assertEqual("aborted", status.status)
def test_edge_patches(self): # Prepare raster dataset. data = [[1, 0, 1, 1, 1, 0, 0], [0, 0, 0, 0, 0, 0, 0], [0, 1, 1, 1, 1, 0, 0], [0, 1, 1, 1, 1, 0, 1], [1, 0, 0, 0, 0, 1, 1], [1, 1, 1, 1, 0, 1, 1]] RasterCheckTestCase.create_raster(self.tmp_raster, np.array(data), 10) self.params.update({ "layers": ["layer1"], "area_pixels": 8, "nodata_value": 9, "groupcodes": [], "output_dir": self.jobdir_manager.output_dir, "step_nr": 1 }) from qc_tool.raster.mmu import run_check status = self.status_class() run_check(self.params, status) self.assertEqual("ok", status.status) self.assertEqual(1, len(status.attachment_filenames)) self.assertIn("s01_test_raster1_lessmmu_exception.gpkg", status.attachment_filenames) self.assertEqual(1, len(status.messages)) self.assertIn("5 exceptional objects under MMU limit", status.messages[0])
def test_fail(self): # Prepare raster dataset with a patch smaller than MMU. data = np.zeros((4000, 3000), dtype=int) data[2047, 1000] = 1 data[2047, 1001] = 1 data[2048, 1000] = 1 data[2048, 1001] = 1 data[2047, 1111] = 1 data[2048, 1111] = 1 data[2049, 1111] = 1 RasterCheckTestCase.create_raster(self.tmp_raster, np.array(data), 10) self.params.update({ "layers": ["layer1"], "area_pixels": 4, "nodata_value": 9, "groupcodes": [], "output_dir": self.jobdir_manager.output_dir, "step_nr": 1 }) from qc_tool.raster.mmu import run_check status = self.status_class() run_check(self.params, status) self.assertEqual("failed", status.status) self.assertEqual(1, len(status.attachment_filenames)) self.assertIn("s01_test_raster1_lessmmu_error.gpkg", status.attachment_filenames) self.assertEqual(1, len(status.messages)) self.assertIn("1 error objects under MMU limit", status.messages[0])
def test_fail_isolated_invalid_pixel(self): from qc_tool.raster.value import run_check self.tmp_raster = self.jobdir_manager.tmp_dir.joinpath( "tmp_big_raster.tif") tmp_array = np.ones((10000, 10000)) tmp_array[5555, 4444] = 253 RasterCheckTestCase.create_raster(self.tmp_raster, np.array(tmp_array), 10, ulx=5000, uly=3000) layer_defs = { "layer_1": { "src_filepath": self.tmp_raster, "src_layer_name": self.tmp_raster.name } } self.params.update({ "raster_layer_defs": layer_defs, "layers": ["layer_1"] }) self.params.update({"validcodes": [1]}) status = self.status_class() run_check(self.params, status) self.assertEqual("failed", status.status) self.assertIn( "Layer tmp_big_raster.tif has pixels with invalid values: 253.", status.messages)
def test(self): from qc_tool.raster.value import run_check self.params.update({"validcodes": [0, 1]}) status = self.status_class() run_check(self.params, status) self.assertEqual("ok", status.status)
def test_vector_aoi_not_intersect(self): layer_filepath1 = TEST_DATA_DIR.joinpath( "raster", "checks", "gap", "raster_for_vector_aoi_ok.tif") layer_defs = { "layer_1": { "src_filepath": layer_filepath1, "src_layer_name": layer_filepath1.name } } self.params.update({ "raster_layer_defs": layer_defs, "aoi_code": "city001" }) from qc_tool.raster.gap import run_check self.params.update({ "layers": ["layer_1"], "outside_area_code": "NODATA", "mask": "aoi_ua_building_heights.gpkg", "du_column_name": "CodeCITY", "boundary_dir": TEST_DATA_DIR.joinpath("boundaries"), "tmp_dir": self.jobdir_manager.tmp_dir, "output_dir": self.jobdir_manager.output_dir, "step_nr": 1 }) status = self.status_class() run_check(self.params, status) self.assertIn( "does not intersect any AOI polygon with CodeCITY=city001 from boundary aoi_ua_building_heights.gpkg", status.messages[0])
def test_fail(self): from qc_tool.raster.value import run_check self.params.update({"validcodes": [1, 2, 255]}) status = self.status_class() run_check(self.params, status) self.assertEqual( "failed", status.status, "Value raster check should fail " "if the raster has invalid codes.") self.assertIn("invalid values: 0.", status.messages[0])
def test_missing_xml_fail(self): from qc_tool.raster.inspire import run_check self.params["raster_layer_defs"] = { "layer0": { "src_filepath": self.xml_dir.joinpath("inspire_missing_xml.tif") } } status = self.status_class() run_check(self.params, status) self.assertEqual("ok", status.status)
def test_excessive_layers(self): from qc_tool.raster.naming import run_check params = { "unzip_dir": TEST_DATA_DIR.joinpath("raster", "checks", "mmu"), "extensions": [".tif"], "layer_names": { "layer_1": "^test_raster1.tif$" } } status = self.status_class() run_check(params, status) self.assertEqual("failed", status.status)
def test(self): from qc_tool.raster.inspire import run_check self.params["raster_layer_defs"] = { "layer0": { "src_filepath": self.xml_dir.joinpath("inspire_good.tif") } } status = self.status_class() run_check(self.params, status) self.assertEqual("ok", status.status) self.assertIn("s01_inspire_good_inspire_report.html", status.attachment_filenames) self.assertIn("s01_inspire_good_inspire_log.txt", status.attachment_filenames)
def test_incorrect_aoi_code(self): from qc_tool.raster.naming import run_check params = { "unzip_dir": TEST_DATA_DIR.joinpath("raster", "checks", "mmu"), "extensions": [".tif"], "layer_names": { "layer_1": "^test_(?P<aoi_code>[a-z0-9]+).tif$" }, "aoi_codes": ["aoi_1", "aoi_2"] } status = self.status_class() run_check(params, status) self.assertEqual("aborted", status.status) self.assertIn("Layer test_raster1.tif has illegal AOI code raster1.", status.messages)
def test_cancelled(self): from qc_tool.raster.gap import run_check self.params.update({ "layers": ["layer_2"], "aoi_code": "non-existing-country", "outside_area_code": 255, "mask": "test", "boundary_dir": TEST_DATA_DIR.joinpath("boundaries"), "tmp_dir": self.jobdir_manager.tmp_dir, "output_dir": self.jobdir_manager.output_dir, "step_nr": 1 }) status = self.status_class() run_check(self.params, status) self.assertIn("cancelled", status.messages[0])
def test_xml_format_fail(self): from qc_tool.raster.inspire import run_check self.params["raster_layer_defs"] = { "layer0": { "src_filepath": self.xml_dir.joinpath("inspire_invalid_xml.tif") } } status = self.status_class() run_check(self.params, status) self.assertEqual("ok", status.status) self.assertEqual(2, len(status.messages)) self.assertIn( "The xml file inspire_invalid_xml.xml does not contain a <gmd:MD_Metadata> top-level element.", status.messages[1])
def test_incorrect_metadata2(self): from qc_tool.raster.inspire import run_check self.params["raster_layer_defs"] = { "layer0": { "src_filepath": self.xml_dir.joinpath("IMD_2018_010m_eu_03035_test.tif") } } status = self.status_class() run_check(self.params, status) self.assertEqual("ok", status.status) self.assertIn("s01_IMD_2018_010m_eu_03035_test_inspire_report.html", status.attachment_filenames) self.assertIn("s01_IMD_2018_010m_eu_03035_test_inspire_log.txt", status.attachment_filenames)
def test_fail(self): from qc_tool.raster.color import run_check self.params.update({ "layers": ["layer_1"], "colors": { "0": [240, 240, 240], "1": [70, 158, 74], "2": [28, 92, 36], "254": [153, 153, 153], "255": [0, 0, 99] } }) status = self.status_class() run_check(self.params, status) self.assertEqual( "failed", status.status, "Color raster check with incorrect colors should fail.")
def test_reference_year(self): from qc_tool.raster.naming import run_check params = { "unzip_dir": TEST_DATA_DIR.joinpath("raster", "checks", "mmu"), "extensions": [".tif"], "layer_names": { "layer_1": "^test_raster1.tif$", "layer_2": "^mmu_raster_incorrect.tif$", "layer_3": "^mmu_raster_correct.tif" }, "reference_year": "2018" } status = self.status_class() run_check(params, status) self.assertEqual("ok", status.status) self.assertIn("reference_year", status.status_properties) self.assertEqual("2018", status.status_properties["reference_year"])
def test(self): from qc_tool.raster.gap import run_check self.params.update({ "layers": ["layer_1"], "aoi_code": "testaoi", "outside_area_code": 255, "mask": "test", "boundary_dir": TEST_DATA_DIR.joinpath("boundaries"), "tmp_dir": self.jobdir_manager.tmp_dir, "output_dir": self.jobdir_manager.output_dir, "step_nr": 1 }) status = self.status_class() run_check(self.params, status) self.assertEqual( "ok", status.status, "Gap raster check should pass " "if the raster does not have NoData values inside the AOI.")
def test_missing_layer(self): from qc_tool.raster.naming import run_check params = { "unzip_dir": TEST_DATA_DIR.joinpath("raster", "checks", "mmu"), "extensions": [".tif"], "layer_names": { "layer_1": "^test_raster1.tif$", "layer_2": "^mmu_raster_incorrect.tif$", "layer_3": "^mmu_raster_correct.tif", "layer_4": "^non_existent_raster.tif" } } status = self.status_class() run_check(params, status) self.assertEqual("aborted", status.status) self.assertIn( "The layer_4 layer name does not match naming convention.", status.messages)
def test_bigger_than_mask(self): mask = [[0, 1, 1, 1, 0], [1, 1, 1, 1, 0], [0, 0, 1, 1, 0], [1, 0, 1, 1, 0]] RasterCheckTestCase.create_raster(self.mask_filepath, np.array(mask), 10, ulx=5010, uly=2990) layer = [[0, 0, 0, 0, 0, 0, 0, 0], [0, 1, 1, 1, 1, 0, 0, 0], [0, 1, 1, 1, 1, 0, 0, 0], [0, 0, 0, 1, 1, 0, 0, 0], [0, 1, 0, 1, 1, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0]] RasterCheckTestCase.create_raster(self.layer_filepath, np.array(layer), 10, ulx=5000, uly=3000) RasterCheckTestCase.create_raster(self.layer_filepath, np.array(layer), 10, ulx=5010, uly=2990) self.params.update({ "layers": ["layer_3"], "boundary_dir": self.jobdir_manager.tmp_dir.joinpath("boundaries"), "tmp_dir": self.jobdir_manager.tmp_dir, "output_dir": self.jobdir_manager.output_dir, "aoi_code": "aoi01", "outside_area_code": 9, "step_nr": 1 }) from qc_tool.raster.gap import run_check status = self.status_class() run_check(self.params, status) self.assertEqual("ok", status.status)
def test_gaps_found(self): from qc_tool.raster.gap import run_check self.params.update({ "layers": ["layer_2"], "aoi_code": "testaoi", "outside_area_code": 255, "mask": "test", "boundary_dir": TEST_DATA_DIR.joinpath("boundaries"), "tmp_dir": self.jobdir_manager.tmp_dir, "output_dir": self.jobdir_manager.output_dir, "step_nr": 1 }) status = self.status_class() run_check(self.params, status) self.assertIn("has 1237 gap pixels", status.messages[0]) self.assertIn("s01_incomplete_raster_100m_testaoi_gap_warning.tif", status.attachment_filenames) self.assertTrue(self.params["output_dir"].joinpath( status.attachment_filenames[0]).exists())
def test(self): # Prepare raster dataset. data = [[0, 1, 0, 255], [0, 0, 9, 255], [0, 0, 0, 255]] RasterCheckTestCase.create_raster(self.tmp_raster, np.array(data), 10, ulx=0, uly=0, epsg=3035, nodata_value=255) self.params.update({ "layers": ["raster"], "nodata_value": 255, "step_nr": 1 }) from qc_tool.raster.nodata import run_check status = self.status_class() run_check(self.params, status) self.assertEqual("ok", status.status)
def test_intersects_mask_lower_right(self): # Prepare mask. mask = [[0, 0, 0, 0, 0, 0, 0, 0], [0, 1, 1, 1, 1, 0, 0, 0], [0, 1, 1, 1, 1, 0, 0, 0], [0, 0, 0, 1, 1, 0, 0, 0], [0, 1, 0, 1, 1, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0]] RasterCheckTestCase.create_raster(self.mask_filepath, np.array(mask), 10, ulx=5000, uly=3000) # Prepare raster located partially outside the mask. layer = [[2, 1, 1, 2], [1, 1, 1, 9], [9, 9, 2, 2]] RasterCheckTestCase.create_raster(self.layer_filepath, np.array(layer), 10, ulx=5050, uly=2960) self.params.update({ "layers": ["layer_3"], "boundary_dir": self.jobdir_manager.tmp_dir.joinpath("boundaries"), "tmp_dir": self.jobdir_manager.tmp_dir, "output_dir": self.jobdir_manager.output_dir, "aoi_code": "aoi01", "outside_area_code": 9, "step_nr": 1 }) from qc_tool.raster.gap import run_check status = self.status_class() run_check(self.params, status) self.assertEqual("ok", status.status)
def test(self): from qc_tool.raster.naming import run_check params = { "unzip_dir": TEST_DATA_DIR.joinpath("raster", "fty_100m", "fty_2018_100m_eu_03035_v0_1"), "aoi_codes": ["mt", "eu"], "extensions": [".tif"], "reference_year": "2018", "layer_names": { "layer_1": "^fty_2018_100m_(?P<aoi_code>.+)_[0-9]{5}.*.tif$" } } status = self.status_class() run_check(params, status) self.assertEqual("ok", status.status) self.assertEqual("eu", status.params["aoi_code"]) self.assertEqual("2018", status.status_properties["reference_year"]) self.assertEqual( "fty_2018_100m_eu_03035_v0_1.tif", status.params["raster_layer_defs"]["layer_1"]["src_layer_name"])
def test_fail_nodata_value_mismatch(self): # Prepare raster dataset. data = [[0, 1, 0, 255], [0, 0, 9, 255], [0, 0, 0, 255]] RasterCheckTestCase.create_raster(self.tmp_raster, np.array(data), 10, ulx=0, uly=0, epsg=3035, nodata_value=0) self.params.update({ "layers": ["raster"], "nodata_value": 255, "step_nr": 1 }) from qc_tool.raster.nodata import run_check status = self.status_class() run_check(self.params, status) self.assertEqual("aborted", status.status) self.assertIn("has invalid NoData value: 0", status.messages[0])