def test_two_layers(self):
from qc_tool.raster.naming import run_check
params = {
"unzip_dir": TEST_DATA_DIR.joinpath("raster", "checks", "mmu"),
"extensions": [".tif"],
"layer_names": {
"layer_1": "^test_raster1.tif$",
"layer_2": "^mmu_raster_incorrect.tif$",
"layer_3": "^mmu_raster_correct.tif"
}
}
status = self.status_class()
run_check(params, status)
self.assertEqual("ok", status.status)
self.assertEqual(
"test_raster1.tif",
status.params["raster_layer_defs"]["layer_1"]["src_layer_name"])
self.assertEqual(
TEST_DATA_DIR.joinpath("raster", "checks", "mmu",
"test_raster1.tif"),
status.params["raster_layer_defs"]["layer_1"]["src_filepath"])
self.assertEqual(
"mmu_raster_incorrect.tif",
status.params["raster_layer_defs"]["layer_2"]["src_layer_name"])
self.assertEqual(
TEST_DATA_DIR.joinpath("raster", "checks", "mmu",
"mmu_raster_incorrect.tif"),
status.params["raster_layer_defs"]["layer_2"]["src_filepath"])
self.assertEqual(
"mmu_raster_correct.tif",
status.params["raster_layer_defs"]["layer_3"]["src_layer_name"])
self.assertEqual(
TEST_DATA_DIR.joinpath("raster", "checks", "mmu",
"mmu_raster_correct.tif"),
status.params["raster_layer_defs"]["layer_3"]["src_filepath"])
def test_vector_aoi_incomplete(self):
layer_filepath1 = TEST_DATA_DIR.joinpath(
"raster", "checks", "gap", "raster_for_vector_aoi_nok01.tif")
layer_defs = {
"layer_1": {
"src_filepath": layer_filepath1,
"src_layer_name": layer_filepath1.name
}
}
self.params.update({
"raster_layer_defs": layer_defs,
"aoi_code": "city002"
})
from qc_tool.raster.gap import run_check
self.params.update({
"layers": ["layer_1"],
"outside_area_code": "NODATA",
"mask": "aoi_ua_building_heights.gpkg",
"du_column_name": "CodeCITY",
"boundary_dir": TEST_DATA_DIR.joinpath("boundaries"),
"tmp_dir": self.jobdir_manager.tmp_dir,
"output_dir": self.jobdir_manager.output_dir,
"step_nr": 1
})
status = self.status_class()
run_check(self.params, status)
self.assertIn("has 136 gap pixels in the mapped area.",
status.messages[0])
self.assertIn("s01_raster_for_vector_aoi_nok01_gap_warning.tif",
status.attachment_filenames)
self.assertTrue(self.params["output_dir"].joinpath(
status.attachment_filenames[0]).exists())
def test_aborted(self):
# Create an example tiled raster layer
width = 2000
height = 256
xmin = 10000
ymax = 10000
pixel_size = 5
# Prepare raster dataset.
mem_drv = gdal.GetDriverByName("MEM")
mem_ds = mem_drv.Create("memory", width, height, 1, gdal.GDT_Byte)
mem_srs = osr.SpatialReference()
mem_srs.ImportFromEPSG(3035)
mem_ds.SetProjection(mem_srs.ExportToWkt())
mem_ds.SetGeoTransform((xmin, pixel_size, 0, ymax, 0, -pixel_size))
mem_ds.GetRasterBand(1).Fill(0)
# Store raster dataset.
dest_driver = gdal.GetDriverByName("GTiff")
dest_ds = dest_driver.CreateCopy(str(
self.params["raster_layer_defs"]["raster_1"]["src_filepath"]),
mem_ds,
False,
options=["TILED=NO"])
# Remove temporary datasource and datasets from memory.
mem_dsrc = None
mem_ds = None
dest_ds = None
# Run check on untiled raster
from qc_tool.raster.tile import run_check
status = self.status_class()
run_check(self.params, status)
self.assertEqual("aborted", status.status)
def test_edge_patches(self):
# Prepare raster dataset.
data = [[1, 0, 1, 1, 1, 0, 0], [0, 0, 0, 0, 0, 0, 0],
[0, 1, 1, 1, 1, 0, 0], [0, 1, 1, 1, 1, 0, 1],
[1, 0, 0, 0, 0, 1, 1], [1, 1, 1, 1, 0, 1, 1]]
RasterCheckTestCase.create_raster(self.tmp_raster, np.array(data), 10)
self.params.update({
"layers": ["layer1"],
"area_pixels": 8,
"nodata_value": 9,
"groupcodes": [],
"output_dir": self.jobdir_manager.output_dir,
"step_nr": 1
})
from qc_tool.raster.mmu import run_check
status = self.status_class()
run_check(self.params, status)
self.assertEqual("ok", status.status)
self.assertEqual(1, len(status.attachment_filenames))
self.assertIn("s01_test_raster1_lessmmu_exception.gpkg",
status.attachment_filenames)
self.assertEqual(1, len(status.messages))
self.assertIn("5 exceptional objects under MMU limit",
status.messages[0])
def test_fail(self):
# Prepare raster dataset with a patch smaller than MMU.
data = np.zeros((4000, 3000), dtype=int)
data[2047, 1000] = 1
data[2047, 1001] = 1
data[2048, 1000] = 1
data[2048, 1001] = 1
data[2047, 1111] = 1
data[2048, 1111] = 1
data[2049, 1111] = 1
RasterCheckTestCase.create_raster(self.tmp_raster, np.array(data), 10)
self.params.update({
"layers": ["layer1"],
"area_pixels": 4,
"nodata_value": 9,
"groupcodes": [],
"output_dir": self.jobdir_manager.output_dir,
"step_nr": 1
})
from qc_tool.raster.mmu import run_check
status = self.status_class()
run_check(self.params, status)
self.assertEqual("failed", status.status)
self.assertEqual(1, len(status.attachment_filenames))
self.assertIn("s01_test_raster1_lessmmu_error.gpkg",
status.attachment_filenames)
self.assertEqual(1, len(status.messages))
self.assertIn("1 error objects under MMU limit", status.messages[0])
def test_fail_isolated_invalid_pixel(self):
from qc_tool.raster.value import run_check
self.tmp_raster = self.jobdir_manager.tmp_dir.joinpath(
"tmp_big_raster.tif")
tmp_array = np.ones((10000, 10000))
tmp_array[5555, 4444] = 253
RasterCheckTestCase.create_raster(self.tmp_raster,
np.array(tmp_array),
10,
ulx=5000,
uly=3000)
layer_defs = {
"layer_1": {
"src_filepath": self.tmp_raster,
"src_layer_name": self.tmp_raster.name
}
}
self.params.update({
"raster_layer_defs": layer_defs,
"layers": ["layer_1"]
})
self.params.update({"validcodes": [1]})
status = self.status_class()
run_check(self.params, status)
self.assertEqual("failed", status.status)
self.assertIn(
"Layer tmp_big_raster.tif has pixels with invalid values: 253.",
status.messages)
def test(self):
from qc_tool.raster.value import run_check
self.params.update({"validcodes": [0, 1]})
status = self.status_class()
run_check(self.params, status)
self.assertEqual("ok", status.status)
def test_vector_aoi_not_intersect(self):
layer_filepath1 = TEST_DATA_DIR.joinpath(
"raster", "checks", "gap", "raster_for_vector_aoi_ok.tif")
layer_defs = {
"layer_1": {
"src_filepath": layer_filepath1,
"src_layer_name": layer_filepath1.name
}
}
self.params.update({
"raster_layer_defs": layer_defs,
"aoi_code": "city001"
})
from qc_tool.raster.gap import run_check
self.params.update({
"layers": ["layer_1"],
"outside_area_code": "NODATA",
"mask": "aoi_ua_building_heights.gpkg",
"du_column_name": "CodeCITY",
"boundary_dir": TEST_DATA_DIR.joinpath("boundaries"),
"tmp_dir": self.jobdir_manager.tmp_dir,
"output_dir": self.jobdir_manager.output_dir,
"step_nr": 1
})
status = self.status_class()
run_check(self.params, status)
self.assertIn(
"does not intersect any AOI polygon with CodeCITY=city001 from boundary aoi_ua_building_heights.gpkg",
status.messages[0])
def test_fail(self):
from qc_tool.raster.value import run_check
self.params.update({"validcodes": [1, 2, 255]})
status = self.status_class()
run_check(self.params, status)
self.assertEqual(
"failed", status.status, "Value raster check should fail "
"if the raster has invalid codes.")
self.assertIn("invalid values: 0.", status.messages[0])
def test_missing_xml_fail(self):
from qc_tool.raster.inspire import run_check
self.params["raster_layer_defs"] = {
"layer0": {
"src_filepath":
self.xml_dir.joinpath("inspire_missing_xml.tif")
}
}
status = self.status_class()
run_check(self.params, status)
self.assertEqual("ok", status.status)
def test_excessive_layers(self):
from qc_tool.raster.naming import run_check
params = {
"unzip_dir": TEST_DATA_DIR.joinpath("raster", "checks", "mmu"),
"extensions": [".tif"],
"layer_names": {
"layer_1": "^test_raster1.tif$"
}
}
status = self.status_class()
run_check(params, status)
self.assertEqual("failed", status.status)
def test(self):
from qc_tool.raster.inspire import run_check
self.params["raster_layer_defs"] = {
"layer0": {
"src_filepath": self.xml_dir.joinpath("inspire_good.tif")
}
}
status = self.status_class()
run_check(self.params, status)
self.assertEqual("ok", status.status)
self.assertIn("s01_inspire_good_inspire_report.html",
status.attachment_filenames)
self.assertIn("s01_inspire_good_inspire_log.txt",
status.attachment_filenames)
def test_incorrect_aoi_code(self):
from qc_tool.raster.naming import run_check
params = {
"unzip_dir": TEST_DATA_DIR.joinpath("raster", "checks", "mmu"),
"extensions": [".tif"],
"layer_names": {
"layer_1": "^test_(?P<aoi_code>[a-z0-9]+).tif$"
},
"aoi_codes": ["aoi_1", "aoi_2"]
}
status = self.status_class()
run_check(params, status)
self.assertEqual("aborted", status.status)
self.assertIn("Layer test_raster1.tif has illegal AOI code raster1.",
status.messages)
def test_cancelled(self):
from qc_tool.raster.gap import run_check
self.params.update({
"layers": ["layer_2"],
"aoi_code": "non-existing-country",
"outside_area_code": 255,
"mask": "test",
"boundary_dir": TEST_DATA_DIR.joinpath("boundaries"),
"tmp_dir": self.jobdir_manager.tmp_dir,
"output_dir": self.jobdir_manager.output_dir,
"step_nr": 1
})
status = self.status_class()
run_check(self.params, status)
self.assertIn("cancelled", status.messages[0])
def test_xml_format_fail(self):
from qc_tool.raster.inspire import run_check
self.params["raster_layer_defs"] = {
"layer0": {
"src_filepath":
self.xml_dir.joinpath("inspire_invalid_xml.tif")
}
}
status = self.status_class()
run_check(self.params, status)
self.assertEqual("ok", status.status)
self.assertEqual(2, len(status.messages))
self.assertIn(
"The xml file inspire_invalid_xml.xml does not contain a <gmd:MD_Metadata> top-level element.",
status.messages[1])
def test_incorrect_metadata2(self):
from qc_tool.raster.inspire import run_check
self.params["raster_layer_defs"] = {
"layer0": {
"src_filepath":
self.xml_dir.joinpath("IMD_2018_010m_eu_03035_test.tif")
}
}
status = self.status_class()
run_check(self.params, status)
self.assertEqual("ok", status.status)
self.assertIn("s01_IMD_2018_010m_eu_03035_test_inspire_report.html",
status.attachment_filenames)
self.assertIn("s01_IMD_2018_010m_eu_03035_test_inspire_log.txt",
status.attachment_filenames)
def test_fail(self):
from qc_tool.raster.color import run_check
self.params.update({
"layers": ["layer_1"],
"colors": {
"0": [240, 240, 240],
"1": [70, 158, 74],
"2": [28, 92, 36],
"254": [153, 153, 153],
"255": [0, 0, 99]
}
})
status = self.status_class()
run_check(self.params, status)
self.assertEqual(
"failed", status.status,
"Color raster check with incorrect colors should fail.")
def test_reference_year(self):
from qc_tool.raster.naming import run_check
params = {
"unzip_dir": TEST_DATA_DIR.joinpath("raster", "checks", "mmu"),
"extensions": [".tif"],
"layer_names": {
"layer_1": "^test_raster1.tif$",
"layer_2": "^mmu_raster_incorrect.tif$",
"layer_3": "^mmu_raster_correct.tif"
},
"reference_year": "2018"
}
status = self.status_class()
run_check(params, status)
self.assertEqual("ok", status.status)
self.assertIn("reference_year", status.status_properties)
self.assertEqual("2018", status.status_properties["reference_year"])
def test(self):
from qc_tool.raster.gap import run_check
self.params.update({
"layers": ["layer_1"],
"aoi_code": "testaoi",
"outside_area_code": 255,
"mask": "test",
"boundary_dir": TEST_DATA_DIR.joinpath("boundaries"),
"tmp_dir": self.jobdir_manager.tmp_dir,
"output_dir": self.jobdir_manager.output_dir,
"step_nr": 1
})
status = self.status_class()
run_check(self.params, status)
self.assertEqual(
"ok", status.status, "Gap raster check should pass "
"if the raster does not have NoData values inside the AOI.")
def test_missing_layer(self):
from qc_tool.raster.naming import run_check
params = {
"unzip_dir": TEST_DATA_DIR.joinpath("raster", "checks", "mmu"),
"extensions": [".tif"],
"layer_names": {
"layer_1": "^test_raster1.tif$",
"layer_2": "^mmu_raster_incorrect.tif$",
"layer_3": "^mmu_raster_correct.tif",
"layer_4": "^non_existent_raster.tif"
}
}
status = self.status_class()
run_check(params, status)
self.assertEqual("aborted", status.status)
self.assertIn(
"The layer_4 layer name does not match naming convention.",
status.messages)
def test_bigger_than_mask(self):
mask = [[0, 1, 1, 1, 0], [1, 1, 1, 1, 0], [0, 0, 1, 1, 0],
[1, 0, 1, 1, 0]]
RasterCheckTestCase.create_raster(self.mask_filepath,
np.array(mask),
10,
ulx=5010,
uly=2990)
layer = [[0, 0, 0, 0, 0, 0, 0, 0], [0, 1, 1, 1, 1, 0, 0, 0],
[0, 1, 1, 1, 1, 0, 0, 0], [0, 0, 0, 1, 1, 0, 0, 0],
[0, 1, 0, 1, 1, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0]]
RasterCheckTestCase.create_raster(self.layer_filepath,
np.array(layer),
10,
ulx=5000,
uly=3000)
RasterCheckTestCase.create_raster(self.layer_filepath,
np.array(layer),
10,
ulx=5010,
uly=2990)
self.params.update({
"layers": ["layer_3"],
"boundary_dir":
self.jobdir_manager.tmp_dir.joinpath("boundaries"),
"tmp_dir":
self.jobdir_manager.tmp_dir,
"output_dir":
self.jobdir_manager.output_dir,
"aoi_code":
"aoi01",
"outside_area_code":
9,
"step_nr":
1
})
from qc_tool.raster.gap import run_check
status = self.status_class()
run_check(self.params, status)
self.assertEqual("ok", status.status)
def test_gaps_found(self):
from qc_tool.raster.gap import run_check
self.params.update({
"layers": ["layer_2"],
"aoi_code": "testaoi",
"outside_area_code": 255,
"mask": "test",
"boundary_dir": TEST_DATA_DIR.joinpath("boundaries"),
"tmp_dir": self.jobdir_manager.tmp_dir,
"output_dir": self.jobdir_manager.output_dir,
"step_nr": 1
})
status = self.status_class()
run_check(self.params, status)
self.assertIn("has 1237 gap pixels", status.messages[0])
self.assertIn("s01_incomplete_raster_100m_testaoi_gap_warning.tif",
status.attachment_filenames)
self.assertTrue(self.params["output_dir"].joinpath(
status.attachment_filenames[0]).exists())
def test(self):
# Prepare raster dataset.
data = [[0, 1, 0, 255], [0, 0, 9, 255], [0, 0, 0, 255]]
RasterCheckTestCase.create_raster(self.tmp_raster,
np.array(data),
10,
ulx=0,
uly=0,
epsg=3035,
nodata_value=255)
self.params.update({
"layers": ["raster"],
"nodata_value": 255,
"step_nr": 1
})
from qc_tool.raster.nodata import run_check
status = self.status_class()
run_check(self.params, status)
self.assertEqual("ok", status.status)
def test_intersects_mask_lower_right(self):
# Prepare mask.
mask = [[0, 0, 0, 0, 0, 0, 0, 0], [0, 1, 1, 1, 1, 0, 0, 0],
[0, 1, 1, 1, 1, 0, 0, 0], [0, 0, 0, 1, 1, 0, 0, 0],
[0, 1, 0, 1, 1, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0]]
RasterCheckTestCase.create_raster(self.mask_filepath,
np.array(mask),
10,
ulx=5000,
uly=3000)
# Prepare raster located partially outside the mask.
layer = [[2, 1, 1, 2], [1, 1, 1, 9], [9, 9, 2, 2]]
RasterCheckTestCase.create_raster(self.layer_filepath,
np.array(layer),
10,
ulx=5050,
uly=2960)
self.params.update({
"layers": ["layer_3"],
"boundary_dir":
self.jobdir_manager.tmp_dir.joinpath("boundaries"),
"tmp_dir":
self.jobdir_manager.tmp_dir,
"output_dir":
self.jobdir_manager.output_dir,
"aoi_code":
"aoi01",
"outside_area_code":
9,
"step_nr":
1
})
from qc_tool.raster.gap import run_check
status = self.status_class()
run_check(self.params, status)
self.assertEqual("ok", status.status)
def test(self):
from qc_tool.raster.naming import run_check
params = {
"unzip_dir":
TEST_DATA_DIR.joinpath("raster", "fty_100m",
"fty_2018_100m_eu_03035_v0_1"),
"aoi_codes": ["mt", "eu"],
"extensions": [".tif"],
"reference_year":
"2018",
"layer_names": {
"layer_1": "^fty_2018_100m_(?P<aoi_code>.+)_[0-9]{5}.*.tif$"
}
}
status = self.status_class()
run_check(params, status)
self.assertEqual("ok", status.status)
self.assertEqual("eu", status.params["aoi_code"])
self.assertEqual("2018", status.status_properties["reference_year"])
self.assertEqual(
"fty_2018_100m_eu_03035_v0_1.tif",
status.params["raster_layer_defs"]["layer_1"]["src_layer_name"])
def test_fail_nodata_value_mismatch(self):
# Prepare raster dataset.
data = [[0, 1, 0, 255], [0, 0, 9, 255], [0, 0, 0, 255]]
RasterCheckTestCase.create_raster(self.tmp_raster,
np.array(data),
10,
ulx=0,
uly=0,
epsg=3035,
nodata_value=0)
self.params.update({
"layers": ["raster"],
"nodata_value": 255,
"step_nr": 1
})
from qc_tool.raster.nodata import run_check
status = self.status_class()
run_check(self.params, status)
self.assertEqual("aborted", status.status)
self.assertIn("has invalid NoData value: 0", status.messages[0])
