def main():
# Parse command line arguments.
parser = ArgumentParser()
parser.add_argument("--job-uuid",
help="The identifier of the job being run.",
dest="job_uuid",
action="store",
nargs=1,
required=True)
parser.add_argument("--product",
help="Identifier of the product.",
dest="product_ident",
action="store",
nargs=1,
required=True)
parser.add_argument("--skip-steps",
help="Comma separated ordinal numbers of job steps to be skipped.",
dest="skip_steps",
default=None,
action="store",
nargs=1,
required=False)
parser.add_argument("username",
help="The name of the user managing the delivery.",
action="store",
nargs=1)
parser.add_argument("filename",
help="Path to delivery file relative to INCOMING_DIR.",
action="store",
nargs=1)
pargs = parser.parse_args()
job_uuid = pargs.job_uuid[0]
username = pargs.username[0]
filename = pargs.filename[0]
filepath = CONFIG["incoming_dir"].joinpath(username, filename)
if pargs.skip_steps is None:
skip_steps = tuple()
else:
skip_steps = tuple(int(i) for i in pargs.skip_steps[0].split(","))
# Create job dir.
job_dir = create_job_dir(job_uuid)
# Set up logging.
init_logging(job_dir)
log.info("Logging of the job {:s} has been started.".format(job_uuid))
# Run the checks.
dispatch(job_uuid, username, filepath, pargs.product_ident[0], skip_steps)
def test_status_json(self):
from qc_tool.common import load_job_result
filepath = TEST_DATA_DIR.joinpath("vector", "clc",
"clc2012_mt.gdb.zip")
job_result = dispatch(self.job_uuid, "user_name", filepath, "clc2012",
[])
job_result_from_file = load_job_result(self.job_uuid)
self.assertEqual(
job_result, job_result_from_file,
"Job result returned by dispatch() must be the same as job result stored in json file."
)
def test(self):
filepath = TEST_DATA_DIR.joinpath("vector", "clc",
"clc2012_mt.gdb.zip")
expected_step_results = ["ok"] * 24
# vector.inspire check is skipped
expected_step_results[7] = "skipped"
job_result = dispatch(self.job_uuid, "user_name", filepath, "clc2012",
(8, ))
step_results = [
step_result["status"] for step_result in job_result["steps"]
]
self.assertListEqual(expected_step_results, step_results)
def test(self):
filepath = TEST_DATA_DIR.joinpath("vector", "rpz",
"rpz_LCLU2012_DU007T.zip")
expected_step_results = ["ok"] * 16
# vector.inspire check is skipped
expected_step_results[5] = "skipped"
job_result = dispatch(self.job_uuid, "user_name", filepath, "rpz_2012",
(6, ))
step_results = [
step_result["status"] for step_result in job_result["steps"]
]
self.assertListEqual(expected_step_results, step_results)
def test(self):
filepath = TEST_DATA_DIR.joinpath(
"vector", "n2k", "gpkg",
"N2K_DU001A_Status2018_LCLU_v1_20200915.gpkg.zip")
expected_step_results = ["ok"] * 17
# vector.inspire check is skipped
expected_step_results[5] = "skipped"
job_result = dispatch(self.job_uuid, "user_name", filepath, "n2k_2018",
(6, ))
step_results = [
step_result["status"] for step_result in job_result["steps"]
]
self.assertListEqual(expected_step_results, step_results)
def test_gdb(self):
filepath = TEST_DATA_DIR.joinpath("vector", "cz", "gpkg",
"CZ_2018_DU001_3035_V1_2.gpkg.zip")
expected_step_results = ["ok"] * 17
# vector.inspire check is skipped
expected_step_results[5] = "skipped"
job_result = dispatch(self.job_uuid, "user_name", filepath, "cz_2018",
(6, ))
step_results = [
step_result["status"] for step_result in job_result["steps"]
]
self.maxDiff = None
self.assertListEqual(expected_step_results, step_results)
def test_gpkg(self):
filepath = TEST_DATA_DIR.joinpath("vector", "ua", "gpkg",
"EE003L1_NARVA_UA2018_stl.gpkg.zip")
expected_step_results = ["ok"] * 14
# vector.inspire check is skipped
expected_step_results[5] = "skipped"
job_result = dispatch(self.job_uuid, "user_name", filepath,
"ua2018_stl", (6, ))
step_results = [
step_result["status"] for step_result in job_result["steps"]
]
self.maxDiff = None
self.assertListEqual(expected_step_results, step_results)
def test_very_long_layer_name(self):
filepath = TEST_DATA_DIR.joinpath(
"vector", "ua", "gpkg",
"UK568L1_CHESHIRE_WEST_AND_CHESTER_change_2012_2018.zip")
expected_step_results = ["ok"] * 16
# vector.inspire check is skipped
expected_step_results[5] = "skipped"
expected_step_results[13] = "skipped"
job_result = dispatch(self.job_uuid, "user_name", filepath,
"ua_change_2012_2018", (6, 14))
step_results = [
step_result["status"] for step_result in job_result["steps"]
]
self.maxDiff = None
self.assertListEqual(expected_step_results, step_results)
def test_general_raster(self):
"""General raster product"""
product_ident = "general_raster"
filepath = self.raster_data_dir.joinpath("general_raster",
"general_raster.zip")
expected_step_results = ["ok"] * 9
# inspire check is skipped
expected_step_results[2] = "skipped"
job_result = dispatch(self.job_uuid, self.username, filepath,
product_ident, (3, ))
step_statuses = [
step_result["status"] for step_result in job_result["steps"]
]
self.assertListEqual(expected_step_results, step_statuses)
def test_gra_2018_010m(self):
"""High resolution grassland (GRA) - 10m"""
product_ident = "gra_2018_010m"
filepath = self.raster_data_dir.joinpath(
"gra_010m", "GRA_2018_010m_eu_03035_v0_1.zip")
expected_step_results = ["ok"] * 13
expected_step_results[2] = "skipped"
job_result = dispatch(self.job_uuid, self.username, filepath,
product_ident, (3, ))
step_statuses = [
step_result["status"] for step_result in job_result["steps"]
]
self.assertListEqual(expected_step_results, step_statuses,
self.show_messages(job_result))
def test_swf_2015_100m(self):
"""High resolution small woody features - 100m raster"""
product_ident = "swf_2015_100m"
filepath = TEST_DATA_DIR.joinpath("raster", "swf_100m",
"swf_2015_100m_eu_3035_v1_1.zip")
expected_step_results = ["ok"] * 10
# inspire check is skipped
expected_step_results[2] = "skipped"
job_result = dispatch(self.job_uuid, "user_name", filepath,
product_ident, (3, ))
step_results = [
step_result["status"] for step_result in job_result["steps"]
]
self.assertListEqual(expected_step_results, step_results)
def test_ua2012_dhm(self):
"""Urban Atlas 2012 Building Heights - 10m raster"""
product_ident = "ua2012_dhm"
filepath = TEST_DATA_DIR.joinpath("raster", "ua2012_dhm",
"EE003Ly_NARVA_ua2012_dhm.zip")
expected_step_results = ["ok"] * 12
# inspire check is skipped
expected_step_results[2] = "skipped"
job_result = dispatch(self.job_uuid, "user_name", filepath,
product_ident, (3, ))
print(job_result)
step_results = [
step_result["status"] for step_result in job_result["steps"]
]
self.assertListEqual(expected_step_results, step_results)
def test_tcd_2018_100m(self):
"""High resolution tree cover density (TCD) - 100m"""
product_ident = "tcd_2018_100m"
filepath = self.raster_data_dir.joinpath(
"tcd_100m", "TCD_2018_100m_eu_03035_v0_1.zip")
expected_step_results = ["ok"] * 12
# inspire check is skipped
expected_step_results[2] = "skipped"
job_result = dispatch(self.job_uuid, self.username, filepath,
product_ident, (3, ))
step_statuses = [
step_result["status"] for step_result in job_result["steps"]
]
self.assertListEqual(expected_step_results, step_statuses,
self.show_messages(job_result))
def test(self):
filepath = TEST_DATA_DIR.joinpath(
"vector_raster", "swf_2015_vec_ras",
"swf_2015_vec_ras_FR_3035_123_pt01.zip")
expected_step_results = ["ok"] * 30
# vector.inspire check is skipped
expected_step_results[16] = "skipped"
# vector.area in the testing data does not match
expected_step_results[20] = "failed"
job_result = dispatch(self.job_uuid, "user_name", filepath,
"swf_2015_vec_ras", (17, ))
step_results = [
step_result["status"] for step_result in job_result["steps"]
]
self.assertListEqual(expected_step_results, step_results)
self.assertListEqual(
['Layer vector has error features with row number: 10.'],
job_result["steps"][20]["messages"])
def test_imd_2018_100m(self):
"""High resolution imperviousness density (IMD) - 100m"""
product_ident = "imd_2018_100m"
filepath = self.raster_data_dir.joinpath(
"imd_100m", "IMD_2018_100m_eu_03035_v1_0.zip")
expected_step_results = ["ok"] * 12
# inspire check is skipped
expected_step_results[2] = "skipped"
# imd_2018_100m has mismatching attributes
expected_step_results[3] = "failed"
job_result = dispatch(self.job_uuid, self.username, filepath,
product_ident, (3, ))
step_statuses = [
step_result["status"] for step_result in job_result["steps"]
]
self.assertListEqual(expected_step_results, step_statuses,
self.show_messages(job_result))
def test_fty_2018_010m(self):
"""High resolution forest type (FTY) - 10m"""
product_ident = "fty_2018_010m"
filepath = self.raster_data_dir.joinpath(
"fty_010m", "FTY_2018_010m_eu_03035_v0_1.zip")
expected_step_results = ["ok"] * 13
# inspire check is skipped
expected_step_results[2] = "skipped"
# fty_010m has extra checks raster.tile and raster.mmu
expected_step_results[11] = "failed"
job_result = dispatch(self.job_uuid, self.username, filepath,
product_ident, (3, ))
step_statuses = [
step_result["status"] for step_result in job_result["steps"]
]
self.assertListEqual(expected_step_results, step_statuses,
self.show_messages(job_result))
