def main(): # Parse command line arguments. parser = ArgumentParser() parser.add_argument("--job-uuid", help="The identifier of the job being run.", dest="job_uuid", action="store", nargs=1, required=True) parser.add_argument("--product", help="Identifier of the product.", dest="product_ident", action="store", nargs=1, required=True) parser.add_argument("--skip-steps", help="Comma separated ordinal numbers of job steps to be skipped.", dest="skip_steps", default=None, action="store", nargs=1, required=False) parser.add_argument("username", help="The name of the user managing the delivery.", action="store", nargs=1) parser.add_argument("filename", help="Path to delivery file relative to INCOMING_DIR.", action="store", nargs=1) pargs = parser.parse_args() job_uuid = pargs.job_uuid[0] username = pargs.username[0] filename = pargs.filename[0] filepath = CONFIG["incoming_dir"].joinpath(username, filename) if pargs.skip_steps is None: skip_steps = tuple() else: skip_steps = tuple(int(i) for i in pargs.skip_steps[0].split(",")) # Create job dir. job_dir = create_job_dir(job_uuid) # Set up logging. init_logging(job_dir) log.info("Logging of the job {:s} has been started.".format(job_uuid)) # Run the checks. dispatch(job_uuid, username, filepath, pargs.product_ident[0], skip_steps)
def test_status_json(self): from qc_tool.common import load_job_result filepath = TEST_DATA_DIR.joinpath("vector", "clc", "clc2012_mt.gdb.zip") job_result = dispatch(self.job_uuid, "user_name", filepath, "clc2012", []) job_result_from_file = load_job_result(self.job_uuid) self.assertEqual( job_result, job_result_from_file, "Job result returned by dispatch() must be the same as job result stored in json file." )
def test(self): filepath = TEST_DATA_DIR.joinpath("vector", "clc", "clc2012_mt.gdb.zip") expected_step_results = ["ok"] * 24 # vector.inspire check is skipped expected_step_results[7] = "skipped" job_result = dispatch(self.job_uuid, "user_name", filepath, "clc2012", (8, )) step_results = [ step_result["status"] for step_result in job_result["steps"] ] self.assertListEqual(expected_step_results, step_results)
def test(self): filepath = TEST_DATA_DIR.joinpath("vector", "rpz", "rpz_LCLU2012_DU007T.zip") expected_step_results = ["ok"] * 16 # vector.inspire check is skipped expected_step_results[5] = "skipped" job_result = dispatch(self.job_uuid, "user_name", filepath, "rpz_2012", (6, )) step_results = [ step_result["status"] for step_result in job_result["steps"] ] self.assertListEqual(expected_step_results, step_results)
def test(self): filepath = TEST_DATA_DIR.joinpath( "vector", "n2k", "gpkg", "N2K_DU001A_Status2018_LCLU_v1_20200915.gpkg.zip") expected_step_results = ["ok"] * 17 # vector.inspire check is skipped expected_step_results[5] = "skipped" job_result = dispatch(self.job_uuid, "user_name", filepath, "n2k_2018", (6, )) step_results = [ step_result["status"] for step_result in job_result["steps"] ] self.assertListEqual(expected_step_results, step_results)
def test_gdb(self): filepath = TEST_DATA_DIR.joinpath("vector", "cz", "gpkg", "CZ_2018_DU001_3035_V1_2.gpkg.zip") expected_step_results = ["ok"] * 17 # vector.inspire check is skipped expected_step_results[5] = "skipped" job_result = dispatch(self.job_uuid, "user_name", filepath, "cz_2018", (6, )) step_results = [ step_result["status"] for step_result in job_result["steps"] ] self.maxDiff = None self.assertListEqual(expected_step_results, step_results)
def test_gpkg(self): filepath = TEST_DATA_DIR.joinpath("vector", "ua", "gpkg", "EE003L1_NARVA_UA2018_stl.gpkg.zip") expected_step_results = ["ok"] * 14 # vector.inspire check is skipped expected_step_results[5] = "skipped" job_result = dispatch(self.job_uuid, "user_name", filepath, "ua2018_stl", (6, )) step_results = [ step_result["status"] for step_result in job_result["steps"] ] self.maxDiff = None self.assertListEqual(expected_step_results, step_results)
def test_very_long_layer_name(self): filepath = TEST_DATA_DIR.joinpath( "vector", "ua", "gpkg", "UK568L1_CHESHIRE_WEST_AND_CHESTER_change_2012_2018.zip") expected_step_results = ["ok"] * 16 # vector.inspire check is skipped expected_step_results[5] = "skipped" expected_step_results[13] = "skipped" job_result = dispatch(self.job_uuid, "user_name", filepath, "ua_change_2012_2018", (6, 14)) step_results = [ step_result["status"] for step_result in job_result["steps"] ] self.maxDiff = None self.assertListEqual(expected_step_results, step_results)
def test_general_raster(self): """General raster product""" product_ident = "general_raster" filepath = self.raster_data_dir.joinpath("general_raster", "general_raster.zip") expected_step_results = ["ok"] * 9 # inspire check is skipped expected_step_results[2] = "skipped" job_result = dispatch(self.job_uuid, self.username, filepath, product_ident, (3, )) step_statuses = [ step_result["status"] for step_result in job_result["steps"] ] self.assertListEqual(expected_step_results, step_statuses)
def test_gra_2018_010m(self): """High resolution grassland (GRA) - 10m""" product_ident = "gra_2018_010m" filepath = self.raster_data_dir.joinpath( "gra_010m", "GRA_2018_010m_eu_03035_v0_1.zip") expected_step_results = ["ok"] * 13 expected_step_results[2] = "skipped" job_result = dispatch(self.job_uuid, self.username, filepath, product_ident, (3, )) step_statuses = [ step_result["status"] for step_result in job_result["steps"] ] self.assertListEqual(expected_step_results, step_statuses, self.show_messages(job_result))
def test_swf_2015_100m(self): """High resolution small woody features - 100m raster""" product_ident = "swf_2015_100m" filepath = TEST_DATA_DIR.joinpath("raster", "swf_100m", "swf_2015_100m_eu_3035_v1_1.zip") expected_step_results = ["ok"] * 10 # inspire check is skipped expected_step_results[2] = "skipped" job_result = dispatch(self.job_uuid, "user_name", filepath, product_ident, (3, )) step_results = [ step_result["status"] for step_result in job_result["steps"] ] self.assertListEqual(expected_step_results, step_results)
def test_ua2012_dhm(self): """Urban Atlas 2012 Building Heights - 10m raster""" product_ident = "ua2012_dhm" filepath = TEST_DATA_DIR.joinpath("raster", "ua2012_dhm", "EE003Ly_NARVA_ua2012_dhm.zip") expected_step_results = ["ok"] * 12 # inspire check is skipped expected_step_results[2] = "skipped" job_result = dispatch(self.job_uuid, "user_name", filepath, product_ident, (3, )) print(job_result) step_results = [ step_result["status"] for step_result in job_result["steps"] ] self.assertListEqual(expected_step_results, step_results)
def test_tcd_2018_100m(self): """High resolution tree cover density (TCD) - 100m""" product_ident = "tcd_2018_100m" filepath = self.raster_data_dir.joinpath( "tcd_100m", "TCD_2018_100m_eu_03035_v0_1.zip") expected_step_results = ["ok"] * 12 # inspire check is skipped expected_step_results[2] = "skipped" job_result = dispatch(self.job_uuid, self.username, filepath, product_ident, (3, )) step_statuses = [ step_result["status"] for step_result in job_result["steps"] ] self.assertListEqual(expected_step_results, step_statuses, self.show_messages(job_result))
def test(self): filepath = TEST_DATA_DIR.joinpath( "vector_raster", "swf_2015_vec_ras", "swf_2015_vec_ras_FR_3035_123_pt01.zip") expected_step_results = ["ok"] * 30 # vector.inspire check is skipped expected_step_results[16] = "skipped" # vector.area in the testing data does not match expected_step_results[20] = "failed" job_result = dispatch(self.job_uuid, "user_name", filepath, "swf_2015_vec_ras", (17, )) step_results = [ step_result["status"] for step_result in job_result["steps"] ] self.assertListEqual(expected_step_results, step_results) self.assertListEqual( ['Layer vector has error features with row number: 10.'], job_result["steps"][20]["messages"])
def test_imd_2018_100m(self): """High resolution imperviousness density (IMD) - 100m""" product_ident = "imd_2018_100m" filepath = self.raster_data_dir.joinpath( "imd_100m", "IMD_2018_100m_eu_03035_v1_0.zip") expected_step_results = ["ok"] * 12 # inspire check is skipped expected_step_results[2] = "skipped" # imd_2018_100m has mismatching attributes expected_step_results[3] = "failed" job_result = dispatch(self.job_uuid, self.username, filepath, product_ident, (3, )) step_statuses = [ step_result["status"] for step_result in job_result["steps"] ] self.assertListEqual(expected_step_results, step_statuses, self.show_messages(job_result))
def test_fty_2018_010m(self): """High resolution forest type (FTY) - 10m""" product_ident = "fty_2018_010m" filepath = self.raster_data_dir.joinpath( "fty_010m", "FTY_2018_010m_eu_03035_v0_1.zip") expected_step_results = ["ok"] * 13 # inspire check is skipped expected_step_results[2] = "skipped" # fty_010m has extra checks raster.tile and raster.mmu expected_step_results[11] = "failed" job_result = dispatch(self.job_uuid, self.username, filepath, product_ident, (3, )) step_statuses = [ step_result["status"] for step_result in job_result["steps"] ] self.assertListEqual(expected_step_results, step_statuses, self.show_messages(job_result))