def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "-s", "--source-dir", required=True, help="Source directory, as downloaded and produced by " "`download-from-drive` script." ) parser.add_argument( "-v", "--version", required=True, choices=ALL_VERSIONS, help=f"Version of the spreadsheets to use (e.g. '{CURRENT_VERSION}')." ) args = parser.parse_args(sys.argv[1:]) if not os.path.isdir(args.source_dir): parser.error(f"No such directory '{args.source_dir}'") version_dir = os.path.join(args.source_dir, args.version) sh = SpreadsheetHandler(version_dir) checks_dir = os.path.join(version_dir, "amf-checks") if not os.path.isdir(checks_dir): os.makedirs(checks_dir) sh.write_yaml(checks_dir)
def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "spreadsheets_dir", help="Directory containing spreadsheet data, as produced by " "download_from_drive.py") parser.add_argument("output_dir", help="Directory to write output YAML files to") args = parser.parse_args(sys.argv[1:]) if not os.path.isdir(args.spreadsheets_dir): parser.error("No such directory '{}'".format(args.spreadsheets_dir)) if not os.path.isdir(args.output_dir): os.mkdir(args.output_dir) args.spreadsheets_dir = os.path.join(args.spreadsheets_dir, 'product-definitions') sh = SpreadsheetHandler(args.spreadsheets_dir) sh.write_yaml(args.output_dir)
def test_global_attrs_yaml_check(self, spreadsheets_dir, tmpdir): s_dir = spreadsheets_dir global_attrs_tsv = s_dir.join("Common.xlsx").join( "Global Attributes.tsv") global_attrs_tsv.write("\n".join(( "Name\tDescription\tExample\tFixed Value\tCompliance checking rules\tConvention Providence", "myattr\td\te\t\tValid email\tc", "aaaa\td\te\t\tSomething strange here\tc", "bbbb\td\te\tf", "otherattr\td\te\t\tInteger\tc"))) sh = SpreadsheetHandler(str(s_dir)) output = tmpdir.mkdir("yaml") sh.write_yaml(str(output)) global_attr_yaml = output.join("AMF_global_attrs.yml") assert global_attr_yaml.check() decoded = yaml.load(global_attr_yaml.read()) assert "suite_name" in decoded assert "checks" in decoded assert decoded["suite_name"] == "global_attrs_checks" assert len(decoded["checks"]) == 2 assert decoded["checks"][0][ "check_id"] == "check_myattr_global_attribute" expected_check_name = "checklib.register.nc_file_checks_register.GlobalAttrRegexCheck" assert decoded["checks"][0]["check_name"] == expected_check_name assert decoded["checks"][1]["check_name"] == expected_check_name attr, email_regex = GlobalAttrCheck.parse_row({ "Name": "myattr", "Description": "d", "Example": "e", "Fixed Value": "", "Compliance checking rules": "Valid email", "Convention Providence": "" }) assert decoded["checks"][0]["parameters"]["regex"] == email_regex
def test_file_info_yaml_check(self, spreadsheets_dir, tmpdir): sh = SpreadsheetHandler(str(spreadsheets_dir)) output = tmpdir.mkdir("yaml") sh.write_yaml(str(output)) file_info_check = output.join("AMF_file_info.yml") assert file_info_check.check() assert yaml.load(file_info_check.read()) == { "suite_name": "file_info_checks", "description": "Check 'file_info' in AMF files", "checks": [{ "check_id": "check_soft_file_size_limit", "check_name": "checklib.register.file_checks_register.FileSizeCheck", "check_level": "LOW", "parameters": { "threshold": 2, "strictness": "soft" } }, { "check_id": "check_hard_file_size_limit", "check_name": "checklib.register.file_checks_register.FileSizeCheck", "check_level": "HIGH", "parameters": { "threshold": 4, "strictness": "hard" } }, { "check_id": "check_filename_structure", "check_name": "checklib.register.file_checks_register.FileNameStructureCheck", "check_level": "HIGH", "parameters": { "delimiter": "_", "extension": ".nc" } }] }
def test_file_structure_yaml_check(self, spreadsheets_dir, tmpdir): sh = SpreadsheetHandler(str(spreadsheets_dir)) output = tmpdir.mkdir("yaml") sh.write_yaml(str(output)) file_structure_check = output.join("AMF_file_structure.yml") assert file_structure_check.check() assert yaml.load(file_structure_check.read()) == { "suite_name": "file_structure_checks", "description": "Check 'file_structure' in AMF files", "checks": [{ "check_id": "check_valid_netcdf4_file", "check_name": "checklib.register.nc_file_checks_register.NetCDFFormatCheck", "parameters": { "format": "NETCDF4_CLASSIC" } }] }
def test_common(self, spreadsheets_dir, tmpdir): s_dir = spreadsheets_dir common_dir = s_dir.join("Common.xlsx") var_air = common_dir.join("Variables - Air.tsv") var_sea = common_dir.join("Variables - Sea.tsv") dim_land = common_dir.join("Dimensions - Land.tsv") var_air.write("\n".join( ("Variable\tAttribute\tValue", "some_air_variable\t\t", "\tthingy\tthis_thing", "\ttype\tfloat32"))) var_sea.write("\n".join( ("Variable\tAttribute\tValue", "some_sea_variable\t\t", "\tthingy\tthat_thing", "\ttype\tstring"))) dim_land.write("\n".join(("Name\tLength\tunits", "some_dim\t42\tm"))) sh = SpreadsheetHandler(str(s_dir)) cv_output = tmpdir.mkdir("cvs") yaml_output = tmpdir.mkdir("yaml") sh.write_cvs(str(cv_output)) sh.write_yaml(str(yaml_output)) # Check CV and YAML files exist var_air_output = cv_output.join("AMF_product_common_variable_air.json") assert var_air_output.check() assert cv_output.join("AMF_product_common_variable_sea.json").check() assert cv_output.join("AMF_product_common_dimension_land.json").check() assert yaml_output.join("AMF_product_common_variable_air.yml").check() assert yaml_output.join("AMF_product_common_variable_sea.yml").check() # Check the content of one of the CVs assert json.load(var_air_output) == { "product_common_variable_air": { "some_air_variable": { "thingy": "this_thing", "type": "float32" } } }
def test_top_level_product_yaml(self, spreadsheets_dir, tmpdir): """ Check that a top level YAML file is written for each product/deployment mode combination, which includes other YAML files """ s_dir = spreadsheets_dir # Write common variables and dimensions common_dir = s_dir.join("Common.xlsx") common_dir.join("Variables - Air.tsv").write("\n".join( ("Variable\tAttribute\tValue", "some_air_variable\t\t", "\ttype\tfloat32"))) common_dir.join("Variables - Land.tsv").write("\n".join( ("Variable\tAttribute\tValue", "some_land_variable\t\t", "\ttype\tfloat32"))) common_dir.join("Dimensions - Land.tsv").write("\n".join( ("Name\tLength\tunits", "index\t<i>\t1"))) # Write common global attributes common_dir.join("Global Attributes.tsv").write("\n".join(( "Name\tDescription\tExample\tFixed Value\tCompliance checking rules\tConvention Providence", "someattr\ta\tb\tc\tInteger\td"))) # Write product variables and dimensions soil_dir = (s_dir.join("Product Definition Spreadsheets").mkdir( "soil").mkdir("soil.xlsx")) soil_dir.join("Variables - Specific.tsv").write("\n".join( ("Variable\tAttribute\tValue", "soil_var\t\t", "\ttype\tfloat32"))) soil_dir.join("Dimensions - Specific.tsv").write("\n".join( ("Name\tLength\tunits", "somedim\t<n>\tK"))) sh = SpreadsheetHandler(str(s_dir)) yaml_output = tmpdir.mkdir("yaml") sh.write_yaml(str(yaml_output)) assert yaml_output.join("AMF_product_common_variable_air.yml").check() assert yaml_output.join("AMF_product_common_variable_land.yml").check() assert yaml_output.join( "AMF_product_common_dimension_land.yml").check() assert yaml_output.join("AMF_file_info.yml").check() assert yaml_output.join("AMF_file_structure.yml").check() assert yaml_output.join("AMF_global_attrs.yml").check() top_level_air = yaml_output.join("AMF_product_soil_air.yml") top_level_land = yaml_output.join("AMF_product_soil_land.yml") assert top_level_air.check() assert top_level_land.check() assert yaml.load(top_level_air.read()) == { "suite_name": "product_soil_air_checks", "description": "Check 'product soil air' in AMF files", "checks": [ # Global checks { "__INCLUDE__": "AMF_file_info.yml" }, { "__INCLUDE__": "AMF_file_structure.yml" }, { "__INCLUDE__": "AMF_global_attrs.yml" }, # Common product checks { "__INCLUDE__": "AMF_product_common_variable_air.yml" }, # Product specific { "__INCLUDE__": "AMF_product_soil_dimension.yml" }, { "__INCLUDE__": "AMF_product_soil_variable.yml" } ] } # Land one should be basically the same as air but s/air/land/, and # there is also common dimensions CV for land assert yaml.load(top_level_land.read()) == { "suite_name": "product_soil_land_checks", "description": "Check 'product soil land' in AMF files", "checks": [{ "__INCLUDE__": "AMF_file_info.yml" }, { "__INCLUDE__": "AMF_file_structure.yml" }, { "__INCLUDE__": "AMF_global_attrs.yml" }, { "__INCLUDE__": "AMF_product_common_dimension_land.yml" }, { "__INCLUDE__": "AMF_product_common_variable_land.yml" }, { "__INCLUDE__": "AMF_product_soil_dimension.yml" }, { "__INCLUDE__": "AMF_product_soil_variable.yml" }] }
def test_basic(self, spreadsheets_dir, tmpdir): s_dir = spreadsheets_dir var = (s_dir.join("Product Definition Spreadsheets").mkdir( "my-great-product").mkdir("my-great-product.xlsx").join( "Variables - Specific.tsv")) var.write("\n".join( ("Variable\tAttribute\tValue", "wind_speed\t\t", "\tname\twind_speed", "\ttype\tfloat32", "eastward_wind\t\t", "\tname\teastward_wind", "\ttype\tfloat32", "\tunits\tm s-1"))) dim = (s_dir.join("Product Definition Spreadsheets").join( "my-great-product").join("my-great-product.xlsx").join( "Dimensions - Specific.tsv")) dim.write("\n".join( ("Name\tLength\tunits", "one\t1\tm", "two\t2\tkm"))) output = tmpdir.mkdir("yaml") sh = SpreadsheetHandler(str(s_dir)) sh.write_yaml(str(output)) var_output_yml = output.join( "AMF_product_my-great-product_variable.yml") dim_output_yml = output.join( "AMF_product_my-great-product_dimension.yml") assert var_output_yml.check() assert dim_output_yml.check() decoded = [] for f in (var_output_yml, dim_output_yml): try: decoded.append(yaml.load(f.read())) except yaml.parser.ParserError: assert False, "{} is invalid YAML".format(str(f)) assert decoded[0] == { "suite_name": "product_my-great-product_variable_checks", "description": "Check 'product my-great-product variable' in AMF files", "checks": [ { "check_id": "check_wind_speed_variable_attrs", "check_name": "checklib.register.nc_file_checks_register.NCVariableMetadataCheck", "comments": "Checks the variable attributes for 'wind_speed'", "parameters": { "pyessv_namespace": "product_my-great-product_variable", "var_id": "wind_speed", "vocabulary_ref": "ncas:amf" } }, { "check_id": "check_wind_speed_variable_type", "check_name": "checklib.register.nc_file_checks_register.VariableTypeCheck", "comments": "Checks the type of variable 'wind_speed'", "parameters": { "var_id": "wind_speed", "dtype": "float32", "vocabulary_ref": "ncas:amf" } }, { "check_id": "check_eastward_wind_variable_attrs", "check_name": "checklib.register.nc_file_checks_register.NCVariableMetadataCheck", "comments": "Checks the variable attributes for 'eastward_wind'", "parameters": { "pyessv_namespace": "product_my-great-product_variable", "var_id": "eastward_wind", "vocabulary_ref": "ncas:amf" } }, { "check_id": "check_eastward_wind_variable_type", "check_name": "checklib.register.nc_file_checks_register.VariableTypeCheck", "comments": "Checks the type of variable 'eastward_wind'", "parameters": { "var_id": "eastward_wind", "dtype": "float32", "vocabulary_ref": "ncas:amf" } }, ] } assert decoded[1] == { "suite_name": "product_my-great-product_dimension_checks", "description": "Check 'product my-great-product dimension' in AMF files", "checks": [{ "check_id": "check_one_dimension_attrs", "check_name": "checklib.register.nc_file_checks_register.NetCDFDimensionCheck", "comments": "Checks the dimension attributes for 'one'", "parameters": { "pyessv_namespace": "product_my-great-product_dimension", "dim_id": "one", "vocabulary_ref": "ncas:amf" } }, { "check_id": "check_two_dimension_attrs", "check_name": "checklib.register.nc_file_checks_register.NetCDFDimensionCheck", "comments": "Checks the dimension attributes for 'two'", "parameters": { "pyessv_namespace": "product_my-great-product_dimension", "dim_id": "two", "vocabulary_ref": "ncas:amf" } }] }