def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"-s", "--source-dir", required=True,
help="Source directory, as downloaded and produced by "
"`download-from-drive` script."
)
parser.add_argument(
"-v", "--version", required=True, choices=ALL_VERSIONS,
help=f"Version of the spreadsheets to use (e.g. '{CURRENT_VERSION}')."
)
args = parser.parse_args(sys.argv[1:])
if not os.path.isdir(args.source_dir):
parser.error(f"No such directory '{args.source_dir}'")
version_dir = os.path.join(args.source_dir, args.version)
sh = SpreadsheetHandler(version_dir)
checks_dir = os.path.join(version_dir, "amf-checks")
if not os.path.isdir(checks_dir):
os.makedirs(checks_dir)
sh.write_yaml(checks_dir)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"spreadsheets_dir",
help="Directory containing spreadsheet data, as produced by "
"download_from_drive.py")
parser.add_argument("output_dir",
help="Directory to write output YAML files to")
args = parser.parse_args(sys.argv[1:])
if not os.path.isdir(args.spreadsheets_dir):
parser.error("No such directory '{}'".format(args.spreadsheets_dir))
if not os.path.isdir(args.output_dir):
os.mkdir(args.output_dir)
args.spreadsheets_dir = os.path.join(args.spreadsheets_dir,
'product-definitions')
sh = SpreadsheetHandler(args.spreadsheets_dir)
sh.write_yaml(args.output_dir)
def test_global_attrs_yaml_check(self, spreadsheets_dir, tmpdir):
s_dir = spreadsheets_dir
global_attrs_tsv = s_dir.join("Common.xlsx").join(
"Global Attributes.tsv")
global_attrs_tsv.write("\n".join((
"Name\tDescription\tExample\tFixed Value\tCompliance checking rules\tConvention Providence",
"myattr\td\te\t\tValid email\tc",
"aaaa\td\te\t\tSomething strange here\tc", "bbbb\td\te\tf",
"otherattr\td\te\t\tInteger\tc")))
sh = SpreadsheetHandler(str(s_dir))
output = tmpdir.mkdir("yaml")
sh.write_yaml(str(output))
global_attr_yaml = output.join("AMF_global_attrs.yml")
assert global_attr_yaml.check()
decoded = yaml.load(global_attr_yaml.read())
assert "suite_name" in decoded
assert "checks" in decoded
assert decoded["suite_name"] == "global_attrs_checks"
assert len(decoded["checks"]) == 2
assert decoded["checks"][0][
"check_id"] == "check_myattr_global_attribute"
expected_check_name = "checklib.register.nc_file_checks_register.GlobalAttrRegexCheck"
assert decoded["checks"][0]["check_name"] == expected_check_name
assert decoded["checks"][1]["check_name"] == expected_check_name
attr, email_regex = GlobalAttrCheck.parse_row({
"Name":
"myattr",
"Description":
"d",
"Example":
"e",
"Fixed Value":
"",
"Compliance checking rules":
"Valid email",
"Convention Providence":
""
})
assert decoded["checks"][0]["parameters"]["regex"] == email_regex
def test_file_info_yaml_check(self, spreadsheets_dir, tmpdir):
sh = SpreadsheetHandler(str(spreadsheets_dir))
output = tmpdir.mkdir("yaml")
sh.write_yaml(str(output))
file_info_check = output.join("AMF_file_info.yml")
assert file_info_check.check()
assert yaml.load(file_info_check.read()) == {
"suite_name":
"file_info_checks",
"description":
"Check 'file_info' in AMF files",
"checks": [{
"check_id": "check_soft_file_size_limit",
"check_name":
"checklib.register.file_checks_register.FileSizeCheck",
"check_level": "LOW",
"parameters": {
"threshold": 2,
"strictness": "soft"
}
}, {
"check_id": "check_hard_file_size_limit",
"check_name":
"checklib.register.file_checks_register.FileSizeCheck",
"check_level": "HIGH",
"parameters": {
"threshold": 4,
"strictness": "hard"
}
}, {
"check_id": "check_filename_structure",
"check_name":
"checklib.register.file_checks_register.FileNameStructureCheck",
"check_level": "HIGH",
"parameters": {
"delimiter": "_",
"extension": ".nc"
}
}]
}
def test_file_structure_yaml_check(self, spreadsheets_dir, tmpdir):
sh = SpreadsheetHandler(str(spreadsheets_dir))
output = tmpdir.mkdir("yaml")
sh.write_yaml(str(output))
file_structure_check = output.join("AMF_file_structure.yml")
assert file_structure_check.check()
assert yaml.load(file_structure_check.read()) == {
"suite_name":
"file_structure_checks",
"description":
"Check 'file_structure' in AMF files",
"checks": [{
"check_id": "check_valid_netcdf4_file",
"check_name":
"checklib.register.nc_file_checks_register.NetCDFFormatCheck",
"parameters": {
"format": "NETCDF4_CLASSIC"
}
}]
}
def test_common(self, spreadsheets_dir, tmpdir):
s_dir = spreadsheets_dir
common_dir = s_dir.join("Common.xlsx")
var_air = common_dir.join("Variables - Air.tsv")
var_sea = common_dir.join("Variables - Sea.tsv")
dim_land = common_dir.join("Dimensions - Land.tsv")
var_air.write("\n".join(
("Variable\tAttribute\tValue", "some_air_variable\t\t",
"\tthingy\tthis_thing", "\ttype\tfloat32")))
var_sea.write("\n".join(
("Variable\tAttribute\tValue", "some_sea_variable\t\t",
"\tthingy\tthat_thing", "\ttype\tstring")))
dim_land.write("\n".join(("Name\tLength\tunits", "some_dim\t42\tm")))
sh = SpreadsheetHandler(str(s_dir))
cv_output = tmpdir.mkdir("cvs")
yaml_output = tmpdir.mkdir("yaml")
sh.write_cvs(str(cv_output))
sh.write_yaml(str(yaml_output))
# Check CV and YAML files exist
var_air_output = cv_output.join("AMF_product_common_variable_air.json")
assert var_air_output.check()
assert cv_output.join("AMF_product_common_variable_sea.json").check()
assert cv_output.join("AMF_product_common_dimension_land.json").check()
assert yaml_output.join("AMF_product_common_variable_air.yml").check()
assert yaml_output.join("AMF_product_common_variable_sea.yml").check()
# Check the content of one of the CVs
assert json.load(var_air_output) == {
"product_common_variable_air": {
"some_air_variable": {
"thingy": "this_thing",
"type": "float32"
}
}
}
def test_top_level_product_yaml(self, spreadsheets_dir, tmpdir):
"""
Check that a top level YAML file is written for each product/deployment
mode combination, which includes other YAML files
"""
s_dir = spreadsheets_dir
# Write common variables and dimensions
common_dir = s_dir.join("Common.xlsx")
common_dir.join("Variables - Air.tsv").write("\n".join(
("Variable\tAttribute\tValue", "some_air_variable\t\t",
"\ttype\tfloat32")))
common_dir.join("Variables - Land.tsv").write("\n".join(
("Variable\tAttribute\tValue", "some_land_variable\t\t",
"\ttype\tfloat32")))
common_dir.join("Dimensions - Land.tsv").write("\n".join(
("Name\tLength\tunits", "index\t<i>\t1")))
# Write common global attributes
common_dir.join("Global Attributes.tsv").write("\n".join((
"Name\tDescription\tExample\tFixed Value\tCompliance checking rules\tConvention Providence",
"someattr\ta\tb\tc\tInteger\td")))
# Write product variables and dimensions
soil_dir = (s_dir.join("Product Definition Spreadsheets").mkdir(
"soil").mkdir("soil.xlsx"))
soil_dir.join("Variables - Specific.tsv").write("\n".join(
("Variable\tAttribute\tValue", "soil_var\t\t", "\ttype\tfloat32")))
soil_dir.join("Dimensions - Specific.tsv").write("\n".join(
("Name\tLength\tunits", "somedim\t<n>\tK")))
sh = SpreadsheetHandler(str(s_dir))
yaml_output = tmpdir.mkdir("yaml")
sh.write_yaml(str(yaml_output))
assert yaml_output.join("AMF_product_common_variable_air.yml").check()
assert yaml_output.join("AMF_product_common_variable_land.yml").check()
assert yaml_output.join(
"AMF_product_common_dimension_land.yml").check()
assert yaml_output.join("AMF_file_info.yml").check()
assert yaml_output.join("AMF_file_structure.yml").check()
assert yaml_output.join("AMF_global_attrs.yml").check()
top_level_air = yaml_output.join("AMF_product_soil_air.yml")
top_level_land = yaml_output.join("AMF_product_soil_land.yml")
assert top_level_air.check()
assert top_level_land.check()
assert yaml.load(top_level_air.read()) == {
"suite_name":
"product_soil_air_checks",
"description":
"Check 'product soil air' in AMF files",
"checks": [
# Global checks
{
"__INCLUDE__": "AMF_file_info.yml"
},
{
"__INCLUDE__": "AMF_file_structure.yml"
},
{
"__INCLUDE__": "AMF_global_attrs.yml"
},
# Common product checks
{
"__INCLUDE__": "AMF_product_common_variable_air.yml"
},
# Product specific
{
"__INCLUDE__": "AMF_product_soil_dimension.yml"
},
{
"__INCLUDE__": "AMF_product_soil_variable.yml"
}
]
}
# Land one should be basically the same as air but s/air/land/, and
# there is also common dimensions CV for land
assert yaml.load(top_level_land.read()) == {
"suite_name":
"product_soil_land_checks",
"description":
"Check 'product soil land' in AMF files",
"checks": [{
"__INCLUDE__": "AMF_file_info.yml"
}, {
"__INCLUDE__": "AMF_file_structure.yml"
}, {
"__INCLUDE__": "AMF_global_attrs.yml"
}, {
"__INCLUDE__": "AMF_product_common_dimension_land.yml"
}, {
"__INCLUDE__": "AMF_product_common_variable_land.yml"
}, {
"__INCLUDE__": "AMF_product_soil_dimension.yml"
}, {
"__INCLUDE__": "AMF_product_soil_variable.yml"
}]
}
def test_basic(self, spreadsheets_dir, tmpdir):
s_dir = spreadsheets_dir
var = (s_dir.join("Product Definition Spreadsheets").mkdir(
"my-great-product").mkdir("my-great-product.xlsx").join(
"Variables - Specific.tsv"))
var.write("\n".join(
("Variable\tAttribute\tValue", "wind_speed\t\t",
"\tname\twind_speed", "\ttype\tfloat32", "eastward_wind\t\t",
"\tname\teastward_wind", "\ttype\tfloat32", "\tunits\tm s-1")))
dim = (s_dir.join("Product Definition Spreadsheets").join(
"my-great-product").join("my-great-product.xlsx").join(
"Dimensions - Specific.tsv"))
dim.write("\n".join(
("Name\tLength\tunits", "one\t1\tm", "two\t2\tkm")))
output = tmpdir.mkdir("yaml")
sh = SpreadsheetHandler(str(s_dir))
sh.write_yaml(str(output))
var_output_yml = output.join(
"AMF_product_my-great-product_variable.yml")
dim_output_yml = output.join(
"AMF_product_my-great-product_dimension.yml")
assert var_output_yml.check()
assert dim_output_yml.check()
decoded = []
for f in (var_output_yml, dim_output_yml):
try:
decoded.append(yaml.load(f.read()))
except yaml.parser.ParserError:
assert False, "{} is invalid YAML".format(str(f))
assert decoded[0] == {
"suite_name":
"product_my-great-product_variable_checks",
"description":
"Check 'product my-great-product variable' in AMF files",
"checks": [
{
"check_id": "check_wind_speed_variable_attrs",
"check_name":
"checklib.register.nc_file_checks_register.NCVariableMetadataCheck",
"comments":
"Checks the variable attributes for 'wind_speed'",
"parameters": {
"pyessv_namespace":
"product_my-great-product_variable",
"var_id": "wind_speed",
"vocabulary_ref": "ncas:amf"
}
},
{
"check_id": "check_wind_speed_variable_type",
"check_name":
"checklib.register.nc_file_checks_register.VariableTypeCheck",
"comments": "Checks the type of variable 'wind_speed'",
"parameters": {
"var_id": "wind_speed",
"dtype": "float32",
"vocabulary_ref": "ncas:amf"
}
},
{
"check_id": "check_eastward_wind_variable_attrs",
"check_name":
"checklib.register.nc_file_checks_register.NCVariableMetadataCheck",
"comments":
"Checks the variable attributes for 'eastward_wind'",
"parameters": {
"pyessv_namespace":
"product_my-great-product_variable",
"var_id": "eastward_wind",
"vocabulary_ref": "ncas:amf"
}
},
{
"check_id": "check_eastward_wind_variable_type",
"check_name":
"checklib.register.nc_file_checks_register.VariableTypeCheck",
"comments": "Checks the type of variable 'eastward_wind'",
"parameters": {
"var_id": "eastward_wind",
"dtype": "float32",
"vocabulary_ref": "ncas:amf"
}
},
]
}
assert decoded[1] == {
"suite_name":
"product_my-great-product_dimension_checks",
"description":
"Check 'product my-great-product dimension' in AMF files",
"checks": [{
"check_id": "check_one_dimension_attrs",
"check_name":
"checklib.register.nc_file_checks_register.NetCDFDimensionCheck",
"comments": "Checks the dimension attributes for 'one'",
"parameters": {
"pyessv_namespace": "product_my-great-product_dimension",
"dim_id": "one",
"vocabulary_ref": "ncas:amf"
}
}, {
"check_id": "check_two_dimension_attrs",
"check_name":
"checklib.register.nc_file_checks_register.NetCDFDimensionCheck",
"comments": "Checks the dimension attributes for 'two'",
"parameters": {
"pyessv_namespace": "product_my-great-product_dimension",
"dim_id": "two",
"vocabulary_ref": "ncas:amf"
}
}]
}