def read_new_metadata(filename,
module_name,
table_name,
scheme_name=None,
subroutine_name=None):
"""Read metadata in new format and convert output to ccpp_prebuild metadata dictionary"""
if not os.path.isfile(filename):
raise Exception("New metadata file {0} not found".format(filename))
# Save metadata, because this routine new_metadata
# is called once for every table in that file
if filename in NEW_METADATA_SAVE.keys():
new_metadata_headers = NEW_METADATA_SAVE[filename]
else:
new_metadata_headers = MetadataHeader.parse_metadata_file(filename)
NEW_METADATA_SAVE[filename] = new_metadata_headers
# Convert new metadata for requested table to old metadata dictionary
metadata = collections.OrderedDict()
for new_metadata_header in new_metadata_headers:
if not scheme_name:
if not new_metadata_header.title == table_name:
# Skip this table, since it is not requested right now
continue
if new_metadata_header.title == module_name:
container = encode_container(module_name)
else:
container = encode_container(module_name,
new_metadata_header.title)
else:
if not new_metadata_header.title == table_name:
# Skip this table, since it is not requested right now
continue
container = encode_container(module_name, scheme_name, table_name)
for new_var in new_metadata_header.variable_list():
standard_name = new_var.get_prop_value('standard_name')
rank = len(new_var.get_prop_value('dimensions'))
var = Var(
standard_name=standard_name,
long_name=new_var.get_prop_value('long_name'),
units=new_var.get_prop_value('units'),
local_name=new_var.get_prop_value('local_name'),
type=new_var.get_prop_value('type'),
container=container,
kind=new_var.get_prop_value('kind'),
intent=new_var.get_prop_value('intent'),
optional='T' if new_var.get_prop_value('optional') else 'F',
)
# Set rank using integer-setter method
var.rank = rank
# Check for duplicates in same table
if standard_name in metadata.keys():
raise Exception(
"Error, multiple definitions of standard name {0} in new metadata table {1}"
.format(standard_name, table_name))
metadata[standard_name] = [var]
return metadata
def convert_to_html(filename_in, outdir, logger):
"""Convert a metadata file into html (one html file for each table)"""
if not os.path.isfile(filename_in):
raise Exception("Metadata file {} not found".format(filename_in))
logger.info("Converting file {} to HTML".format(filename_in))
metadata_headers = MetadataHeader.parse_metadata_file(filename_in)
for metadata_header in metadata_headers:
filename_out = metadata_header.to_html(outdir, ATTRIBUTES)
if filename_out:
logger.info(" ... wrote {}".format(filename_out))
def test_MetadataHeader_parse_table(tmpdir):
path = str(tmpdir.join("table.meta"))
with open(path, "w") as f:
f.write(example_table)
table1, table2 = MetadataHeader.parse_metadata_file(path)
# check first table
assert table1.name == "<name>"
assert table1.type == "scheme"
assert table1.dependencies == ["path/a.f", "path/b.f"]
# check second table
assert table2.name == "<name>"
assert table2.type == "scheme"
(im_data,) = table2.variable_list()
assert isinstance(im_data, Var)
assert im_data.get_dimensions() == []
def read_new_metadata(filename,
module_name,
table_name,
scheme_name=None,
subroutine_name=None):
"""Read metadata in new format and convert output to ccpp_prebuild metadata dictionary"""
if not os.path.isfile(filename):
raise Exception("New metadata file {0} not found".format(filename))
# Save metadata, because this routine new_metadata
# is called once for every table in that file
if filename in NEW_METADATA_SAVE.keys():
new_metadata_headers = NEW_METADATA_SAVE[filename]
else:
new_metadata_headers = MetadataHeader.parse_metadata_file(filename)
NEW_METADATA_SAVE[filename] = new_metadata_headers
# Record dependencies for the metadata table (only applies to schemes)
has_property_table = False
dependencies = []
# Convert new metadata for requested table to old metadata dictionary
metadata = collections.OrderedDict()
for new_metadata_header in new_metadata_headers:
# Module or DDT tables
if not scheme_name:
# Module property tables
if new_metadata_header.property_table and new_metadata_header.title == module_name:
# If this is a ccpp-table-properties table for a module, it can only contain dependencies;
# ensure that for module tables, the header type is "module"
if not new_metadata_header.header_type == 'module':
raise Exception(
"Unsupported header_type '{}' for table properties for modules"
.format(new_metadata_header.header_type))
dependencies += new_metadata_header.dependencies
has_property_table = True
continue
# DDT property tables
elif new_metadata_header.property_table:
# If this is a ccpp-table-properties table for a DDT, it can only contain dependencies;
# ensure that for DDT tables, the header type is "ddt"
if not new_metadata_header.header_type == 'ddt':
raise Exception(
"Unsupported header_type '{}' for table properties for DDTs"
.format(new_metadata_header.header_type))
dependencies += new_metadata_header.dependencies
has_property_table = True
continue
# Module or DDT argument tables
else:
if not new_metadata_header.title == table_name:
# Skip this table, since it is not requested right now
continue
# Distinguish between module argument tables and DDT argument tables
if new_metadata_header.title == module_name:
container = encode_container(module_name)
else:
container = encode_container(module_name,
new_metadata_header.title)
else:
# Scheme property tables
if new_metadata_header.property_table and new_metadata_header.title == scheme_name:
# If this is a ccpp-table-properties table for a scheme, it can only contain dependencies;
# ensure that for scheme tables, the header type is "scheme"
if not new_metadata_header.header_type == 'scheme':
raise Exception(
"Unsupported header_type '{}' for table properties for schemes"
.format(new_metadata_header.header_type))
dependencies += new_metadata_header.dependencies
has_property_table = True
continue
# Scheme argument tables
else:
if not new_metadata_header.title == table_name:
# Skip this table, since it is not requested right now
continue
container = encode_container(module_name, scheme_name,
table_name)
for new_var in new_metadata_header.variable_list():
standard_name = new_var.get_prop_value('standard_name')
# DH* 2020-05-26
# Legacy extension for inconsistent metadata (use of horizontal_dimension versus horizontal_loop_extent).
# Since horizontal_dimension and horizontal_loop_extent have the same attributes (otherwise it doesn't
# make sense), we swap the standard name and add a note to the long name
legacy_note = ''
if standard_name == 'horizontal_loop_extent' and scheme_name and \
(table_name.endswith("_init") or table_name.endswith("_finalize")):
logging.warn("Legacy extension - replacing variable 'horizontal_loop_extent'" + \
" with 'horizontal_dimension' in table {}".format(table_name))
standard_name = 'horizontal_dimension'
legacy_note = ' replaced by horizontal dimension (legacy extension)'
elif standard_name == 'horizontal_dimension' and scheme_name and table_name.endswith(
"_run"):
logging.warn("Legacy extension - replacing variable 'horizontal_dimension' " + \
"with 'horizontal_loop_extent' in table {}".format(table_name))
standard_name = 'horizontal_loop_extent'
legacy_note = ' replaced by horizontal loop extent (legacy extension)'
# Adjust dimensions
dimensions = new_var.get_prop_value('dimensions')
if scheme_name and (table_name.endswith("_init") or table_name.endswith("_finalize")) \
and 'horizontal_loop_extent' in dimensions:
logging.warn("Legacy extension - replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \
"for variable {} in table {}".format(standard_name,table_name))
dimensions = [
'horizontal_dimension'
if x == 'horizontal_loop_extent' else x for x in dimensions
]
elif scheme_name and table_name.endswith(
"_run") and 'horizontal_dimension' in dimensions:
logging.warn("Legacy extension - replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \
"for variable {} in table {}".format(standard_name,table_name))
dimensions = [
'horizontal_loop_extent'
if x == 'horizontal_dimension' else x for x in dimensions
]
# *DH 2020-05-26
if new_var.get_prop_value('active').lower() == '.true.':
active = 'T'
elif new_var.get_prop_value('active').lower() == '.false.':
active = 'F'
else:
# Replace multiple whitespaces, preserve case
active = ' '.join(new_var.get_prop_value('active').split())
var = Var(
standard_name=standard_name,
long_name=new_var.get_prop_value('long_name') + legacy_note,
units=new_var.get_prop_value('units'),
local_name=new_var.get_prop_value('local_name'),
type=new_var.get_prop_value('type'),
dimensions=dimensions,
container=container,
kind=new_var.get_prop_value('kind'),
intent=new_var.get_prop_value('intent'),
optional='T' if new_var.get_prop_value('optional') else 'F',
active=active,
)
# Check for duplicates in same table
if standard_name in metadata.keys():
raise Exception(
"Error, multiple definitions of standard name {} in new metadata table {}"
.format(standard_name, table_name))
metadata[standard_name] = [var]
# CCPP property tables are mandatory
if not has_property_table:
if scheme_name:
raise Exception("Metadata file {} for scheme {} does not have a [ccpp-table-properties] section,".format(filename, scheme_name) + \
" or the 'name = ...' attribute in the [ccpp-table-properties] is wrong")
else:
raise Exception("Metadata file {} for table {} does not have a [ccpp-table-properties] section,".format(filename, table_name) + \
" or the 'name = ...' attribute in the [ccpp-table-properties] is wrong")
return (metadata, dependencies)
def parse_metadata_tables_typedefs(model):
# Lookup table local_name -> dimensions
dimensions = {
'ccpp_error_flag': [],
'ccpp_error_message': [],
'ccpp_loop_counter': [],
'ccpp_block_number': [],
'ccpp_thread_number': [],
'ccpp_t': [],
}
for filename in METADATA_TYPEDEFS[model]:
metadata_headers = MetadataHeader.parse_metadata_file(filename)
for metadata_header in metadata_headers:
for var in metadata_header.variable_list():
standard_name = var.get_prop_value('standard_name')
if standard_name in dimensions.keys():
raise ValueError(
"Duplicate standard name {} in type/variable definition metadata tables"
.format(standard_name))
dimensions[standard_name] = var.get_prop_value('dimensions')
#
# Add missing variables (not used by FV3)
dimensions['lw_heating_rate_spectral'] = [
'horizontal_dimension',
'adjusted_vertical_layer_dimension_for_radiation',
'number_of_aerosol_bands_for_longwave_radiation'
]
dimensions['lw_fluxes'] = [
'horizontal_dimension',
'adjusted_vertical_level_dimension_for_radiation'
]
dimensions['cloud_optical_depth'] = [
'horizontal_dimension',
'adjusted_vertical_layer_dimension_for_radiation'
]
#
dimensions['sw_heating_rate_spectral'] = [
'horizontal_dimension',
'adjusted_vertical_layer_dimension_for_radiation',
'number_of_aerosol_bands_for_shortwave_radiation'
]
dimensions['sw_fluxes'] = [
'horizontal_dimension',
'adjusted_vertical_level_dimension_for_radiation'
]
dimensions['cloud_single_scattering_albedo'] = [
'horizontal_dimension',
'adjusted_vertical_layer_dimension_for_radiation'
]
dimensions['cloud_asymmetry_parameter'] = [
'horizontal_dimension',
'adjusted_vertical_layer_dimension_for_radiation'
]
#
dimensions['specified_kinematic_surface_upward_sensible_heat_flux'] = [
'horizontal_dimension'
]
dimensions['specified_kinematic_surface_upward_latent_heat_flux'] = [
'horizontal_dimension'
]
dimensions['vonKarman_constant'] = []
#
return dimensions
def read_new_metadata(filename,
module_name,
table_name,
scheme_name=None,
subroutine_name=None):
"""Read metadata in new format and convert output to ccpp_prebuild metadata dictionary"""
if not os.path.isfile(filename):
raise Exception("New metadata file {0} not found".format(filename))
# Save metadata, because this routine new_metadata
# is called once for every table in that file
if filename in NEW_METADATA_SAVE.keys():
new_metadata_headers = NEW_METADATA_SAVE[filename]
else:
new_metadata_headers = MetadataHeader.parse_metadata_file(filename)
NEW_METADATA_SAVE[filename] = new_metadata_headers
# Convert new metadata for requested table to old metadata dictionary
metadata = collections.OrderedDict()
for new_metadata_header in new_metadata_headers:
if not scheme_name:
if not new_metadata_header.title == table_name:
# Skip this table, since it is not requested right now
continue
if new_metadata_header.title == module_name:
container = encode_container(module_name)
else:
container = encode_container(module_name,
new_metadata_header.title)
else:
if not new_metadata_header.title == table_name:
# Skip this table, since it is not requested right now
continue
container = encode_container(module_name, scheme_name, table_name)
for new_var in new_metadata_header.variable_list():
standard_name = new_var.get_prop_value('standard_name')
# DH* 2020-05-26
# Legacy extension for inconsistent metadata (use of horizontal_dimension versus horizontal_loop_extent).
# Since horizontal_dimension and horizontal_loop_extent have the same attributes (otherwise it doesn't
# make sense), we swap the standard name and add a note to the long name
legacy_note = ''
if standard_name == 'horizontal_loop_extent' and scheme_name and \
(table_name.endswith("_init") or table_name.endswith("finalize")):
logging.warn("Legacy extension - replacing variable 'horizontal_loop_extent'" + \
" with 'horizontal_dimension' in table {}".format(table_name))
standard_name = 'horizontal_dimension'
legacy_note = ' replaced by horizontal dimension (legacy extension)'
elif standard_name == 'horizontal_dimension' and scheme_name and table_name.endswith(
"_run"):
logging.warn("Legacy extension - replacing variable 'horizontal_dimension' " + \
"with 'horizontal_loop_extent' in table {}".format(table_name))
standard_name = 'horizontal_loop_extent'
legacy_note = ' replaced by horizontal loop extent (legacy extension)'
# Adjust dimensions
dimensions = new_var.get_prop_value('dimensions')
if scheme_name and (table_name.endswith("_init") or table_name.endswith("finalize")) \
and 'horizontal_loop_extent' in dimensions:
logging.warn("Legacy extension - replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \
"for variable {} in table {}".format(standard_name,table_name))
dimensions = [
'horizontal_dimension'
if x == 'horizontal_loop_extent' else x for x in dimensions
]
elif scheme_name and table_name.endswith(
"_run") and 'horizontal_dimension' in dimensions:
logging.warn("Legacy extension - replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \
"for variable {} in table {}".format(standard_name,table_name))
dimensions = [
'horizontal_loop_extent'
if x == 'horizontal_dimension' else x for x in dimensions
]
# *DH 2020-05-26
if new_var.get_prop_value('active').lower() == '.true.':
active = 'T'
elif new_var.get_prop_value('active').lower() == '.false.':
active = 'F'
else:
# Replace multiple whitespaces and use lowercase throughout
active = ' '.join(
new_var.get_prop_value('active').lower().split())
var = Var(
standard_name=standard_name,
long_name=new_var.get_prop_value('long_name') + legacy_note,
units=new_var.get_prop_value('units'),
local_name=new_var.get_prop_value('local_name'),
type=new_var.get_prop_value('type'),
dimensions=dimensions,
container=container,
kind=new_var.get_prop_value('kind'),
intent=new_var.get_prop_value('intent'),
optional='T' if new_var.get_prop_value('optional') else 'F',
active=active,
)
# Check for duplicates in same table
if standard_name in metadata.keys():
raise Exception(
"Error, multiple definitions of standard name {0} in new metadata table {1}"
.format(standard_name, table_name))
metadata[standard_name] = [var]
return metadata
print("Found {} files with arg tables".format(len(tfilenames)))
total_headers = 0
for tfile in tfilenames:
try:
tbase = os.path.basename(tfile)
file = os.path.join(pdir, tbase)
mbase = "{}.meta".format('.'.join(tbase.split('.')[:-1]))
mdfile = os.path.join(pdir, mbase)
if not os.path.exists(file):
infile = tfile
if not os.path.exists(infile):
print("WARNING: Cannot find '{}'".format(infile))
else:
convert_metadata.convert_file(infile, file, mdfile, logger)
# End if
# End if
if os.path.exists(mdfile):
mh = MetadataHeader.parse_metadata_file(mdfile)
print("{} metadata headers parsed in {}".format(
len(mh), mdfile))
total_headers = total_headers + len(mh)
else:
print("{} not found!".format(mdfile))
# End if
except ValueError as ve:
print("{}: {}".format(infile, ve))
# End except
# End for
print("Found {} total metadata headers".format(total_headers))
# End if __main__