def read_new_metadata(filename,
module_name,
table_name,
scheme_name=None,
subroutine_name=None):
"""Read metadata in new format and convert output to ccpp_prebuild metadata dictionary"""
if not os.path.isfile(filename):
raise Exception("New metadata file {0} not found".format(filename))
# Save metadata, because this routine new_metadata
# is called once for every table in that file
if filename in NEW_METADATA_SAVE.keys():
new_metadata_headers = NEW_METADATA_SAVE[filename]
else:
new_metadata_headers = MetadataHeader.parse_metadata_file(filename)
NEW_METADATA_SAVE[filename] = new_metadata_headers
# Convert new metadata for requested table to old metadata dictionary
metadata = collections.OrderedDict()
for new_metadata_header in new_metadata_headers:
if not scheme_name:
if not new_metadata_header.title == table_name:
# Skip this table, since it is not requested right now
continue
if new_metadata_header.title == module_name:
container = encode_container(module_name)
else:
container = encode_container(module_name,
new_metadata_header.title)
else:
if not new_metadata_header.title == table_name:
# Skip this table, since it is not requested right now
continue
container = encode_container(module_name, scheme_name, table_name)
for new_var in new_metadata_header.variable_list():
standard_name = new_var.get_prop_value('standard_name')
rank = len(new_var.get_prop_value('dimensions'))
var = Var(
standard_name=standard_name,
long_name=new_var.get_prop_value('long_name'),
units=new_var.get_prop_value('units'),
local_name=new_var.get_prop_value('local_name'),
type=new_var.get_prop_value('type'),
container=container,
kind=new_var.get_prop_value('kind'),
intent=new_var.get_prop_value('intent'),
optional='T' if new_var.get_prop_value('optional') else 'F',
)
# Set rank using integer-setter method
var.rank = rank
# Check for duplicates in same table
if standard_name in metadata.keys():
raise Exception(
"Error, multiple definitions of standard name {0} in new metadata table {1}"
.format(standard_name, table_name))
metadata[standard_name] = [var]
return metadata
def parse_preamble_data(statements, pobj, spec_name, endmatch, logger):
"""Parse module variables or DDT definitions from a module preamble
or parse program variables from the beginning of a program.
"""
inspec = True
mheaders = list()
var_dict = VarDictionary(spec_name)
psrc = ParseSource(spec_name, 'MODULE', pobj)
active_table = None
while inspec and (statements is not None):
while len(statements) > 0:
statement = statements.pop(0)
# End program or module
pmatch = endmatch.match(statement)
asmatch = arg_table_start_re.match(statement)
type_def = fortran_type_definition(statement)
if asmatch is not None:
active_table = asmatch.group(1).lower()
elif (pmatch is not None) or is_contains_statement(
statement, inspec):
# We are done with the specification
inspec = False
if len(var_dict.variable_list()) > 0:
mheader = MetadataHeader(title=spec_name,
type_in='MODULE',
module=spec_name,
var_dict=var_dict,
logger=logger)
mheaders.append(mheader)
# End if
break
elif (type_def is not None) and (type_def[0].lower()
== active_table):
statements, ddt = parse_type_def(statements, type_def,
spec_name, pobj, logger)
mheaders.append(ddt)
active_table = None
else:
# We should have a variable definition to add
if ((not is_comment_statement(statement, logger)) and
(not parse_use_statement({}, statement, pobj, logger))
and (active_table == spec_name)):
vars = parse_fortran_var_decl(statement,
psrc,
logger=logger)
for var in vars:
var_dict.add_variable(var)
# End for
# End if
# End if
# End while
if inspec and (len(statements) == 0):
statements = read_statements(pobj)
# End if
# End while
return statements, mheaders
def convert_to_html(filename_in, outdir, logger):
"""Convert a metadata file into html (one html file for each table)"""
if not os.path.isfile(filename_in):
raise Exception("Metadata file {} not found".format(filename_in))
logger.info("Converting file {} to HTML".format(filename_in))
metadata_headers = MetadataHeader.parse_metadata_file(filename_in)
for metadata_header in metadata_headers:
filename_out = metadata_header.to_html(outdir, ATTRIBUTES)
if filename_out:
logger.info(" ... wrote {}".format(filename_out))
def test_MetadataHeader_parse_table(tmpdir):
path = str(tmpdir.join("table.meta"))
with open(path, "w") as f:
f.write(example_table)
table1, table2 = MetadataHeader.parse_metadata_file(path)
# check first table
assert table1.name == "<name>"
assert table1.type == "scheme"
assert table1.dependencies == ["path/a.f", "path/b.f"]
# check second table
assert table2.name == "<name>"
assert table2.type == "scheme"
(im_data,) = table2.variable_list()
assert isinstance(im_data, Var)
assert im_data.get_dimensions() == []
def parse_type_def(statements, type_def, mod_name, pobj, logger):
psrc = ParseSource(mod_name, 'DDT', pobj)
seen_contains = False
mheader = None
var_dict = VarDictionary(type_def[0])
inspec = True
while inspec and (statements is not None):
while len(statements) > 0:
statement = statements.pop(0)
# End program or module
pmatch = end_type_re.match(statement)
if pmatch is not None:
# We hit the end of the type, make a header
mheader = MetadataHeader(title=type_def[0],
type_in='DDT',
module=mod_name,
var_dict=var_dict,
logger=logger)
inspec = False
elif is_contains_statement(statement, inspec):
seen_contains = True
elif not seen_contains:
# Comment of variable
if ((not is_comment_statement(statement, logger)) and
(not parse_use_statement({}, statement, pobj, logger))):
vars = parse_fortran_var_decl(statement,
psrc,
logger=logger)
for var in vars:
var_dict.add_variable(var)
# End for
# End if
else:
# We are just skipping lines until the end type
pass
# End if
# End while
if inspec and (len(statements) == 0):
statements = read_statements(pobj)
# End if
# End while
return statements, mheader
def read_new_metadata(filename,
module_name,
table_name,
scheme_name=None,
subroutine_name=None):
"""Read metadata in new format and convert output to ccpp_prebuild metadata dictionary"""
if not os.path.isfile(filename):
raise Exception("New metadata file {0} not found".format(filename))
# Save metadata, because this routine new_metadata
# is called once for every table in that file
if filename in NEW_METADATA_SAVE.keys():
new_metadata_headers = NEW_METADATA_SAVE[filename]
else:
new_metadata_headers = MetadataHeader.parse_metadata_file(filename)
NEW_METADATA_SAVE[filename] = new_metadata_headers
# Record dependencies for the metadata table (only applies to schemes)
has_property_table = False
dependencies = []
# Convert new metadata for requested table to old metadata dictionary
metadata = collections.OrderedDict()
for new_metadata_header in new_metadata_headers:
# Module or DDT tables
if not scheme_name:
# Module property tables
if new_metadata_header.property_table and new_metadata_header.title == module_name:
# If this is a ccpp-table-properties table for a module, it can only contain dependencies;
# ensure that for module tables, the header type is "module"
if not new_metadata_header.header_type == 'module':
raise Exception(
"Unsupported header_type '{}' for table properties for modules"
.format(new_metadata_header.header_type))
dependencies += new_metadata_header.dependencies
has_property_table = True
continue
# DDT property tables
elif new_metadata_header.property_table:
# If this is a ccpp-table-properties table for a DDT, it can only contain dependencies;
# ensure that for DDT tables, the header type is "ddt"
if not new_metadata_header.header_type == 'ddt':
raise Exception(
"Unsupported header_type '{}' for table properties for DDTs"
.format(new_metadata_header.header_type))
dependencies += new_metadata_header.dependencies
has_property_table = True
continue
# Module or DDT argument tables
else:
if not new_metadata_header.title == table_name:
# Skip this table, since it is not requested right now
continue
# Distinguish between module argument tables and DDT argument tables
if new_metadata_header.title == module_name:
container = encode_container(module_name)
else:
container = encode_container(module_name,
new_metadata_header.title)
else:
# Scheme property tables
if new_metadata_header.property_table and new_metadata_header.title == scheme_name:
# If this is a ccpp-table-properties table for a scheme, it can only contain dependencies;
# ensure that for scheme tables, the header type is "scheme"
if not new_metadata_header.header_type == 'scheme':
raise Exception(
"Unsupported header_type '{}' for table properties for schemes"
.format(new_metadata_header.header_type))
dependencies += new_metadata_header.dependencies
has_property_table = True
continue
# Scheme argument tables
else:
if not new_metadata_header.title == table_name:
# Skip this table, since it is not requested right now
continue
container = encode_container(module_name, scheme_name,
table_name)
for new_var in new_metadata_header.variable_list():
standard_name = new_var.get_prop_value('standard_name')
# DH* 2020-05-26
# Legacy extension for inconsistent metadata (use of horizontal_dimension versus horizontal_loop_extent).
# Since horizontal_dimension and horizontal_loop_extent have the same attributes (otherwise it doesn't
# make sense), we swap the standard name and add a note to the long name
legacy_note = ''
if standard_name == 'horizontal_loop_extent' and scheme_name and \
(table_name.endswith("_init") or table_name.endswith("_finalize")):
logging.warn("Legacy extension - replacing variable 'horizontal_loop_extent'" + \
" with 'horizontal_dimension' in table {}".format(table_name))
standard_name = 'horizontal_dimension'
legacy_note = ' replaced by horizontal dimension (legacy extension)'
elif standard_name == 'horizontal_dimension' and scheme_name and table_name.endswith(
"_run"):
logging.warn("Legacy extension - replacing variable 'horizontal_dimension' " + \
"with 'horizontal_loop_extent' in table {}".format(table_name))
standard_name = 'horizontal_loop_extent'
legacy_note = ' replaced by horizontal loop extent (legacy extension)'
# Adjust dimensions
dimensions = new_var.get_prop_value('dimensions')
if scheme_name and (table_name.endswith("_init") or table_name.endswith("_finalize")) \
and 'horizontal_loop_extent' in dimensions:
logging.warn("Legacy extension - replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \
"for variable {} in table {}".format(standard_name,table_name))
dimensions = [
'horizontal_dimension'
if x == 'horizontal_loop_extent' else x for x in dimensions
]
elif scheme_name and table_name.endswith(
"_run") and 'horizontal_dimension' in dimensions:
logging.warn("Legacy extension - replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \
"for variable {} in table {}".format(standard_name,table_name))
dimensions = [
'horizontal_loop_extent'
if x == 'horizontal_dimension' else x for x in dimensions
]
# *DH 2020-05-26
if new_var.get_prop_value('active').lower() == '.true.':
active = 'T'
elif new_var.get_prop_value('active').lower() == '.false.':
active = 'F'
else:
# Replace multiple whitespaces, preserve case
active = ' '.join(new_var.get_prop_value('active').split())
var = Var(
standard_name=standard_name,
long_name=new_var.get_prop_value('long_name') + legacy_note,
units=new_var.get_prop_value('units'),
local_name=new_var.get_prop_value('local_name'),
type=new_var.get_prop_value('type'),
dimensions=dimensions,
container=container,
kind=new_var.get_prop_value('kind'),
intent=new_var.get_prop_value('intent'),
optional='T' if new_var.get_prop_value('optional') else 'F',
active=active,
)
# Check for duplicates in same table
if standard_name in metadata.keys():
raise Exception(
"Error, multiple definitions of standard name {} in new metadata table {}"
.format(standard_name, table_name))
metadata[standard_name] = [var]
# CCPP property tables are mandatory
if not has_property_table:
if scheme_name:
raise Exception("Metadata file {} for scheme {} does not have a [ccpp-table-properties] section,".format(filename, scheme_name) + \
" or the 'name = ...' attribute in the [ccpp-table-properties] is wrong")
else:
raise Exception("Metadata file {} for table {} does not have a [ccpp-table-properties] section,".format(filename, table_name) + \
" or the 'name = ...' attribute in the [ccpp-table-properties] is wrong")
return (metadata, dependencies)
def parse_metadata_tables_typedefs(model):
# Lookup table local_name -> dimensions
dimensions = {
'ccpp_error_flag': [],
'ccpp_error_message': [],
'ccpp_loop_counter': [],
'ccpp_block_number': [],
'ccpp_thread_number': [],
'ccpp_t': [],
}
for filename in METADATA_TYPEDEFS[model]:
metadata_headers = MetadataHeader.parse_metadata_file(filename)
for metadata_header in metadata_headers:
for var in metadata_header.variable_list():
standard_name = var.get_prop_value('standard_name')
if standard_name in dimensions.keys():
raise ValueError(
"Duplicate standard name {} in type/variable definition metadata tables"
.format(standard_name))
dimensions[standard_name] = var.get_prop_value('dimensions')
#
# Add missing variables (not used by FV3)
dimensions['lw_heating_rate_spectral'] = [
'horizontal_dimension',
'adjusted_vertical_layer_dimension_for_radiation',
'number_of_aerosol_bands_for_longwave_radiation'
]
dimensions['lw_fluxes'] = [
'horizontal_dimension',
'adjusted_vertical_level_dimension_for_radiation'
]
dimensions['cloud_optical_depth'] = [
'horizontal_dimension',
'adjusted_vertical_layer_dimension_for_radiation'
]
#
dimensions['sw_heating_rate_spectral'] = [
'horizontal_dimension',
'adjusted_vertical_layer_dimension_for_radiation',
'number_of_aerosol_bands_for_shortwave_radiation'
]
dimensions['sw_fluxes'] = [
'horizontal_dimension',
'adjusted_vertical_level_dimension_for_radiation'
]
dimensions['cloud_single_scattering_albedo'] = [
'horizontal_dimension',
'adjusted_vertical_layer_dimension_for_radiation'
]
dimensions['cloud_asymmetry_parameter'] = [
'horizontal_dimension',
'adjusted_vertical_layer_dimension_for_radiation'
]
#
dimensions['specified_kinematic_surface_upward_sensible_heat_flux'] = [
'horizontal_dimension'
]
dimensions['specified_kinematic_surface_upward_latent_heat_flux'] = [
'horizontal_dimension'
]
dimensions['vonKarman_constant'] = []
#
return dimensions
def parse_scheme_metadata(statements, pobj, spec_name, table_name, logger):
"Parse dummy argument information from a subroutine"
psrc = None
mheader = None
var_dict = None
scheme_name = None
# Find the subroutine line, should be first executable statement
inpreamble = False
insub = True
while insub and (statements is not None):
while len(statements) > 0:
statement = statements.pop(0)
smatch = subroutine_re.match(statement)
esmatch = end_subroutine_re.match(statement)
pmatch = endmodule_re.match(statement)
asmatch = arg_table_start_re.match(statement)
if asmatch is not None:
# We have run off the end of something, hope that is okay
# Put this statement back for the caller to deal with
statements.insert(0, statement)
insub = False
break
elif (pmatch is not None):
# We have run off the end of the module, hope that is okay
pobj.leave_region('MODULE', region_name=spec_name)
insub = False
break
elif smatch is not None:
scheme_name = smatch.group(1)
inpreamble = scheme_name == table_name
if inpreamble:
if smatch.group(2) is not None:
scheme_args = [
x.strip().lower()
for x in smatch.group(2).split(',')
]
else:
scheme_args = list()
# End if
scheme_set = set(scheme_args)
var_dict = VarDictionary(scheme_name)
psrc = ParseSource(scheme_name, 'SCHEME', pobj)
# End if
elif inpreamble:
# Process a preamble statement (use or argument declaration)
if esmatch is not None:
inpreamble = False
insub = False
elif ((not is_comment_statement(statement, logger)) and
(not parse_use_statement({}, statement, pobj, logger))
and ('intent' in statement.lower())):
vars = parse_fortran_var_decl(statement,
psrc,
logger=logger)
for var in vars:
lname = var.get_prop_value('local_name').lower()
if lname in scheme_set:
scheme_set.remove(lname)
else:
raise ParseSyntaxError('dummy argument',
token=lname,
context=pobj)
# End if
var_dict.add_variable(var)
# End for
# End if
# End if
# End while
if insub and (len(statements) == 0):
statements = read_statements(pobj)
# End if
# End while
if (scheme_name is not None) and (var_dict is not None):
mheader = MetadataHeader(title=scheme_name,
type_in='SCHEME',
module=spec_name,
var_dict=var_dict,
logger=logger)
# End if
return statements, mheader
def read_new_metadata(filename,
module_name,
table_name,
scheme_name=None,
subroutine_name=None):
"""Read metadata in new format and convert output to ccpp_prebuild metadata dictionary"""
if not os.path.isfile(filename):
raise Exception("New metadata file {0} not found".format(filename))
# Save metadata, because this routine new_metadata
# is called once for every table in that file
if filename in NEW_METADATA_SAVE.keys():
new_metadata_headers = NEW_METADATA_SAVE[filename]
else:
new_metadata_headers = MetadataHeader.parse_metadata_file(filename)
NEW_METADATA_SAVE[filename] = new_metadata_headers
# Convert new metadata for requested table to old metadata dictionary
metadata = collections.OrderedDict()
for new_metadata_header in new_metadata_headers:
if not scheme_name:
if not new_metadata_header.title == table_name:
# Skip this table, since it is not requested right now
continue
if new_metadata_header.title == module_name:
container = encode_container(module_name)
else:
container = encode_container(module_name,
new_metadata_header.title)
else:
if not new_metadata_header.title == table_name:
# Skip this table, since it is not requested right now
continue
container = encode_container(module_name, scheme_name, table_name)
for new_var in new_metadata_header.variable_list():
standard_name = new_var.get_prop_value('standard_name')
# DH* 2020-05-26
# Legacy extension for inconsistent metadata (use of horizontal_dimension versus horizontal_loop_extent).
# Since horizontal_dimension and horizontal_loop_extent have the same attributes (otherwise it doesn't
# make sense), we swap the standard name and add a note to the long name
legacy_note = ''
if standard_name == 'horizontal_loop_extent' and scheme_name and \
(table_name.endswith("_init") or table_name.endswith("finalize")):
logging.warn("Legacy extension - replacing variable 'horizontal_loop_extent'" + \
" with 'horizontal_dimension' in table {}".format(table_name))
standard_name = 'horizontal_dimension'
legacy_note = ' replaced by horizontal dimension (legacy extension)'
elif standard_name == 'horizontal_dimension' and scheme_name and table_name.endswith(
"_run"):
logging.warn("Legacy extension - replacing variable 'horizontal_dimension' " + \
"with 'horizontal_loop_extent' in table {}".format(table_name))
standard_name = 'horizontal_loop_extent'
legacy_note = ' replaced by horizontal loop extent (legacy extension)'
# Adjust dimensions
dimensions = new_var.get_prop_value('dimensions')
if scheme_name and (table_name.endswith("_init") or table_name.endswith("finalize")) \
and 'horizontal_loop_extent' in dimensions:
logging.warn("Legacy extension - replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \
"for variable {} in table {}".format(standard_name,table_name))
dimensions = [
'horizontal_dimension'
if x == 'horizontal_loop_extent' else x for x in dimensions
]
elif scheme_name and table_name.endswith(
"_run") and 'horizontal_dimension' in dimensions:
logging.warn("Legacy extension - replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \
"for variable {} in table {}".format(standard_name,table_name))
dimensions = [
'horizontal_loop_extent'
if x == 'horizontal_dimension' else x for x in dimensions
]
# *DH 2020-05-26
if new_var.get_prop_value('active').lower() == '.true.':
active = 'T'
elif new_var.get_prop_value('active').lower() == '.false.':
active = 'F'
else:
# Replace multiple whitespaces and use lowercase throughout
active = ' '.join(
new_var.get_prop_value('active').lower().split())
var = Var(
standard_name=standard_name,
long_name=new_var.get_prop_value('long_name') + legacy_note,
units=new_var.get_prop_value('units'),
local_name=new_var.get_prop_value('local_name'),
type=new_var.get_prop_value('type'),
dimensions=dimensions,
container=container,
kind=new_var.get_prop_value('kind'),
intent=new_var.get_prop_value('intent'),
optional='T' if new_var.get_prop_value('optional') else 'F',
active=active,
)
# Check for duplicates in same table
if standard_name in metadata.keys():
raise Exception(
"Error, multiple definitions of standard name {0} in new metadata table {1}"
.format(standard_name, table_name))
metadata[standard_name] = [var]
return metadata
print("Found {} files with arg tables".format(len(tfilenames)))
total_headers = 0
for tfile in tfilenames:
try:
tbase = os.path.basename(tfile)
file = os.path.join(pdir, tbase)
mbase = "{}.meta".format('.'.join(tbase.split('.')[:-1]))
mdfile = os.path.join(pdir, mbase)
if not os.path.exists(file):
infile = tfile
if not os.path.exists(infile):
print("WARNING: Cannot find '{}'".format(infile))
else:
convert_metadata.convert_file(infile, file, mdfile, logger)
# End if
# End if
if os.path.exists(mdfile):
mh = MetadataHeader.parse_metadata_file(mdfile)
print("{} metadata headers parsed in {}".format(
len(mh), mdfile))
total_headers = total_headers + len(mh)
else:
print("{} not found!".format(mdfile))
# End if
except ValueError as ve:
print("{}: {}".format(infile, ve))
# End except
# End for
print("Found {} total metadata headers".format(total_headers))
# End if __main__