def test_dependencies_rel_path(self):
"""Test that relative_path and dependencies from ccpp-table-properties are read in correctly"""
known_ddts = list()
logger = None
filename = os.path.join(SAMPLE_FILES_DIR,
"test_dependencies_rel_path.meta")
result = parse_metadata_file(filename, known_ddts, logger)
dependencies = result[0].dependencies
rel_path = result[0].relative_path
titles = [elem.table_name for elem in result]
self.assertEqual(len(dependencies), 4)
self.assertIn('machine.F',
dependencies,
msg="Dependency 'machine.F' is expected but not found")
self.assertIn(
'physcons.F90',
dependencies,
msg="Dependency 'physcons.F90' is expected but not found")
self.assertIn(
'GFDL_parse_tracers.F90',
dependencies,
msg="Dependency 'GFDL_parse_tracers.F90' is expected but not found"
)
self.assertIn('rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90', dependencies, \
msg="Header name 'rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90' is expected but not found")
self.assertIn(rel_path, "../../ccpp/physics/physics")
self.assertEqual(len(result), 1)
self.assertIn('test_host',
titles,
msg="Table name 'test_host' is expected but not found")
def test_MetadataTable_parse_table(tmpdir):
path = str(tmpdir.join("table.meta"))
with open(path, "w") as f:
f.write(example_table)
dummy_run_env = CCPPFrameworkEnv(None,
ndict={
'host_files': '',
'scheme_files': '',
'suites': ''
})
metadata_headers = parse_metadata_file(
path, known_ddts=registered_fortran_ddt_names(), run_env=dummy_run_env)
# check metadata header
assert len(metadata_headers) == 1
metadata_header = metadata_headers[0]
assert metadata_header.table_name == "<name>"
assert metadata_header.table_type == "scheme"
assert metadata_header.relative_path == "path"
assert metadata_header.dependencies == ["a.f", "b.f"]
# check metadata section
assert len(metadata_header.sections()) == 1
metadata_section = metadata_header.sections()[0]
assert metadata_section.name == "<name>"
assert metadata_section.type == "scheme"
(im_data, ) = metadata_section.variable_list()
assert isinstance(im_data, Var)
assert im_data.get_dimensions() == []
def test_MetadataTable_parse_table(tmpdir):
path = str(tmpdir.join("table.meta"))
with open(path, "w") as f:
f.write(example_table)
metadata_headers = parse_metadata_file(
path,
known_ddts=registered_fortran_ddt_names(),
logger=logging.getLogger(__name__))
# check metadata header
assert len(metadata_headers) == 1
metadata_header = metadata_headers[0]
assert metadata_header.table_name == "<name>"
assert metadata_header.table_type == "scheme"
assert metadata_header.relative_path == "path"
assert metadata_header.dependencies == ["a.f", "b.f"]
# check metadata section
assert len(metadata_header.sections()) == 1
metadata_section = metadata_header.sections()[0]
assert metadata_section.name == "<name>"
assert metadata_section.type == "scheme"
(im_data, ) = metadata_section.variable_list()
assert isinstance(im_data, Var)
assert im_data.get_dimensions() == []
def test_good_multi_ccpp_arg_table(self):
"""Test that good file with 4 ccpp-arg-table returns 4 headers"""
known_ddts = list()
logger = None
filename = os.path.join(SAMPLE_FILES_DIR,
"test_multi_ccpp_arg_tables.meta")
#Exercise
result = parse_metadata_file(filename, known_ddts, logger)
#Verify that size of returned list equals number of ccpp-table-properties in the test file
# ccpp-arg-tables are returned in result[0].sections() and result[1].sections()
self.assertEqual(len(result), 2)
titles = list()
for table in result:
titles.extend([x.title for x in table.sections()])
self.assertIn('vmr_type',
titles,
msg="Header name 'vmr_type' is expected but not found")
self.assertIn(
'make_ddt_run',
titles,
msg="Header name 'make_ddt_run' is expected but not found")
self.assertIn(
'make_ddt_init',
titles,
msg="Header name 'make_ddt_init' is expected but not found")
self.assertIn(
'make_ddt_finalize',
titles,
msg="Header name 'make_ddt_finalize' is expected but not found")
def parse_host_model_files(host_filenames, preproc_defs, host_name, logger):
###############################################################################
"""
Gather information from host files (e.g., DDTs, registry) and
return a host model object with the information.
"""
header_dict = {}
table_dict = {}
known_ddts = list()
for filename in host_filenames:
logger.info('Reading host model data from {}'.format(filename))
# parse metadata file
mtables = parse_metadata_file(filename, known_ddts, logger)
fort_file = find_associated_fortran_file(filename)
ftables = parse_fortran_file(fort_file,
preproc_defs=preproc_defs,
logger=logger)
# Check Fortran against metadata (will raise an exception on error)
mheaders = list()
for sect in [x.sections() for x in mtables]:
mheaders.extend(sect)
# end for
fheaders = list()
for sect in [x.sections() for x in ftables]:
fheaders.extend(sect)
# end for
check_fortran_against_metadata(mheaders, fheaders, filename, fort_file,
logger)
# Check for duplicate tables, then add to dict
for table in mtables:
if table.table_name in table_dict:
duplicate_item_error(table.table_name, filename,
table.table_type,
table_dict[header.title])
else:
table_dict[table.table_name] = table
# end if
# end for
# Check for duplicate headers, then add to dict
for header in mheaders:
if header.title in header_dict:
duplicate_item_error(header.title, filename,
header.header_type,
header_dict[header.title])
else:
header_dict[header.title] = header
if header.header_type == 'ddt':
known_ddts.append(header.title)
# end if
# end for
# end for
if not host_name:
host_name = None
# end if
host_model = HostModel(table_dict, host_name, logger)
return host_model
def test_invalid_intent(self):
"""Test that an invalid intent returns expected error"""
known_ddts = list()
filename = os.path.join(SAMPLE_FILES_DIR, "test_invalid_intent.meta")
with self.assertRaises(Exception) as context:
_ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV)
#print("The exception is", context.exception)
self.assertTrue('Invalid \'intent\' property value, \'banana\', at ' in
str(context.exception))
def test_double_header(self):
"""Test that a duplicate header returns expected error"""
known_ddts = list()
filename = os.path.join(SAMPLE_FILES_DIR, "double_header.meta")
with self.assertRaises(Exception) as context:
_ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV)
#print("The exception is", context.exception)
self.assertTrue(
'table already contains \'test_host\'' in str(context.exception))
def test_missing_units(self):
"""Test that a missing units attribute returns expected error"""
known_ddts = list()
filename = os.path.join(SAMPLE_FILES_DIR, "test_missing_units.meta")
with self.assertRaises(Exception) as context:
_ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV)
#print("The exception is", context.exception)
emsg = "Required property, 'units', missing, at"
self.assertTrue(emsg in str(context.exception))
def test_unknown_ddt_type(self):
"""Test that a DDT type = banana returns expected error"""
known_ddts = list()
filename = os.path.join(SAMPLE_FILES_DIR, "test_unknown_ddt_type.meta")
with self.assertRaises(Exception) as context:
_ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV)
#print("The exception is", context.exception)
emsg = "Unknown DDT type, banana, at "
self.assertTrue(emsg in str(context.exception))
def test_bad_table_type(self):
"""Test that a mismatched table type returns expected error"""
known_ddts = list()
filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_table_type.meta")
with self.assertRaises(Exception) as context:
_ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV)
#print("The exception is", context.exception)
emsg = "Section type, 'host', does not match table type, 'scheme'"
self.assertTrue(emsg in str(context.exception))
def test_bad_line_split(self):
"""Test that a bad split line with | returns expected error"""
known_ddts = list()
filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_line_split.meta")
with self.assertRaises(Exception) as context:
_ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV)
#print("The exception is", context.exception)
emsg = "Invalid variable property syntax, \'\', at "
self.assertTrue(emsg in str(context.exception))
def test_bad_table_key(self):
"""Test that a bad table key returns expected error"""
known_ddts = list()
filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_table_key.meta")
with self.assertRaises(Exception) as context:
_ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV)
#print("The exception is", context.exception)
emsg = "Invalid metadata table start property, 'something', at "
self.assertTrue(emsg in str(context.exception))
def test_bad_2nd_ccpp_arg_table(self):
"""Test that second arg table named ccpp-farg-table returns expected error"""
known_ddts = list()
filename = os.path.join(SAMPLE_FILES_DIR,
"test_bad_2nd_arg_table_header.meta")
with self.assertRaises(Exception) as context:
_ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV)
#print("The exception is", context.exception)
emsg = "Invalid variable property syntax, '[ccpp-farg-table]', at "
self.assertTrue(emsg in str(context.exception))
def test_bad_var_property_name(self):
"""Test that a ddt_type = None returns expected error"""
known_ddts = list()
filename = os.path.join(SAMPLE_FILES_DIR,
"test_bad_var_property_name.meta")
with self.assertRaises(Exception) as context:
_ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV)
#print("The exception is", context.exception)
emsg = "Invalid variable property name, 'none', at "
self.assertTrue(emsg in str(context.exception))
def test_double_table_properties(self):
"""Test that duplicate ccpp-table-properties returns expected error"""
known_ddts = list()
filename = os.path.join(SAMPLE_FILES_DIR,
"double_table_properties.meta")
with self.assertRaises(Exception) as context:
_ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV)
#print("The exception is", context.exception)
emsg = "Duplicate metadata table, test_host, at "
self.assertTrue(emsg in str(context.exception))
def test_missing_table_properties(self):
"""Test that a missing ccpp-table-properties returns expected error"""
known_ddts = list()
filename = os.path.join(SAMPLE_FILES_DIR,
"missing_table_properties.meta")
with self.assertRaises(Exception) as context:
_ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV)
#print("The exception is", context.exception)
emsg = "Invalid CCPP metadata line, '[ccpp-arg-table]', at "
self.assertTrue(emsg in str(context.exception))
def test_missing_table_type(self):
"""Test that a missing table type returns expected error"""
known_ddts = list()
filename = os.path.join(SAMPLE_FILES_DIR,
"test_missing_table_type.meta")
with self.assertRaises(Exception) as context:
_ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV)
#print("The exception is", context.exception)
emsg = "Invalid section type, 'None'"
self.assertTrue(emsg in str(context.exception))
def parse_scheme_files(scheme_filenames, preproc_defs, logger):
###############################################################################
"""
Gather information from scheme files (e.g., init, run, and finalize
methods) and return resulting dictionary.
"""
table_dict = {} # Duplicate check and for dependencies processing
header_dict = {} # To check for duplicates
known_ddts = list()
for filename in scheme_filenames:
logger.info('Reading CCPP schemes from {}'.format(filename))
# parse metadata file
mtables = parse_metadata_file(filename, known_ddts, logger)
fort_file = find_associated_fortran_file(filename)
ftables = parse_fortran_file(fort_file,
preproc_defs=preproc_defs,
logger=logger)
# Check Fortran against metadata (will raise an exception on error)
mheaders = list()
for sect in [x.sections() for x in mtables]:
mheaders.extend(sect)
# end for
fheaders = list()
for sect in [x.sections() for x in ftables]:
fheaders.extend(sect)
# end for
check_fortran_against_metadata(mheaders, fheaders, filename, fort_file,
logger)
# Check for duplicate tables, then add to dict
for table in mtables:
if table.table_name in table_dict:
duplicate_item_error(table.table_name, filename,
table.table_type,
table_dict[header.title])
else:
table_dict[table.table_name] = table
# end if
# end for
# Check for duplicate headers, then add to dict
for header in mheaders:
if header.title in header_dict:
duplicate_item_error(header.title, filename,
header.header_type,
header_dict[header.title])
else:
header_dict[header.title] = header
if header.header_type == 'ddt':
known_ddts.append(header.title)
# end if
# end if
# end for
# end for
return header_dict.values(), table_dict
def test_missing_intent(self):
"""Test that a missing intent returns expected error"""
known_ddts = list()
logger = None
filename = os.path.join(SAMPLE_FILES_DIR, "test_missing_intent.meta")
with self.assertRaises(Exception) as context:
tables = parse_metadata_file(filename, known_ddts, logger)
#print("The exception is", context.exception)
emsg = "Required property, 'intent', missing, at "
self.assertTrue(emsg in str(context.exception))
def test_bad_dimension(self):
"""Test that `dimension = banana` returns expected error"""
known_ddts = list()
logger = None
filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_dimension.meta")
with self.assertRaises(Exception) as context:
tables = parse_metadata_file(filename, known_ddts, logger)
#print("The exception is", context.exception)
self.assertTrue('Invalid \'dimensions\' property value, \'' in str(
context.exception))
def test_invalid_table_properties_type(self):
"""Test that an invalid ccpp-table-properties type returns expected error"""
known_ddts = list()
filename = os.path.join(SAMPLE_FILES_DIR,
"test_invalid_table_properties_type.meta")
with self.assertRaises(Exception) as context:
_ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV)
#print("The exception is", context.exception)
emsg = "Invalid metadata table type, 'banana', at "
self.assertTrue(emsg in str(context.exception))
def test_mismatch_section_table_title(self):
"""Test that mismatched section name and table title returns expected error"""
known_ddts = list()
logger = None
filename = os.path.join(SAMPLE_FILES_DIR,
"test_mismatch_section_table_title.meta")
with self.assertRaises(Exception) as context:
tables = parse_metadata_file(filename, known_ddts, logger)
#print("The exception is", context.exception)
emsg = "Section name, 'test_host', does not match table title, 'banana', at "
self.assertTrue(emsg in str(context.exception))
def test_duplicate_variable(self):
"""Test that a duplicate variable returns expected error"""
known_ddts = list()
filename = os.path.join(SAMPLE_FILES_DIR,
"test_duplicate_variable.meta")
with self.assertRaises(Exception) as context:
_ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV)
#print("The exception is", context.exception)
self.assertTrue(
'Invalid (duplicate) standard name in temp_calc_adjust_run, defined at '
in str(context.exception))
def test_bad_type_name(self):
"""Test that `type = banana` returns expected error"""
#Setup
known_ddts = list()
filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_type_name.meta")
#Exercise
with self.assertRaises(Exception) as context:
_ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV)
#Verify
#print("The exception is", context.exception)
self.assertTrue(
"Section type, 'banana', does not match table type, 'scheme'" in
str(context.exception))
def test_good_host_file(self):
"""Test that good host file test_host.meta returns one header named test_host"""
#Setup
known_ddts = list()
filename = os.path.join(SAMPLE_FILES_DIR, "test_host.meta")
#Exercise
result = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV)
#Verify that:
# no dependencies is returned as ''
# rel_path is returned as None
# size of returned list equals number of headers in the test file
# ccpp-table-properties name is 'test_host'
dependencies = result[0].dependencies
rel_path = result[0].relative_path
self.assertFalse('' in dependencies)
self.assertEqual(len(dependencies), 0)
self.assertIsNone(rel_path)
self.assertEqual(len(result), 1)
titles = [elem.table_name for elem in result]
self.assertIn('test_host',
titles,
msg="Header name 'test_host' is expected but not found")
def read_new_metadata(filename,
module_name,
table_name,
scheme_name=None,
subroutine_name=None):
"""Read metadata in new format and convert output to ccpp_prebuild metadata dictionary"""
if not os.path.isfile(filename):
raise Exception("New metadata file {0} not found".format(filename))
# Save metadata, because this routine new_metadata
# is called once for every table in that file
if filename in NEW_METADATA_SAVE.keys():
new_metadata_headers = NEW_METADATA_SAVE[filename]
else:
new_metadata_headers = parse_metadata_file(
filename,
known_ddts=registered_fortran_ddt_names(),
logger=logging.getLogger(__name__))
NEW_METADATA_SAVE[filename] = new_metadata_headers
# Record dependencies for the metadata table (only applies to schemes)
dependencies = []
# Convert new metadata for requested table to old metadata dictionary
metadata = collections.OrderedDict()
for new_metadata_header in new_metadata_headers:
for metadata_section in new_metadata_header.sections():
# Module or DDT tables
if not scheme_name:
# Module property tables
if not metadata_section.title == table_name:
# Skip this table, since it is not requested right now
continue
# Distinguish between module argument tables and DDT argument tables
if metadata_section.title == module_name:
container = encode_container(module_name)
else:
container = encode_container(module_name,
metadata_section.title)
# Add to dependencies
if new_metadata_header.relative_path:
dependencies += [
os.path.join(new_metadata_header.relative_path, x)
for x in new_metadata_header.dependencies
]
else:
dependencies += new_metadata_header.dependencies
else:
# Scheme property tables
if not metadata_section.title == table_name:
# Skip this table, since it is not requested right now
continue
container = encode_container(module_name, scheme_name,
table_name)
# Add to dependencies
if new_metadata_header.relative_path:
dependencies += [
os.path.join(new_metadata_header.relative_path, x)
for x in new_metadata_header.dependencies
]
else:
dependencies += new_metadata_header.dependencies
for new_var in metadata_section.variable_list():
standard_name = new_var.get_prop_value('standard_name')
# DH* 2020-05-26
# Legacy extension for inconsistent metadata (use of horizontal_dimension versus horizontal_loop_extent).
# Since horizontal_dimension and horizontal_loop_extent have the same attributes (otherwise it doesn't
# make sense), we swap the standard name and add a note to the long name - 2021-05-26: this is now an error.
legacy_note = ''
if standard_name == 'horizontal_loop_extent' and scheme_name and \
(table_name.endswith("_init") or table_name.endswith("_finalize")):
#logging.warn("Legacy extension - replacing variable 'horizontal_loop_extent'" + \
# " with 'horizontal_dimension' in table {}".format(table_name))
#standard_name = 'horizontal_dimension'
#legacy_note = ' replaced by horizontal dimension (legacy extension)'
raise Exception("Legacy extension DISABLED: replacing variable 'horizontal_loop_extent'" + \
" with 'horizontal_dimension' in table {}".format(table_name))
elif standard_name == 'horizontal_dimension' and scheme_name and table_name.endswith(
"_run"):
#logging.warn("Legacy extension - replacing variable 'horizontal_dimension'" + \
# " with 'horizontal_loop_extent' in table {}".format(table_name))
#standard_name = 'horizontal_loop_extent'
#legacy_note = ' replaced by horizontal loop extent (legacy extension)'
raise Exception("Legacy extension DISABLED: replacing variable 'horizontal_dimension'" + \
" with 'horizontal_loop_extent' in table {}".format(table_name))
# Adjust dimensions
dimensions = new_var.get_prop_value('dimensions')
if scheme_name and (table_name.endswith("_init") or table_name.endswith("_finalize")) \
and 'horizontal_loop_extent' in dimensions:
#logging.warn("Legacy extension - replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \
# "for variable {} in table {}".format(standard_name,table_name))
#dimensions = ['horizontal_dimension' if x=='horizontal_loop_extent' else x for x in dimensions]
raise Exception("Legacy extension DISABLED: replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \
"for variable {} in table {}".format(standard_name,table_name))
elif scheme_name and table_name.endswith(
"_run") and 'horizontal_dimension' in dimensions:
#logging.warn("Legacy extension - replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \
# "for variable {} in table {}".format(standard_name,table_name))
#dimensions = ['horizontal_loop_extent' if x=='horizontal_dimension' else x for x in dimensions]
raise Exception("Legacy extension DISABLED: replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \
"for variable {} in table {}".format(standard_name,table_name))
elif not scheme_name and 'horizontal_dimension' in dimensions:
raise Exception("Legacy extension DISABLED: replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \
"for variable {} in table {}".format(standard_name,table_name))
# *DH 2020-05-26
if not new_var.get_prop_value('active'):
# If it doesn't have an active attribute, then the variable is always active (default)
active = 'T'
elif new_var.get_prop_value('active').lower() == '.true.':
active = 'T'
elif new_var.get_prop_value(
'active') and new_var.get_prop_value(
'active').lower() == '.false.':
active = 'F'
else:
# Replace multiple whitespaces, preserve case
active = ' '.join(new_var.get_prop_value('active').split())
# DH* 20210812
# Workaround for Fortran DDTs incorrectly having the type of
# the DDT copied into the kind attribute in parse_metadata_file
if new_var.is_ddt() and new_var.get_prop_value('kind'):
kind = ''
else:
kind = new_var.get_prop_value('kind')
#kind = new_var.get_prop_value('kind')
# *DH 20210812
var = Var(
standard_name=standard_name,
long_name=new_var.get_prop_value('long_name') +
legacy_note,
units=new_var.get_prop_value('units'),
local_name=new_var.get_prop_value('local_name'),
type=new_var.get_prop_value('type').lower(),
dimensions=dimensions,
container=container,
kind=kind,
intent=new_var.get_prop_value('intent'),
optional='T'
if new_var.get_prop_value('optional') else 'F',
active=active,
)
# Check for duplicates in same table
if standard_name in metadata.keys():
raise Exception(
"Error, multiple definitions of standard name {} in new metadata table {}"
.format(standard_name, table_name))
metadata[standard_name] = [var]
return (metadata, dependencies)
