def test_dependencies_rel_path(self): """Test that relative_path and dependencies from ccpp-table-properties are read in correctly""" known_ddts = list() logger = None filename = os.path.join(SAMPLE_FILES_DIR, "test_dependencies_rel_path.meta") result = parse_metadata_file(filename, known_ddts, logger) dependencies = result[0].dependencies rel_path = result[0].relative_path titles = [elem.table_name for elem in result] self.assertEqual(len(dependencies), 4) self.assertIn('machine.F', dependencies, msg="Dependency 'machine.F' is expected but not found") self.assertIn( 'physcons.F90', dependencies, msg="Dependency 'physcons.F90' is expected but not found") self.assertIn( 'GFDL_parse_tracers.F90', dependencies, msg="Dependency 'GFDL_parse_tracers.F90' is expected but not found" ) self.assertIn('rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90', dependencies, \ msg="Header name 'rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90' is expected but not found") self.assertIn(rel_path, "../../ccpp/physics/physics") self.assertEqual(len(result), 1) self.assertIn('test_host', titles, msg="Table name 'test_host' is expected but not found")
def test_MetadataTable_parse_table(tmpdir): path = str(tmpdir.join("table.meta")) with open(path, "w") as f: f.write(example_table) dummy_run_env = CCPPFrameworkEnv(None, ndict={ 'host_files': '', 'scheme_files': '', 'suites': '' }) metadata_headers = parse_metadata_file( path, known_ddts=registered_fortran_ddt_names(), run_env=dummy_run_env) # check metadata header assert len(metadata_headers) == 1 metadata_header = metadata_headers[0] assert metadata_header.table_name == "<name>" assert metadata_header.table_type == "scheme" assert metadata_header.relative_path == "path" assert metadata_header.dependencies == ["a.f", "b.f"] # check metadata section assert len(metadata_header.sections()) == 1 metadata_section = metadata_header.sections()[0] assert metadata_section.name == "<name>" assert metadata_section.type == "scheme" (im_data, ) = metadata_section.variable_list() assert isinstance(im_data, Var) assert im_data.get_dimensions() == []
def test_MetadataTable_parse_table(tmpdir): path = str(tmpdir.join("table.meta")) with open(path, "w") as f: f.write(example_table) metadata_headers = parse_metadata_file( path, known_ddts=registered_fortran_ddt_names(), logger=logging.getLogger(__name__)) # check metadata header assert len(metadata_headers) == 1 metadata_header = metadata_headers[0] assert metadata_header.table_name == "<name>" assert metadata_header.table_type == "scheme" assert metadata_header.relative_path == "path" assert metadata_header.dependencies == ["a.f", "b.f"] # check metadata section assert len(metadata_header.sections()) == 1 metadata_section = metadata_header.sections()[0] assert metadata_section.name == "<name>" assert metadata_section.type == "scheme" (im_data, ) = metadata_section.variable_list() assert isinstance(im_data, Var) assert im_data.get_dimensions() == []
def test_good_multi_ccpp_arg_table(self): """Test that good file with 4 ccpp-arg-table returns 4 headers""" known_ddts = list() logger = None filename = os.path.join(SAMPLE_FILES_DIR, "test_multi_ccpp_arg_tables.meta") #Exercise result = parse_metadata_file(filename, known_ddts, logger) #Verify that size of returned list equals number of ccpp-table-properties in the test file # ccpp-arg-tables are returned in result[0].sections() and result[1].sections() self.assertEqual(len(result), 2) titles = list() for table in result: titles.extend([x.title for x in table.sections()]) self.assertIn('vmr_type', titles, msg="Header name 'vmr_type' is expected but not found") self.assertIn( 'make_ddt_run', titles, msg="Header name 'make_ddt_run' is expected but not found") self.assertIn( 'make_ddt_init', titles, msg="Header name 'make_ddt_init' is expected but not found") self.assertIn( 'make_ddt_finalize', titles, msg="Header name 'make_ddt_finalize' is expected but not found")
def parse_host_model_files(host_filenames, preproc_defs, host_name, logger): ############################################################################### """ Gather information from host files (e.g., DDTs, registry) and return a host model object with the information. """ header_dict = {} table_dict = {} known_ddts = list() for filename in host_filenames: logger.info('Reading host model data from {}'.format(filename)) # parse metadata file mtables = parse_metadata_file(filename, known_ddts, logger) fort_file = find_associated_fortran_file(filename) ftables = parse_fortran_file(fort_file, preproc_defs=preproc_defs, logger=logger) # Check Fortran against metadata (will raise an exception on error) mheaders = list() for sect in [x.sections() for x in mtables]: mheaders.extend(sect) # end for fheaders = list() for sect in [x.sections() for x in ftables]: fheaders.extend(sect) # end for check_fortran_against_metadata(mheaders, fheaders, filename, fort_file, logger) # Check for duplicate tables, then add to dict for table in mtables: if table.table_name in table_dict: duplicate_item_error(table.table_name, filename, table.table_type, table_dict[header.title]) else: table_dict[table.table_name] = table # end if # end for # Check for duplicate headers, then add to dict for header in mheaders: if header.title in header_dict: duplicate_item_error(header.title, filename, header.header_type, header_dict[header.title]) else: header_dict[header.title] = header if header.header_type == 'ddt': known_ddts.append(header.title) # end if # end for # end for if not host_name: host_name = None # end if host_model = HostModel(table_dict, host_name, logger) return host_model
def test_invalid_intent(self): """Test that an invalid intent returns expected error""" known_ddts = list() filename = os.path.join(SAMPLE_FILES_DIR, "test_invalid_intent.meta") with self.assertRaises(Exception) as context: _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) #print("The exception is", context.exception) self.assertTrue('Invalid \'intent\' property value, \'banana\', at ' in str(context.exception))
def test_double_header(self): """Test that a duplicate header returns expected error""" known_ddts = list() filename = os.path.join(SAMPLE_FILES_DIR, "double_header.meta") with self.assertRaises(Exception) as context: _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) #print("The exception is", context.exception) self.assertTrue( 'table already contains \'test_host\'' in str(context.exception))
def test_missing_units(self): """Test that a missing units attribute returns expected error""" known_ddts = list() filename = os.path.join(SAMPLE_FILES_DIR, "test_missing_units.meta") with self.assertRaises(Exception) as context: _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) #print("The exception is", context.exception) emsg = "Required property, 'units', missing, at" self.assertTrue(emsg in str(context.exception))
def test_unknown_ddt_type(self): """Test that a DDT type = banana returns expected error""" known_ddts = list() filename = os.path.join(SAMPLE_FILES_DIR, "test_unknown_ddt_type.meta") with self.assertRaises(Exception) as context: _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) #print("The exception is", context.exception) emsg = "Unknown DDT type, banana, at " self.assertTrue(emsg in str(context.exception))
def test_bad_table_type(self): """Test that a mismatched table type returns expected error""" known_ddts = list() filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_table_type.meta") with self.assertRaises(Exception) as context: _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) #print("The exception is", context.exception) emsg = "Section type, 'host', does not match table type, 'scheme'" self.assertTrue(emsg in str(context.exception))
def test_bad_line_split(self): """Test that a bad split line with | returns expected error""" known_ddts = list() filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_line_split.meta") with self.assertRaises(Exception) as context: _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) #print("The exception is", context.exception) emsg = "Invalid variable property syntax, \'\', at " self.assertTrue(emsg in str(context.exception))
def test_bad_table_key(self): """Test that a bad table key returns expected error""" known_ddts = list() filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_table_key.meta") with self.assertRaises(Exception) as context: _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) #print("The exception is", context.exception) emsg = "Invalid metadata table start property, 'something', at " self.assertTrue(emsg in str(context.exception))
def test_bad_2nd_ccpp_arg_table(self): """Test that second arg table named ccpp-farg-table returns expected error""" known_ddts = list() filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_2nd_arg_table_header.meta") with self.assertRaises(Exception) as context: _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) #print("The exception is", context.exception) emsg = "Invalid variable property syntax, '[ccpp-farg-table]', at " self.assertTrue(emsg in str(context.exception))
def test_bad_var_property_name(self): """Test that a ddt_type = None returns expected error""" known_ddts = list() filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_var_property_name.meta") with self.assertRaises(Exception) as context: _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) #print("The exception is", context.exception) emsg = "Invalid variable property name, 'none', at " self.assertTrue(emsg in str(context.exception))
def test_double_table_properties(self): """Test that duplicate ccpp-table-properties returns expected error""" known_ddts = list() filename = os.path.join(SAMPLE_FILES_DIR, "double_table_properties.meta") with self.assertRaises(Exception) as context: _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) #print("The exception is", context.exception) emsg = "Duplicate metadata table, test_host, at " self.assertTrue(emsg in str(context.exception))
def test_missing_table_properties(self): """Test that a missing ccpp-table-properties returns expected error""" known_ddts = list() filename = os.path.join(SAMPLE_FILES_DIR, "missing_table_properties.meta") with self.assertRaises(Exception) as context: _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) #print("The exception is", context.exception) emsg = "Invalid CCPP metadata line, '[ccpp-arg-table]', at " self.assertTrue(emsg in str(context.exception))
def test_missing_table_type(self): """Test that a missing table type returns expected error""" known_ddts = list() filename = os.path.join(SAMPLE_FILES_DIR, "test_missing_table_type.meta") with self.assertRaises(Exception) as context: _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) #print("The exception is", context.exception) emsg = "Invalid section type, 'None'" self.assertTrue(emsg in str(context.exception))
def parse_scheme_files(scheme_filenames, preproc_defs, logger): ############################################################################### """ Gather information from scheme files (e.g., init, run, and finalize methods) and return resulting dictionary. """ table_dict = {} # Duplicate check and for dependencies processing header_dict = {} # To check for duplicates known_ddts = list() for filename in scheme_filenames: logger.info('Reading CCPP schemes from {}'.format(filename)) # parse metadata file mtables = parse_metadata_file(filename, known_ddts, logger) fort_file = find_associated_fortran_file(filename) ftables = parse_fortran_file(fort_file, preproc_defs=preproc_defs, logger=logger) # Check Fortran against metadata (will raise an exception on error) mheaders = list() for sect in [x.sections() for x in mtables]: mheaders.extend(sect) # end for fheaders = list() for sect in [x.sections() for x in ftables]: fheaders.extend(sect) # end for check_fortran_against_metadata(mheaders, fheaders, filename, fort_file, logger) # Check for duplicate tables, then add to dict for table in mtables: if table.table_name in table_dict: duplicate_item_error(table.table_name, filename, table.table_type, table_dict[header.title]) else: table_dict[table.table_name] = table # end if # end for # Check for duplicate headers, then add to dict for header in mheaders: if header.title in header_dict: duplicate_item_error(header.title, filename, header.header_type, header_dict[header.title]) else: header_dict[header.title] = header if header.header_type == 'ddt': known_ddts.append(header.title) # end if # end if # end for # end for return header_dict.values(), table_dict
def test_missing_intent(self): """Test that a missing intent returns expected error""" known_ddts = list() logger = None filename = os.path.join(SAMPLE_FILES_DIR, "test_missing_intent.meta") with self.assertRaises(Exception) as context: tables = parse_metadata_file(filename, known_ddts, logger) #print("The exception is", context.exception) emsg = "Required property, 'intent', missing, at " self.assertTrue(emsg in str(context.exception))
def test_bad_dimension(self): """Test that `dimension = banana` returns expected error""" known_ddts = list() logger = None filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_dimension.meta") with self.assertRaises(Exception) as context: tables = parse_metadata_file(filename, known_ddts, logger) #print("The exception is", context.exception) self.assertTrue('Invalid \'dimensions\' property value, \'' in str( context.exception))
def test_invalid_table_properties_type(self): """Test that an invalid ccpp-table-properties type returns expected error""" known_ddts = list() filename = os.path.join(SAMPLE_FILES_DIR, "test_invalid_table_properties_type.meta") with self.assertRaises(Exception) as context: _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) #print("The exception is", context.exception) emsg = "Invalid metadata table type, 'banana', at " self.assertTrue(emsg in str(context.exception))
def test_mismatch_section_table_title(self): """Test that mismatched section name and table title returns expected error""" known_ddts = list() logger = None filename = os.path.join(SAMPLE_FILES_DIR, "test_mismatch_section_table_title.meta") with self.assertRaises(Exception) as context: tables = parse_metadata_file(filename, known_ddts, logger) #print("The exception is", context.exception) emsg = "Section name, 'test_host', does not match table title, 'banana', at " self.assertTrue(emsg in str(context.exception))
def test_duplicate_variable(self): """Test that a duplicate variable returns expected error""" known_ddts = list() filename = os.path.join(SAMPLE_FILES_DIR, "test_duplicate_variable.meta") with self.assertRaises(Exception) as context: _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) #print("The exception is", context.exception) self.assertTrue( 'Invalid (duplicate) standard name in temp_calc_adjust_run, defined at ' in str(context.exception))
def test_bad_type_name(self): """Test that `type = banana` returns expected error""" #Setup known_ddts = list() filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_type_name.meta") #Exercise with self.assertRaises(Exception) as context: _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) #Verify #print("The exception is", context.exception) self.assertTrue( "Section type, 'banana', does not match table type, 'scheme'" in str(context.exception))
def test_good_host_file(self): """Test that good host file test_host.meta returns one header named test_host""" #Setup known_ddts = list() filename = os.path.join(SAMPLE_FILES_DIR, "test_host.meta") #Exercise result = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) #Verify that: # no dependencies is returned as '' # rel_path is returned as None # size of returned list equals number of headers in the test file # ccpp-table-properties name is 'test_host' dependencies = result[0].dependencies rel_path = result[0].relative_path self.assertFalse('' in dependencies) self.assertEqual(len(dependencies), 0) self.assertIsNone(rel_path) self.assertEqual(len(result), 1) titles = [elem.table_name for elem in result] self.assertIn('test_host', titles, msg="Header name 'test_host' is expected but not found")
def read_new_metadata(filename, module_name, table_name, scheme_name=None, subroutine_name=None): """Read metadata in new format and convert output to ccpp_prebuild metadata dictionary""" if not os.path.isfile(filename): raise Exception("New metadata file {0} not found".format(filename)) # Save metadata, because this routine new_metadata # is called once for every table in that file if filename in NEW_METADATA_SAVE.keys(): new_metadata_headers = NEW_METADATA_SAVE[filename] else: new_metadata_headers = parse_metadata_file( filename, known_ddts=registered_fortran_ddt_names(), logger=logging.getLogger(__name__)) NEW_METADATA_SAVE[filename] = new_metadata_headers # Record dependencies for the metadata table (only applies to schemes) dependencies = [] # Convert new metadata for requested table to old metadata dictionary metadata = collections.OrderedDict() for new_metadata_header in new_metadata_headers: for metadata_section in new_metadata_header.sections(): # Module or DDT tables if not scheme_name: # Module property tables if not metadata_section.title == table_name: # Skip this table, since it is not requested right now continue # Distinguish between module argument tables and DDT argument tables if metadata_section.title == module_name: container = encode_container(module_name) else: container = encode_container(module_name, metadata_section.title) # Add to dependencies if new_metadata_header.relative_path: dependencies += [ os.path.join(new_metadata_header.relative_path, x) for x in new_metadata_header.dependencies ] else: dependencies += new_metadata_header.dependencies else: # Scheme property tables if not metadata_section.title == table_name: # Skip this table, since it is not requested right now continue container = encode_container(module_name, scheme_name, table_name) # Add to dependencies if new_metadata_header.relative_path: dependencies += [ os.path.join(new_metadata_header.relative_path, x) for x in new_metadata_header.dependencies ] else: dependencies += new_metadata_header.dependencies for new_var in metadata_section.variable_list(): standard_name = new_var.get_prop_value('standard_name') # DH* 2020-05-26 # Legacy extension for inconsistent metadata (use of horizontal_dimension versus horizontal_loop_extent). # Since horizontal_dimension and horizontal_loop_extent have the same attributes (otherwise it doesn't # make sense), we swap the standard name and add a note to the long name - 2021-05-26: this is now an error. legacy_note = '' if standard_name == 'horizontal_loop_extent' and scheme_name and \ (table_name.endswith("_init") or table_name.endswith("_finalize")): #logging.warn("Legacy extension - replacing variable 'horizontal_loop_extent'" + \ # " with 'horizontal_dimension' in table {}".format(table_name)) #standard_name = 'horizontal_dimension' #legacy_note = ' replaced by horizontal dimension (legacy extension)' raise Exception("Legacy extension DISABLED: replacing variable 'horizontal_loop_extent'" + \ " with 'horizontal_dimension' in table {}".format(table_name)) elif standard_name == 'horizontal_dimension' and scheme_name and table_name.endswith( "_run"): #logging.warn("Legacy extension - replacing variable 'horizontal_dimension'" + \ # " with 'horizontal_loop_extent' in table {}".format(table_name)) #standard_name = 'horizontal_loop_extent' #legacy_note = ' replaced by horizontal loop extent (legacy extension)' raise Exception("Legacy extension DISABLED: replacing variable 'horizontal_dimension'" + \ " with 'horizontal_loop_extent' in table {}".format(table_name)) # Adjust dimensions dimensions = new_var.get_prop_value('dimensions') if scheme_name and (table_name.endswith("_init") or table_name.endswith("_finalize")) \ and 'horizontal_loop_extent' in dimensions: #logging.warn("Legacy extension - replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \ # "for variable {} in table {}".format(standard_name,table_name)) #dimensions = ['horizontal_dimension' if x=='horizontal_loop_extent' else x for x in dimensions] raise Exception("Legacy extension DISABLED: replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \ "for variable {} in table {}".format(standard_name,table_name)) elif scheme_name and table_name.endswith( "_run") and 'horizontal_dimension' in dimensions: #logging.warn("Legacy extension - replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \ # "for variable {} in table {}".format(standard_name,table_name)) #dimensions = ['horizontal_loop_extent' if x=='horizontal_dimension' else x for x in dimensions] raise Exception("Legacy extension DISABLED: replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \ "for variable {} in table {}".format(standard_name,table_name)) elif not scheme_name and 'horizontal_dimension' in dimensions: raise Exception("Legacy extension DISABLED: replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \ "for variable {} in table {}".format(standard_name,table_name)) # *DH 2020-05-26 if not new_var.get_prop_value('active'): # If it doesn't have an active attribute, then the variable is always active (default) active = 'T' elif new_var.get_prop_value('active').lower() == '.true.': active = 'T' elif new_var.get_prop_value( 'active') and new_var.get_prop_value( 'active').lower() == '.false.': active = 'F' else: # Replace multiple whitespaces, preserve case active = ' '.join(new_var.get_prop_value('active').split()) # DH* 20210812 # Workaround for Fortran DDTs incorrectly having the type of # the DDT copied into the kind attribute in parse_metadata_file if new_var.is_ddt() and new_var.get_prop_value('kind'): kind = '' else: kind = new_var.get_prop_value('kind') #kind = new_var.get_prop_value('kind') # *DH 20210812 var = Var( standard_name=standard_name, long_name=new_var.get_prop_value('long_name') + legacy_note, units=new_var.get_prop_value('units'), local_name=new_var.get_prop_value('local_name'), type=new_var.get_prop_value('type').lower(), dimensions=dimensions, container=container, kind=kind, intent=new_var.get_prop_value('intent'), optional='T' if new_var.get_prop_value('optional') else 'F', active=active, ) # Check for duplicates in same table if standard_name in metadata.keys(): raise Exception( "Error, multiple definitions of standard name {} in new metadata table {}" .format(standard_name, table_name)) metadata[standard_name] = [var] return (metadata, dependencies)