def read_new_metadata(filename, module_name, table_name, scheme_name=None, subroutine_name=None): """Read metadata in new format and convert output to ccpp_prebuild metadata dictionary""" if not os.path.isfile(filename): raise Exception("New metadata file {0} not found".format(filename)) # Save metadata, because this routine new_metadata # is called once for every table in that file if filename in NEW_METADATA_SAVE.keys(): new_metadata_headers = NEW_METADATA_SAVE[filename] else: new_metadata_headers = MetadataHeader.parse_metadata_file(filename) NEW_METADATA_SAVE[filename] = new_metadata_headers # Convert new metadata for requested table to old metadata dictionary metadata = collections.OrderedDict() for new_metadata_header in new_metadata_headers: if not scheme_name: if not new_metadata_header.title == table_name: # Skip this table, since it is not requested right now continue if new_metadata_header.title == module_name: container = encode_container(module_name) else: container = encode_container(module_name, new_metadata_header.title) else: if not new_metadata_header.title == table_name: # Skip this table, since it is not requested right now continue container = encode_container(module_name, scheme_name, table_name) for new_var in new_metadata_header.variable_list(): standard_name = new_var.get_prop_value('standard_name') rank = len(new_var.get_prop_value('dimensions')) var = Var( standard_name=standard_name, long_name=new_var.get_prop_value('long_name'), units=new_var.get_prop_value('units'), local_name=new_var.get_prop_value('local_name'), type=new_var.get_prop_value('type'), container=container, kind=new_var.get_prop_value('kind'), intent=new_var.get_prop_value('intent'), optional='T' if new_var.get_prop_value('optional') else 'F', ) # Set rank using integer-setter method var.rank = rank # Check for duplicates in same table if standard_name in metadata.keys(): raise Exception( "Error, multiple definitions of standard name {0} in new metadata table {1}" .format(standard_name, table_name)) metadata[standard_name] = [var] return metadata
def parse_preamble_data(statements, pobj, spec_name, endmatch, logger): """Parse module variables or DDT definitions from a module preamble or parse program variables from the beginning of a program. """ inspec = True mheaders = list() var_dict = VarDictionary(spec_name) psrc = ParseSource(spec_name, 'MODULE', pobj) active_table = None while inspec and (statements is not None): while len(statements) > 0: statement = statements.pop(0) # End program or module pmatch = endmatch.match(statement) asmatch = arg_table_start_re.match(statement) type_def = fortran_type_definition(statement) if asmatch is not None: active_table = asmatch.group(1).lower() elif (pmatch is not None) or is_contains_statement( statement, inspec): # We are done with the specification inspec = False if len(var_dict.variable_list()) > 0: mheader = MetadataHeader(title=spec_name, type_in='MODULE', module=spec_name, var_dict=var_dict, logger=logger) mheaders.append(mheader) # End if break elif (type_def is not None) and (type_def[0].lower() == active_table): statements, ddt = parse_type_def(statements, type_def, spec_name, pobj, logger) mheaders.append(ddt) active_table = None else: # We should have a variable definition to add if ((not is_comment_statement(statement, logger)) and (not parse_use_statement({}, statement, pobj, logger)) and (active_table == spec_name)): vars = parse_fortran_var_decl(statement, psrc, logger=logger) for var in vars: var_dict.add_variable(var) # End for # End if # End if # End while if inspec and (len(statements) == 0): statements = read_statements(pobj) # End if # End while return statements, mheaders
def convert_to_html(filename_in, outdir, logger): """Convert a metadata file into html (one html file for each table)""" if not os.path.isfile(filename_in): raise Exception("Metadata file {} not found".format(filename_in)) logger.info("Converting file {} to HTML".format(filename_in)) metadata_headers = MetadataHeader.parse_metadata_file(filename_in) for metadata_header in metadata_headers: filename_out = metadata_header.to_html(outdir, ATTRIBUTES) if filename_out: logger.info(" ... wrote {}".format(filename_out))
def test_MetadataHeader_parse_table(tmpdir): path = str(tmpdir.join("table.meta")) with open(path, "w") as f: f.write(example_table) table1, table2 = MetadataHeader.parse_metadata_file(path) # check first table assert table1.name == "<name>" assert table1.type == "scheme" assert table1.dependencies == ["path/a.f", "path/b.f"] # check second table assert table2.name == "<name>" assert table2.type == "scheme" (im_data,) = table2.variable_list() assert isinstance(im_data, Var) assert im_data.get_dimensions() == []
def parse_type_def(statements, type_def, mod_name, pobj, logger): psrc = ParseSource(mod_name, 'DDT', pobj) seen_contains = False mheader = None var_dict = VarDictionary(type_def[0]) inspec = True while inspec and (statements is not None): while len(statements) > 0: statement = statements.pop(0) # End program or module pmatch = end_type_re.match(statement) if pmatch is not None: # We hit the end of the type, make a header mheader = MetadataHeader(title=type_def[0], type_in='DDT', module=mod_name, var_dict=var_dict, logger=logger) inspec = False elif is_contains_statement(statement, inspec): seen_contains = True elif not seen_contains: # Comment of variable if ((not is_comment_statement(statement, logger)) and (not parse_use_statement({}, statement, pobj, logger))): vars = parse_fortran_var_decl(statement, psrc, logger=logger) for var in vars: var_dict.add_variable(var) # End for # End if else: # We are just skipping lines until the end type pass # End if # End while if inspec and (len(statements) == 0): statements = read_statements(pobj) # End if # End while return statements, mheader
def read_new_metadata(filename, module_name, table_name, scheme_name=None, subroutine_name=None): """Read metadata in new format and convert output to ccpp_prebuild metadata dictionary""" if not os.path.isfile(filename): raise Exception("New metadata file {0} not found".format(filename)) # Save metadata, because this routine new_metadata # is called once for every table in that file if filename in NEW_METADATA_SAVE.keys(): new_metadata_headers = NEW_METADATA_SAVE[filename] else: new_metadata_headers = MetadataHeader.parse_metadata_file(filename) NEW_METADATA_SAVE[filename] = new_metadata_headers # Record dependencies for the metadata table (only applies to schemes) has_property_table = False dependencies = [] # Convert new metadata for requested table to old metadata dictionary metadata = collections.OrderedDict() for new_metadata_header in new_metadata_headers: # Module or DDT tables if not scheme_name: # Module property tables if new_metadata_header.property_table and new_metadata_header.title == module_name: # If this is a ccpp-table-properties table for a module, it can only contain dependencies; # ensure that for module tables, the header type is "module" if not new_metadata_header.header_type == 'module': raise Exception( "Unsupported header_type '{}' for table properties for modules" .format(new_metadata_header.header_type)) dependencies += new_metadata_header.dependencies has_property_table = True continue # DDT property tables elif new_metadata_header.property_table: # If this is a ccpp-table-properties table for a DDT, it can only contain dependencies; # ensure that for DDT tables, the header type is "ddt" if not new_metadata_header.header_type == 'ddt': raise Exception( "Unsupported header_type '{}' for table properties for DDTs" .format(new_metadata_header.header_type)) dependencies += new_metadata_header.dependencies has_property_table = True continue # Module or DDT argument tables else: if not new_metadata_header.title == table_name: # Skip this table, since it is not requested right now continue # Distinguish between module argument tables and DDT argument tables if new_metadata_header.title == module_name: container = encode_container(module_name) else: container = encode_container(module_name, new_metadata_header.title) else: # Scheme property tables if new_metadata_header.property_table and new_metadata_header.title == scheme_name: # If this is a ccpp-table-properties table for a scheme, it can only contain dependencies; # ensure that for scheme tables, the header type is "scheme" if not new_metadata_header.header_type == 'scheme': raise Exception( "Unsupported header_type '{}' for table properties for schemes" .format(new_metadata_header.header_type)) dependencies += new_metadata_header.dependencies has_property_table = True continue # Scheme argument tables else: if not new_metadata_header.title == table_name: # Skip this table, since it is not requested right now continue container = encode_container(module_name, scheme_name, table_name) for new_var in new_metadata_header.variable_list(): standard_name = new_var.get_prop_value('standard_name') # DH* 2020-05-26 # Legacy extension for inconsistent metadata (use of horizontal_dimension versus horizontal_loop_extent). # Since horizontal_dimension and horizontal_loop_extent have the same attributes (otherwise it doesn't # make sense), we swap the standard name and add a note to the long name legacy_note = '' if standard_name == 'horizontal_loop_extent' and scheme_name and \ (table_name.endswith("_init") or table_name.endswith("_finalize")): logging.warn("Legacy extension - replacing variable 'horizontal_loop_extent'" + \ " with 'horizontal_dimension' in table {}".format(table_name)) standard_name = 'horizontal_dimension' legacy_note = ' replaced by horizontal dimension (legacy extension)' elif standard_name == 'horizontal_dimension' and scheme_name and table_name.endswith( "_run"): logging.warn("Legacy extension - replacing variable 'horizontal_dimension' " + \ "with 'horizontal_loop_extent' in table {}".format(table_name)) standard_name = 'horizontal_loop_extent' legacy_note = ' replaced by horizontal loop extent (legacy extension)' # Adjust dimensions dimensions = new_var.get_prop_value('dimensions') if scheme_name and (table_name.endswith("_init") or table_name.endswith("_finalize")) \ and 'horizontal_loop_extent' in dimensions: logging.warn("Legacy extension - replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \ "for variable {} in table {}".format(standard_name,table_name)) dimensions = [ 'horizontal_dimension' if x == 'horizontal_loop_extent' else x for x in dimensions ] elif scheme_name and table_name.endswith( "_run") and 'horizontal_dimension' in dimensions: logging.warn("Legacy extension - replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \ "for variable {} in table {}".format(standard_name,table_name)) dimensions = [ 'horizontal_loop_extent' if x == 'horizontal_dimension' else x for x in dimensions ] # *DH 2020-05-26 if new_var.get_prop_value('active').lower() == '.true.': active = 'T' elif new_var.get_prop_value('active').lower() == '.false.': active = 'F' else: # Replace multiple whitespaces, preserve case active = ' '.join(new_var.get_prop_value('active').split()) var = Var( standard_name=standard_name, long_name=new_var.get_prop_value('long_name') + legacy_note, units=new_var.get_prop_value('units'), local_name=new_var.get_prop_value('local_name'), type=new_var.get_prop_value('type'), dimensions=dimensions, container=container, kind=new_var.get_prop_value('kind'), intent=new_var.get_prop_value('intent'), optional='T' if new_var.get_prop_value('optional') else 'F', active=active, ) # Check for duplicates in same table if standard_name in metadata.keys(): raise Exception( "Error, multiple definitions of standard name {} in new metadata table {}" .format(standard_name, table_name)) metadata[standard_name] = [var] # CCPP property tables are mandatory if not has_property_table: if scheme_name: raise Exception("Metadata file {} for scheme {} does not have a [ccpp-table-properties] section,".format(filename, scheme_name) + \ " or the 'name = ...' attribute in the [ccpp-table-properties] is wrong") else: raise Exception("Metadata file {} for table {} does not have a [ccpp-table-properties] section,".format(filename, table_name) + \ " or the 'name = ...' attribute in the [ccpp-table-properties] is wrong") return (metadata, dependencies)
def parse_metadata_tables_typedefs(model): # Lookup table local_name -> dimensions dimensions = { 'ccpp_error_flag': [], 'ccpp_error_message': [], 'ccpp_loop_counter': [], 'ccpp_block_number': [], 'ccpp_thread_number': [], 'ccpp_t': [], } for filename in METADATA_TYPEDEFS[model]: metadata_headers = MetadataHeader.parse_metadata_file(filename) for metadata_header in metadata_headers: for var in metadata_header.variable_list(): standard_name = var.get_prop_value('standard_name') if standard_name in dimensions.keys(): raise ValueError( "Duplicate standard name {} in type/variable definition metadata tables" .format(standard_name)) dimensions[standard_name] = var.get_prop_value('dimensions') # # Add missing variables (not used by FV3) dimensions['lw_heating_rate_spectral'] = [ 'horizontal_dimension', 'adjusted_vertical_layer_dimension_for_radiation', 'number_of_aerosol_bands_for_longwave_radiation' ] dimensions['lw_fluxes'] = [ 'horizontal_dimension', 'adjusted_vertical_level_dimension_for_radiation' ] dimensions['cloud_optical_depth'] = [ 'horizontal_dimension', 'adjusted_vertical_layer_dimension_for_radiation' ] # dimensions['sw_heating_rate_spectral'] = [ 'horizontal_dimension', 'adjusted_vertical_layer_dimension_for_radiation', 'number_of_aerosol_bands_for_shortwave_radiation' ] dimensions['sw_fluxes'] = [ 'horizontal_dimension', 'adjusted_vertical_level_dimension_for_radiation' ] dimensions['cloud_single_scattering_albedo'] = [ 'horizontal_dimension', 'adjusted_vertical_layer_dimension_for_radiation' ] dimensions['cloud_asymmetry_parameter'] = [ 'horizontal_dimension', 'adjusted_vertical_layer_dimension_for_radiation' ] # dimensions['specified_kinematic_surface_upward_sensible_heat_flux'] = [ 'horizontal_dimension' ] dimensions['specified_kinematic_surface_upward_latent_heat_flux'] = [ 'horizontal_dimension' ] dimensions['vonKarman_constant'] = [] # return dimensions
def parse_scheme_metadata(statements, pobj, spec_name, table_name, logger): "Parse dummy argument information from a subroutine" psrc = None mheader = None var_dict = None scheme_name = None # Find the subroutine line, should be first executable statement inpreamble = False insub = True while insub and (statements is not None): while len(statements) > 0: statement = statements.pop(0) smatch = subroutine_re.match(statement) esmatch = end_subroutine_re.match(statement) pmatch = endmodule_re.match(statement) asmatch = arg_table_start_re.match(statement) if asmatch is not None: # We have run off the end of something, hope that is okay # Put this statement back for the caller to deal with statements.insert(0, statement) insub = False break elif (pmatch is not None): # We have run off the end of the module, hope that is okay pobj.leave_region('MODULE', region_name=spec_name) insub = False break elif smatch is not None: scheme_name = smatch.group(1) inpreamble = scheme_name == table_name if inpreamble: if smatch.group(2) is not None: scheme_args = [ x.strip().lower() for x in smatch.group(2).split(',') ] else: scheme_args = list() # End if scheme_set = set(scheme_args) var_dict = VarDictionary(scheme_name) psrc = ParseSource(scheme_name, 'SCHEME', pobj) # End if elif inpreamble: # Process a preamble statement (use or argument declaration) if esmatch is not None: inpreamble = False insub = False elif ((not is_comment_statement(statement, logger)) and (not parse_use_statement({}, statement, pobj, logger)) and ('intent' in statement.lower())): vars = parse_fortran_var_decl(statement, psrc, logger=logger) for var in vars: lname = var.get_prop_value('local_name').lower() if lname in scheme_set: scheme_set.remove(lname) else: raise ParseSyntaxError('dummy argument', token=lname, context=pobj) # End if var_dict.add_variable(var) # End for # End if # End if # End while if insub and (len(statements) == 0): statements = read_statements(pobj) # End if # End while if (scheme_name is not None) and (var_dict is not None): mheader = MetadataHeader(title=scheme_name, type_in='SCHEME', module=spec_name, var_dict=var_dict, logger=logger) # End if return statements, mheader
def read_new_metadata(filename, module_name, table_name, scheme_name=None, subroutine_name=None): """Read metadata in new format and convert output to ccpp_prebuild metadata dictionary""" if not os.path.isfile(filename): raise Exception("New metadata file {0} not found".format(filename)) # Save metadata, because this routine new_metadata # is called once for every table in that file if filename in NEW_METADATA_SAVE.keys(): new_metadata_headers = NEW_METADATA_SAVE[filename] else: new_metadata_headers = MetadataHeader.parse_metadata_file(filename) NEW_METADATA_SAVE[filename] = new_metadata_headers # Convert new metadata for requested table to old metadata dictionary metadata = collections.OrderedDict() for new_metadata_header in new_metadata_headers: if not scheme_name: if not new_metadata_header.title == table_name: # Skip this table, since it is not requested right now continue if new_metadata_header.title == module_name: container = encode_container(module_name) else: container = encode_container(module_name, new_metadata_header.title) else: if not new_metadata_header.title == table_name: # Skip this table, since it is not requested right now continue container = encode_container(module_name, scheme_name, table_name) for new_var in new_metadata_header.variable_list(): standard_name = new_var.get_prop_value('standard_name') # DH* 2020-05-26 # Legacy extension for inconsistent metadata (use of horizontal_dimension versus horizontal_loop_extent). # Since horizontal_dimension and horizontal_loop_extent have the same attributes (otherwise it doesn't # make sense), we swap the standard name and add a note to the long name legacy_note = '' if standard_name == 'horizontal_loop_extent' and scheme_name and \ (table_name.endswith("_init") or table_name.endswith("finalize")): logging.warn("Legacy extension - replacing variable 'horizontal_loop_extent'" + \ " with 'horizontal_dimension' in table {}".format(table_name)) standard_name = 'horizontal_dimension' legacy_note = ' replaced by horizontal dimension (legacy extension)' elif standard_name == 'horizontal_dimension' and scheme_name and table_name.endswith( "_run"): logging.warn("Legacy extension - replacing variable 'horizontal_dimension' " + \ "with 'horizontal_loop_extent' in table {}".format(table_name)) standard_name = 'horizontal_loop_extent' legacy_note = ' replaced by horizontal loop extent (legacy extension)' # Adjust dimensions dimensions = new_var.get_prop_value('dimensions') if scheme_name and (table_name.endswith("_init") or table_name.endswith("finalize")) \ and 'horizontal_loop_extent' in dimensions: logging.warn("Legacy extension - replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \ "for variable {} in table {}".format(standard_name,table_name)) dimensions = [ 'horizontal_dimension' if x == 'horizontal_loop_extent' else x for x in dimensions ] elif scheme_name and table_name.endswith( "_run") and 'horizontal_dimension' in dimensions: logging.warn("Legacy extension - replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \ "for variable {} in table {}".format(standard_name,table_name)) dimensions = [ 'horizontal_loop_extent' if x == 'horizontal_dimension' else x for x in dimensions ] # *DH 2020-05-26 if new_var.get_prop_value('active').lower() == '.true.': active = 'T' elif new_var.get_prop_value('active').lower() == '.false.': active = 'F' else: # Replace multiple whitespaces and use lowercase throughout active = ' '.join( new_var.get_prop_value('active').lower().split()) var = Var( standard_name=standard_name, long_name=new_var.get_prop_value('long_name') + legacy_note, units=new_var.get_prop_value('units'), local_name=new_var.get_prop_value('local_name'), type=new_var.get_prop_value('type'), dimensions=dimensions, container=container, kind=new_var.get_prop_value('kind'), intent=new_var.get_prop_value('intent'), optional='T' if new_var.get_prop_value('optional') else 'F', active=active, ) # Check for duplicates in same table if standard_name in metadata.keys(): raise Exception( "Error, multiple definitions of standard name {0} in new metadata table {1}" .format(standard_name, table_name)) metadata[standard_name] = [var] return metadata
print("Found {} files with arg tables".format(len(tfilenames))) total_headers = 0 for tfile in tfilenames: try: tbase = os.path.basename(tfile) file = os.path.join(pdir, tbase) mbase = "{}.meta".format('.'.join(tbase.split('.')[:-1])) mdfile = os.path.join(pdir, mbase) if not os.path.exists(file): infile = tfile if not os.path.exists(infile): print("WARNING: Cannot find '{}'".format(infile)) else: convert_metadata.convert_file(infile, file, mdfile, logger) # End if # End if if os.path.exists(mdfile): mh = MetadataHeader.parse_metadata_file(mdfile) print("{} metadata headers parsed in {}".format( len(mh), mdfile)) total_headers = total_headers + len(mh) else: print("{} not found!".format(mdfile)) # End if except ValueError as ve: print("{}: {}".format(infile, ve)) # End except # End for print("Found {} total metadata headers".format(total_headers)) # End if __main__