def read_new_metadata(filename, module_name, table_name, scheme_name=None, subroutine_name=None): """Read metadata in new format and convert output to ccpp_prebuild metadata dictionary""" if not os.path.isfile(filename): raise Exception("New metadata file {0} not found".format(filename)) # Save metadata, because this routine new_metadata # is called once for every table in that file if filename in NEW_METADATA_SAVE.keys(): new_metadata_headers = NEW_METADATA_SAVE[filename] else: new_metadata_headers = MetadataHeader.parse_metadata_file(filename) NEW_METADATA_SAVE[filename] = new_metadata_headers # Convert new metadata for requested table to old metadata dictionary metadata = collections.OrderedDict() for new_metadata_header in new_metadata_headers: if not scheme_name: if not new_metadata_header.title == table_name: # Skip this table, since it is not requested right now continue if new_metadata_header.title == module_name: container = encode_container(module_name) else: container = encode_container(module_name, new_metadata_header.title) else: if not new_metadata_header.title == table_name: # Skip this table, since it is not requested right now continue container = encode_container(module_name, scheme_name, table_name) for new_var in new_metadata_header.variable_list(): standard_name = new_var.get_prop_value('standard_name') rank = len(new_var.get_prop_value('dimensions')) var = Var( standard_name=standard_name, long_name=new_var.get_prop_value('long_name'), units=new_var.get_prop_value('units'), local_name=new_var.get_prop_value('local_name'), type=new_var.get_prop_value('type'), container=container, kind=new_var.get_prop_value('kind'), intent=new_var.get_prop_value('intent'), optional='T' if new_var.get_prop_value('optional') else 'F', ) # Set rank using integer-setter method var.rank = rank # Check for duplicates in same table if standard_name in metadata.keys(): raise Exception( "Error, multiple definitions of standard name {0} in new metadata table {1}" .format(standard_name, table_name)) metadata[standard_name] = [var] return metadata
def generate_scheme_caps(metadata, arguments, caps_dir, module_use_template_scheme_cap): """Generate scheme caps for all schemes parsed.""" success = True # Change to physics directory os.chdir(caps_dir) # List of filenames of scheme caps scheme_caps = [] for module_name in arguments.keys(): for scheme_name in arguments[module_name].keys(): for subroutine_name in arguments[module_name][scheme_name].keys(): # Skip subroutines without argument table or with empty argument table if not arguments[module_name][scheme_name][subroutine_name]: continue # Create cap cap = Cap() cap.filename = "{0}_cap.F90".format(scheme_name) scheme_caps.append(cap.filename) # Parse all subroutines and their arguments to generate the cap capdata = collections.OrderedDict() for subroutine_name in arguments[module_name][scheme_name].keys(): capdata[subroutine_name] = [] for var_name in arguments[module_name][scheme_name][subroutine_name]: container = encode_container(module_name, scheme_name, subroutine_name) for var in metadata[var_name]: if var.container == container: capdata[subroutine_name].append(var) break # If required (not at the moment), add module use statements to module_use_template_scheme_cap module_use_statement = module_use_template_scheme_cap # Write cap cap.write(module_name, module_use_statement, capdata) # os.chdir(BASEDIR) return (success, scheme_caps)
def generate_scheme_caps(metadata_define, metadata_request, arguments, pset_schemes, ccpp_field_maps, caps_dir): """Generate scheme caps for all schemes parsed.""" success = True # Change to caps directory os.chdir(caps_dir) # List of filenames of scheme caps scheme_caps = [] for module_name in arguments.keys(): for scheme_name in arguments[module_name].keys(): for subroutine_name in arguments[module_name][scheme_name].keys(): # Skip subroutines without argument table or with empty argument table if not arguments[module_name][scheme_name][subroutine_name]: continue # Create cap cap = Cap() cap.filename = "{0}_cap.F90".format(scheme_name) scheme_caps.append(cap.filename) # Parse all subroutines and their arguments to generate the cap capdata = collections.OrderedDict() for subroutine_name in arguments[module_name][scheme_name].keys(): capdata[subroutine_name] = [] for var_name in arguments[module_name][scheme_name][ subroutine_name]: container = encode_container(module_name, scheme_name, subroutine_name) for var in metadata_request[var_name]: if var.container == container: capdata[subroutine_name].append(var) break # Write cap using the unique physics set for the scheme pset = pset_schemes[scheme_name][0] cap.write(module_name, capdata, ccpp_field_maps[pset], metadata_define) # os.chdir(BASEDIR) return (success, scheme_caps)
def parse_scheme_tables(filepath, filename): """Parses metadata tables for a physics scheme that requests/requires variables as input arguments. Metadata tables can only describe variables required by a subroutine 'subroutine_name' of scheme 'scheme_name' inside a module 'module_name'. Each variable (standard_name) can exist only once, i.e. each entry (list of variables) in the metadata dictionary contains only one element (variable = instance of class Var defined in mkcap.py). The metadata dictionaries of the individual schemes are merged afterwards (called from ccpp_prebuild.py) using merge_metadata_dicts, where multiple instances of variables are compared for compatibility and collected in a list (entry in the merged metadata dictionary). The merged metadata dictionary of all schemes (which contains only compatible variable instances in the list referred to by standard_name) is then compared to the unique definition in the metadata dictionary of the variables provided by the host model using compare_metadata in ccpp_prebuild.py.""" # Set debug to true if logging level is debug debug = logging.getLogger().getEffectiveLevel() == logging.DEBUG # Final metadata container for all variables in file metadata = collections.OrderedDict() # Registry of modules and derived data types in file registry = collections.OrderedDict() # Argument lists of each subroutine in the file arguments = collections.OrderedDict() # List of Dependencies for this scheme dependencies = collections.OrderedDict() # Read all lines of the file at once with (open(filename, 'r')) as file: try: file_lines = file.readlines() except UnicodeDecodeError: raise Exception( "Decoding error while trying to read file {}, check that the file only contains ASCII characters" .format(filename)) lines = [] original_line_numbers = [] buffer = '' for i in range(len(file_lines)): line = file_lines[i].rstrip('\n').strip() # Skip empty lines if line == '' or line == '&': continue # Remove line continuations: concatenate with following lines if line.endswith('&'): buffer += file_lines[i].rstrip('\n').replace('&', ' ') continue # Write out line with buffer and reset buffer lines.append(buffer + file_lines[i].rstrip('\n').replace('&', ' ')) original_line_numbers.append(i + 1) buffer = '' del file_lines # Find all modules within the file, and save the start and end lines module_lines = collections.OrderedDict() line_counter = 0 for line in lines: # For the purpose of identifying module constructs, remove any trailing comments from line if '!' in line and not line.startswith('!'): line = line[:line.find('!')] words = line.split() if len(words) > 1 and words[0].lower( ) == 'module' and not words[1].lower() == 'procedure': module_name = words[1].strip() if module_name in registry.keys(): raise Exception( 'Duplicate module name {0}'.format(module_name)) registry[module_name] = collections.OrderedDict() module_lines[module_name] = {'startline': line_counter} elif len(words) > 1 and words[0].lower() == 'end' and words[1].lower( ) == 'module': try: test_module_name = words[2] except IndexError: logging.warning( 'Warning, encountered closing statement "end module" without module name; assume module_name is {0}' .format(module_name)) test_module_name = module_name if not module_name == test_module_name: raise Exception( 'Module names in opening/closing statement do not match: {0} vs {1}' .format(module_name, test_module_name)) module_lines[module_name]['endline'] = line_counter line_counter += 1 # Parse each module in the file separately for module_name in registry.keys(): startline = module_lines[module_name]['startline'] endline = module_lines[module_name]['endline'] line_counter = 0 in_subroutine = False for line in lines[startline:endline]: # For the purpose of identifying scheme constructs, remove any trailing comments from line if '!' in line and not line.startswith('!'): line = line[:line.find('!')] current_line_number = startline + line_counter words = line.split() for j in range(len(words)): # Check for the word 'subroutine', that it is the first word in the line, # and that a name exists afterwards. Nested subroutines are ignored. if words[j].lower( ) == 'subroutine' and j == 0 and len(words) > 1: if in_subroutine: logging.debug( 'Warning, ignoring nested subroutine in module {0} and subroutine {1}' .format(module_name, subroutine_name)) continue subroutine_name = words[j + 1].split('(')[0].strip() # Consider the last substring separated by a '_' of the subroutine name as a 'postfix' if subroutine_name.find('_') >= 0: scheme_name = None subroutine_suffix = None for ccpp_stage in CCPP_STAGES: pattern = '^(.*)_{}$'.format(ccpp_stage) match = re.match(pattern, subroutine_name) if match: scheme_name = match.group(1) subroutine_suffix = ccpp_stage break if match: if not scheme_name == module_name: raise Exception( 'Scheme name differs from module name: module_name="{0}" vs. scheme_name="{1}"' .format(module_name, scheme_name)) if not scheme_name in registry[module_name].keys(): registry[module_name][ scheme_name] = collections.OrderedDict() if subroutine_name in registry[module_name][ scheme_name].keys(): raise Exception( 'Duplicate subroutine name {0} in module {1}' .format(subroutine_name, module_name)) registry[module_name][scheme_name][ subroutine_name] = [current_line_number] in_subroutine = True elif words[j].lower() == 'subroutine' and j == 1 and words[ j - 1].lower() == 'end': try: test_subroutine_name = words[j + 1] except IndexError: logging.warning('Warning, encountered closing statement "end subroutine" without subroutine name; ' +\ ' assume subroutine_name is {0}'.format(subroutine_name)) test_subroutine_name = subroutine_name if in_subroutine and subroutine_name == test_subroutine_name: in_subroutine = False registry[module_name][scheme_name][ subroutine_name].append(current_line_number) # Avoid problems by enforcing end statements to carry a descriptor (subroutine, module, ...) elif in_subroutine and len( words) == 1 and words[0].lower() == 'end': raise Exception('Encountered closing statement "end" without descriptor (subroutine, module, ...): ' +\ 'line {0}="{1}" in file {2}'.format(original_line_numbers[current_line_number], line, filename)) line_counter += 1 # Check that for each registered subroutine the start and end lines were found for scheme_name in registry[module_name].keys(): for subroutine_name in registry[module_name][scheme_name].keys(): if not len(registry[module_name][scheme_name] [subroutine_name]) == 2: raise Exception( 'Error parsing start and end lines for subroutine {0} in module {1}' .format(subroutine_name, module_name)) logging.debug('Parsing file {0} with registry {1}'.format( filename, registry)) for scheme_name in registry[module_name].keys(): # Record the dependencies for the scheme if not scheme_name in dependencies.keys(): dependencies[scheme_name] = [] for subroutine_name in registry[module_name][scheme_name].keys(): # Record the order of variables in the call list to each subroutine in a list if not scheme_name in arguments.keys(): arguments[scheme_name] = collections.OrderedDict() if not subroutine_name in arguments[scheme_name].keys(): arguments[scheme_name][subroutine_name] = [] # Find the argument table corresponding to each subroutine by searching # "upward" from the subroutine definition line for the "arg_table_SubroutineName" section table_found = False header_line_number = None for line_number in range( registry[module_name][scheme_name][subroutine_name][0], -1, -1): line = lines[line_number] words = line.split() for word in words: if (len(words) > 2 and words[0] in ['!!', '!>'] and '\section' in words[1] and 'arg_table_{0}'.format(subroutine_name) in words[2]): table_found = True header_line_number = line_number + 1 table_name = subroutine_name break else: for word in words: if 'arg_table_{0}'.format( subroutine_name) in word: raise Exception( "Malformatted table found in {0} / {1} / {2} / {3}" .format(filename, module_name, scheme_name, subroutine_name)) if table_found: break # If an argument table is found, parse it if table_found: if 'htmlinclude' in lines[header_line_number].lower(): words = lines[header_line_number].split() if words[0] == '!!' and words[ 1] == '\\htmlinclude' and len(words) == 3: filename_parts = filename.split('.') metadata_filename = '.'.join( filename_parts[0:len(filename_parts) - 1]) + '.meta' (this_metadata, these_dependencies) = read_new_metadata( metadata_filename, module_name, table_name, scheme_name=scheme_name, subroutine_name=subroutine_name) if these_dependencies: # Remove duplicates when combining lists dependencies[scheme_name] = list( set(dependencies[scheme_name] + these_dependencies)) for var_name in this_metadata.keys(): # Add standard_name to argument list for this subroutine arguments[scheme_name][subroutine_name].append( var_name) # For all instances of this var (can be only one) in this subroutine's metadata, # add to global metadata and check for compatibility with existing variables for var in this_metadata[var_name]: if not var_name in metadata.keys(): metadata[var_name] = [var] else: for existing_var in metadata[var_name]: if not existing_var.compatible( var): raise Exception('New entry for variable {0}'.format(var_name) + \ ' in argument table of subroutine {0}'.format(subroutine_name) +\ ' is incompatible with existing entry:\n' +\ ' existing: {0}\n'.format(existing_var.print_debug()) +\ ' vs. new: {0}'.format(var.print_debug())) metadata[var_name].append(var) else: raise Exception( "Invalid definition of new metadata format in file {}, \htmlinclude must be preceeded by '!! ' : {}" .format(filename, lines[header_line_number])) # Next line must denote the end of table, # i.e. look for a line containing only '!!' line_number = header_line_number + 1 nextline = lines[line_number] nextwords = nextline.split() if len(nextwords) == 1 and nextwords[0].strip( ) == '!!': end_of_table = True else: raise Exception( 'Encountered invalid format "{0}" of new metadata table hook in table {1}' .format(line, table_name)) line_number += 1 else: words = lines[header_line_number].split() if len(words) == 1 and words[0].strip() == '!!': logging.info( "Legacy extension - skip empty table for {}". format(table_name)) end_of_table = True line_number += 1 else: raise Exception( "Invalid definition of metadata in file {0}: {1}" .format(filename, words)) # After parsing entire metadata table for the subroutine, check that all # mandatory CCPP variables are present - skip empty tables. if arguments[scheme_name][subroutine_name]: for var_name in CCPP_MANDATORY_VARIABLES.keys(): if not var_name in arguments[scheme_name][ subroutine_name]: raise Exception( 'Mandatory CCPP variable {0} not declared in metadata table of subroutine {1}' .format(var_name, subroutine_name)) # Sort the dependencies to avoid differences in the auto-generated code dependencies[scheme_name].sort() # Debugging output to screen and to XML if debug and len(metadata.keys()) > 0: # To screen logging.debug('Module name: {}'.format(module_name)) for scheme_name in registry[module_name].keys(): logging.debug('Scheme name: {}'.format(scheme_name)) logging.debug('Scheme dependencies: {}'.format( dependencies[scheme_name])) for subroutine_name in registry[module_name][scheme_name].keys( ): container = encode_container(module_name, scheme_name, subroutine_name) vars_in_subroutine = [] for var_name in metadata.keys(): for var in metadata[var_name]: if var.container == container: vars_in_subroutine.append(var_name) logging.debug('Variables in subroutine {}: {}'.format( subroutine_name, ', '.join(vars_in_subroutine))) # Standard output to screen elif len(metadata.keys()) > 0: for scheme_name in registry[module_name].keys(): if dependencies[scheme_name]: logging.info( 'Parsed tables in scheme {} with dependencies {}'. format(scheme_name, dependencies[scheme_name])) else: logging.info( 'Parsed tables in scheme {}'.format(scheme_name)) # End of loop over all module_names # Add absolute path to dependencies for scheme_name in dependencies.keys(): if dependencies[scheme_name]: dependencies[scheme_name] = [ os.path.join(filepath, x) for x in dependencies[scheme_name] ] for dependency in dependencies[scheme_name]: if not os.path.isfile(dependency): raise Exception( "Dependency {} for scheme table {} does not exit". format(dependency, scheme_name)) return (metadata, arguments, dependencies)
def parse_variable_tables(filepath, filename): """Parses metadata tables on the host model side that define the available variables. Metadata tables can refer to variables inside a module or as part of a derived datatype, which itself is defined inside a module (depending on the location of the metadata table). Each variable (standard_name) can exist only once, i.e. each entry (list of variables) in the metadata dictionary contains only one element (variable = instance of class Var defined in mkcap.py)""" # Set debug to true if logging level is debug debug = logging.getLogger().getEffectiveLevel() == logging.DEBUG # Final metadata container for all variables in file metadata = collections.OrderedDict() # Registry of modules and derived data types in file registry = collections.OrderedDict() # List of dependencies for this scheme dependencies = collections.OrderedDict() # Read all lines of the file at once with (open(filename, 'r')) as file: try: file_lines = file.readlines() except UnicodeDecodeError: raise Exception( "Decoding error while trying to read file {}, check that the file only contains ASCII characters" .format(filename)) lines = [] buffer = '' for i in range(len(file_lines)): line = file_lines[i].rstrip('\n').strip() # Skip empty lines if line == '' or line == '&': continue # Remove line continuations: concatenate with following lines if line.endswith('&'): buffer += file_lines[i].rstrip('\n').replace('&', ' ') continue # Write out line with buffer and reset buffer lines.append(buffer + file_lines[i].rstrip('\n').replace('&', ' ')) buffer = '' del file_lines # Find all modules within the file, and save the start and end lines module_lines = collections.OrderedDict() line_counter = 0 for line in lines: words = line.split() if len(words) > 1 and words[0].lower() in [ 'module', 'program' ] and not words[1].lower() == 'procedure': module_name = words[1].strip() if module_name in registry.keys(): raise Exception( 'Duplicate module name {0}'.format(module_name)) registry[module_name] = collections.OrderedDict() module_lines[module_name] = {'startline': line_counter} elif len(words) > 1 and words[0].lower() == 'end' and words[1].lower( ) in ['module', 'program']: try: test_module_name = words[2] except IndexError: logging.warning( 'Encountered closing statement "end module" without module name; assume module_name is {0}' .format(module_name)) test_module_name = module_name if not module_name == test_module_name: raise Exception( 'Module names in opening/closing statement do not match: {0} vs {1}' .format(module_name, test_module_name)) module_lines[module_name]['endline'] = line_counter line_counter += 1 # Parse each module in the file separately for module_name in registry.keys(): startline = module_lines[module_name]['startline'] endline = module_lines[module_name]['endline'] line_counter = 0 in_type = False for line in lines[startline:endline]: # For the purpose of identifying module, type and scheme constructs, remove any trailing comments from line if '!' in line and not line.startswith('!'): line = line[:line.find('!')] current_line_number = startline + line_counter words = line.split() for j in range(len(words)): # Check for the word 'type', that it is the first word in the line, # and that a name exists afterwards. It is assumed that definitions # (not usage) of derived types cannot be nested - reasonable for Fortran. # The following if / elif / else statements filter lines that do not # contain a type definition. # # Ignore words containing type that are not 'type', 'type,', 'type::'; # this includes variable declarations of a user defined type, e.g. 'type(mytype) ::' if not (words[j].lower()=='type' or \ words[j].lower().startswith('type,') or \ words[j].lower().startswith('type::')): continue # Ignore variable declarations of a user defined type with a space # between 'type' and '(', e.g. 'type (mytype) ::' elif j == 0 and len(words) > 1 and words[j + 1].startswith('('): continue # Ignore lines starting with 'type is' or 'type is(' (select type statements) elif (words[j].lower() == 'type' and j==0 and j<len(words)-1 and \ (words[j+1].lower() == 'is' or words[j+1].lower().startswith('is('))): continue # Detect 'end type TYPENAME' and (fallback) unlabeled 'end type' statements elif words[j].lower() == 'type' and j == 1 and words[ j - 1].lower() == 'end': if not in_type: raise Exception( 'Encountered "end_type" without corresponding "type" statement' ) try: test_type_name = words[j + 1] except IndexError: logging.warning( 'Encountered closing statement "end type" without type name; assume type_name is {0}' .format(type_name)) test_type_name = type_name if not type_name == test_type_name: raise Exception( 'Type names in opening/closing statement do not match: {0} vs {1}' .format(type_name, test_type_name)) in_type = False registry[module_name][type_name].append( current_line_number) # If type is not the first word, ignore the word elif j > 0: continue # Detect type definition using Ftype_type_decl class, routine # type_def_line and extract type_name else: type_declaration = Ftype_type_decl.type_def_line( line.strip()) if in_type: raise Exception( 'Nested definitions of derived types not supported' ) in_type = True type_name = type_declaration[0] if type_name in registry[module_name].keys(): raise Exception( 'Duplicate derived type name {0} in module {1}'. format(type_name, module_name)) registry[module_name][type_name] = [current_line_number] # Done with user defined type detection line_counter += 1 logging.debug('Parsing file {0} with registry {1}'.format( filename, registry)) # Variables can either be defined at module-level or in derived types - alongside with their tables line_counter = 0 in_table = False in_type = False for line in lines[startline:endline]: current_line_number = startline + line_counter # Check for beginning of new table words = line.split() # This is case sensitive if len(words) > 2 and words[0] in [ '!!', '!>' ] and '\section' in words[1] and 'arg_table_' in words[2]: if in_table: raise Exception( 'Encountered table start for table {0} while still in table {1}' .format(words[2].replace('arg_table_', ''), table_name)) table_name = words[2].replace('arg_table_', '') if not (table_name == module_name or table_name in registry[module_name].keys()): raise Exception( 'Encountered table with name {0} without corresponding module or type name' .format(table_name)) in_table = True if not table_name in dependencies.keys(): dependencies[table_name] = [] header_line_number = current_line_number + 1 line_counter += 1 continue elif (words[0].startswith('!!') or words[0].startswith('!>')) and '\section' in words[0]: raise Exception( "Malformatted table found in {0} / {1} / {2}".format( filename, module_name, table_name)) # If an argument table is found, parse it if in_table: words = line.split('|') # Separate the table headers if current_line_number == header_line_number: if 'htmlinclude' in line.lower(): words = line.split() if words[0] == '!!' and words[ 1] == '\\htmlinclude' and len(words) == 3: filename_parts = filename.split('.') metadata_filename = '.'.join( filename_parts[0:len(filename_parts) - 1]) + '.meta' (this_metadata, these_dependencies) = read_new_metadata( metadata_filename, module_name, table_name) if these_dependencies: # Remove duplicates when combining lists dependencies[table_name] = list( set(dependencies[table_name] + these_dependencies)) for var_name in this_metadata.keys(): for var in this_metadata[var_name]: if var_name in CCPP_MANDATORY_VARIABLES.keys( ) and not CCPP_MANDATORY_VARIABLES[ var_name].compatible(var): raise Exception('Entry for variable {0}'.format(var_name) + \ ' in argument table {0}'.format(table_name) +\ ' is incompatible with mandatory variable:\n' +\ ' existing: {0}\n'.format(CCPP_MANDATORY_VARIABLES[var_name].print_debug()) +\ ' vs. new: {0}'.format(var.print_debug())) # Add variable to metadata dictionary if not var_name in metadata.keys(): metadata[var_name] = [var] else: for existing_var in metadata[var_name]: if not existing_var.compatible( var): raise Exception('New entry for variable {0}'.format(var_name) + \ ' in argument table {0}'.format(table_name) +\ ' is incompatible with existing entry:\n' +\ ' existing: {0}\n'.format(existing_var.print_debug()) +\ ' vs. new: {0}'.format(var.print_debug())) metadata[var_name].append(var) else: raise Exception( "Invalid definition of new metadata format in file {}, \htmlinclude must be preceeded by '!! ' : {}" .format(filename, line)) line_counter += 1 continue # Check for blank table if len(words) <= 1: logging.debug( 'Skipping blank table {0}'.format(table_name)) in_table = False line_counter += 1 continue table_header = [x.strip() for x in words[1:-1]] # Check that only valid table headers are used for item in table_header: if not item in VALID_ITEMS['header']: raise Exception( 'Invalid column header {0} in argument table {1}' .format(item, table_name)) # Locate mandatory column 'standard_name' try: standard_name_index = table_header.index( 'standard_name') except ValueError: raise Exception( 'Mandatory column standard_name not found in argument table {0}' .format(table_name)) line_counter += 1 # DH* warn or raise error for old metadata format logging.warn( "Old metadata table found for table {}".format( table_name)) #raise Exception("Old metadata table found for table {}".format(table_name)) # *DH continue else: if len(words) == 1: # End of table if words[0].strip() == '!!': if not current_line_number == header_line_number + 1: raise Exception( "Invalid definition of new metadata format in file {0}" .format(filename)) in_table = False line_counter += 1 continue else: raise Exception( 'Encountered invalid line "{0}" in argument table {1}' .format(line, table_name)) else: raise Exception( "Invalid definition of metadata in file {0}: {1}". format(filename, words)) line_counter += 1 # Informative output to screen if debug and len(metadata.keys()) > 0: for module_name in registry.keys(): logging.debug('Module name: {0}'.format(module_name)) container = encode_container(module_name) vars_in_module = [] for var_name in metadata.keys(): for var in metadata[var_name]: if var.container == container: vars_in_module.append(var_name) logging.debug('Module variables: {0}'.format( ', '.join(vars_in_module))) for type_name in registry[module_name].keys(): container = encode_container(module_name, type_name) vars_in_type = [] for var_name in metadata.keys(): for var in metadata[var_name]: if var.container == container: vars_in_type.append(var_name) logging.debug('Variables in derived type {0}: {1}'.format( type_name, ', '.join(vars_in_type))) if len(metadata.keys()) > 0: logging.info( 'Parsed variable definition tables in module {0}'.format( module_name)) # Add absolute path to dependencies for table_name in dependencies.keys(): if dependencies[table_name]: dependencies[table_name] = [ os.path.join(filepath, x) for x in dependencies[table_name] ] for dependency in dependencies[table_name]: if not os.path.isfile(dependency): raise Exception( "Dependency {} for variable table {} does not exit". format(dependency, table_name)) return (metadata, dependencies)
def read_new_metadata(filename, module_name, table_name, scheme_name=None, subroutine_name=None): """Read metadata in new format and convert output to ccpp_prebuild metadata dictionary""" if not os.path.isfile(filename): raise Exception("New metadata file {0} not found".format(filename)) # Save metadata, because this routine new_metadata # is called once for every table in that file if filename in NEW_METADATA_SAVE.keys(): new_metadata_headers = NEW_METADATA_SAVE[filename] else: new_metadata_headers = MetadataHeader.parse_metadata_file(filename) NEW_METADATA_SAVE[filename] = new_metadata_headers # Record dependencies for the metadata table (only applies to schemes) has_property_table = False dependencies = [] # Convert new metadata for requested table to old metadata dictionary metadata = collections.OrderedDict() for new_metadata_header in new_metadata_headers: # Module or DDT tables if not scheme_name: # Module property tables if new_metadata_header.property_table and new_metadata_header.title == module_name: # If this is a ccpp-table-properties table for a module, it can only contain dependencies; # ensure that for module tables, the header type is "module" if not new_metadata_header.header_type == 'module': raise Exception( "Unsupported header_type '{}' for table properties for modules" .format(new_metadata_header.header_type)) dependencies += new_metadata_header.dependencies has_property_table = True continue # DDT property tables elif new_metadata_header.property_table: # If this is a ccpp-table-properties table for a DDT, it can only contain dependencies; # ensure that for DDT tables, the header type is "ddt" if not new_metadata_header.header_type == 'ddt': raise Exception( "Unsupported header_type '{}' for table properties for DDTs" .format(new_metadata_header.header_type)) dependencies += new_metadata_header.dependencies has_property_table = True continue # Module or DDT argument tables else: if not new_metadata_header.title == table_name: # Skip this table, since it is not requested right now continue # Distinguish between module argument tables and DDT argument tables if new_metadata_header.title == module_name: container = encode_container(module_name) else: container = encode_container(module_name, new_metadata_header.title) else: # Scheme property tables if new_metadata_header.property_table and new_metadata_header.title == scheme_name: # If this is a ccpp-table-properties table for a scheme, it can only contain dependencies; # ensure that for scheme tables, the header type is "scheme" if not new_metadata_header.header_type == 'scheme': raise Exception( "Unsupported header_type '{}' for table properties for schemes" .format(new_metadata_header.header_type)) dependencies += new_metadata_header.dependencies has_property_table = True continue # Scheme argument tables else: if not new_metadata_header.title == table_name: # Skip this table, since it is not requested right now continue container = encode_container(module_name, scheme_name, table_name) for new_var in new_metadata_header.variable_list(): standard_name = new_var.get_prop_value('standard_name') # DH* 2020-05-26 # Legacy extension for inconsistent metadata (use of horizontal_dimension versus horizontal_loop_extent). # Since horizontal_dimension and horizontal_loop_extent have the same attributes (otherwise it doesn't # make sense), we swap the standard name and add a note to the long name legacy_note = '' if standard_name == 'horizontal_loop_extent' and scheme_name and \ (table_name.endswith("_init") or table_name.endswith("_finalize")): logging.warn("Legacy extension - replacing variable 'horizontal_loop_extent'" + \ " with 'horizontal_dimension' in table {}".format(table_name)) standard_name = 'horizontal_dimension' legacy_note = ' replaced by horizontal dimension (legacy extension)' elif standard_name == 'horizontal_dimension' and scheme_name and table_name.endswith( "_run"): logging.warn("Legacy extension - replacing variable 'horizontal_dimension' " + \ "with 'horizontal_loop_extent' in table {}".format(table_name)) standard_name = 'horizontal_loop_extent' legacy_note = ' replaced by horizontal loop extent (legacy extension)' # Adjust dimensions dimensions = new_var.get_prop_value('dimensions') if scheme_name and (table_name.endswith("_init") or table_name.endswith("_finalize")) \ and 'horizontal_loop_extent' in dimensions: logging.warn("Legacy extension - replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \ "for variable {} in table {}".format(standard_name,table_name)) dimensions = [ 'horizontal_dimension' if x == 'horizontal_loop_extent' else x for x in dimensions ] elif scheme_name and table_name.endswith( "_run") and 'horizontal_dimension' in dimensions: logging.warn("Legacy extension - replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \ "for variable {} in table {}".format(standard_name,table_name)) dimensions = [ 'horizontal_loop_extent' if x == 'horizontal_dimension' else x for x in dimensions ] # *DH 2020-05-26 if new_var.get_prop_value('active').lower() == '.true.': active = 'T' elif new_var.get_prop_value('active').lower() == '.false.': active = 'F' else: # Replace multiple whitespaces, preserve case active = ' '.join(new_var.get_prop_value('active').split()) var = Var( standard_name=standard_name, long_name=new_var.get_prop_value('long_name') + legacy_note, units=new_var.get_prop_value('units'), local_name=new_var.get_prop_value('local_name'), type=new_var.get_prop_value('type'), dimensions=dimensions, container=container, kind=new_var.get_prop_value('kind'), intent=new_var.get_prop_value('intent'), optional='T' if new_var.get_prop_value('optional') else 'F', active=active, ) # Check for duplicates in same table if standard_name in metadata.keys(): raise Exception( "Error, multiple definitions of standard name {} in new metadata table {}" .format(standard_name, table_name)) metadata[standard_name] = [var] # CCPP property tables are mandatory if not has_property_table: if scheme_name: raise Exception("Metadata file {} for scheme {} does not have a [ccpp-table-properties] section,".format(filename, scheme_name) + \ " or the 'name = ...' attribute in the [ccpp-table-properties] is wrong") else: raise Exception("Metadata file {} for table {} does not have a [ccpp-table-properties] section,".format(filename, table_name) + \ " or the 'name = ...' attribute in the [ccpp-table-properties] is wrong") return (metadata, dependencies)
def parse_scheme_tables(filename): """Parses metadata tables for a physics scheme that requests/requires variables as input arguments. Metadata tables can only describe variables required by a subroutine 'subroutine_name' of scheme 'scheme_name' inside a module 'module_name'. Each variable (standard_name) can exist only once, i.e. each entry (list of variables) in the metadata dictionary contains only one element (variable = instance of class Var defined in mkcap.py). The metadata dictionaries of the individual schemes are merged afterwards (called from ccpp_prebuild.py) using merge_metadata_dicts, where multiple instances of variables are compared for compatibility and collected in a list (entry in the merged metadata dictionary). The merged metadata dictionary of all schemes (which contains only compatible variable instances in the list referred to by standard_name) is then compared to the unique definition in the metadata dictionary of the variables provided by the host model using compare_metadata in ccpp_prebuild.py.""" # Set debug to true if logging level is debug debug = logging.getLogger().getEffectiveLevel() == logging.DEBUG # Valid suffices for physics scheme routines subroutine_suffices = ['init', 'run', 'finalize'] # Final metadata container for all variables in file metadata = collections.OrderedDict() # Registry of modules and derived data types in file #registry = {} registry = collections.OrderedDict() # Argument lists of each subroutine in the file arguments = collections.OrderedDict() # Read all lines of the file at once with (open(filename, 'r')) as file: file_lines = file.readlines() lines = [] original_line_numbers = [] buffer = '' for i in range(len(file_lines)): line = file_lines[i].rstrip('\n').strip() # Skip empty lines if line == '' or line == '&': continue # Remove line continuations: concatenate with following lines if line.endswith('&'): buffer += file_lines[i].rstrip('\n').replace('&', ' ') continue # Write out line with buffer and reset buffer lines.append(buffer + file_lines[i].rstrip('\n').replace('&', ' ')) original_line_numbers.append(i + 1) buffer = '' del file_lines # Find all modules within the file, and save the start and end lines module_lines = {} line_counter = 0 for line in lines: # For the purpose of identifying module constructs, remove any trailing comments from line if '!' in line and not line.startswith('!'): line = line[:line.find('!')] words = line.split() if len(words) > 1 and words[0].lower( ) == 'module' and not words[1].lower() == 'procedure': module_name = words[1].strip() if module_name in registry.keys(): raise Exception( 'Duplicate module name {0}'.format(module_name)) registry[module_name] = {} module_lines[module_name] = {'startline': line_counter} elif len(words) > 1 and words[0].lower() == 'end' and words[1].lower( ) == 'module': try: test_module_name = words[2] except IndexError: logging.warning( 'Warning, encountered closing statement "end module" without module name; assume module_name is {0}' .format(module_name)) test_module_name = module_name if not module_name == test_module_name: raise Exception( 'Module names in opening/closing statement do not match: {0} vs {1}' .format(module_name, test_module_name)) module_lines[module_name]['endline'] = line_counter line_counter += 1 # Parse each module in the file separately for module_name in registry.keys(): startline = module_lines[module_name]['startline'] endline = module_lines[module_name]['endline'] line_counter = 0 in_subroutine = False for line in lines[startline:endline]: # For the purpose of identifying scheme constructs, remove any trailing comments from line if '!' in line and not line.startswith('!'): line = line[:line.find('!')] current_line_number = startline + line_counter words = line.split() for j in range(len(words)): # Check for the word 'subroutine', that it is the first word in the line, # and that a name exists afterwards. Nested subroutines are ignored. if words[j].lower( ) == 'subroutine' and j == 0 and len(words) > 1: if in_subroutine: logging.debug( 'Warning, ignoring nested subroutine in module {0} and subroutine {1}' .format(module_name, subroutine_name)) continue subroutine_name = words[j + 1].split('(')[0].strip() # Consider the last substring separated by a '_' of the subroutine name as a 'postfix' if subroutine_name.find('_') >= 0: subroutine_suffix = subroutine_name.split('_')[-1] if subroutine_suffix in subroutine_suffices: scheme_name = subroutine_name[0:subroutine_name. rfind('_')] if not scheme_name == module_name: raise Exception( 'Scheme name differs from module name: module_name="{0}" vs. scheme_name="{1}"' .format(module_name, scheme_name)) if not scheme_name in registry[module_name].keys(): registry[module_name][scheme_name] = {} if subroutine_name in registry[module_name][ scheme_name].keys(): raise Exception( 'Duplicate subroutine name {0} in module {1}' .format(subroutine_name, module_name)) registry[module_name][scheme_name][ subroutine_name] = [current_line_number] in_subroutine = True elif words[j].lower() == 'subroutine' and j == 1 and words[ j - 1].lower() == 'end': try: test_subroutine_name = words[j + 1] except IndexError: logging.warning('Warning, encountered closing statement "end subroutine" without subroutine name; ' +\ ' assume subroutine_name is {0}'.format(subroutine_name)) test_subroutine_name = subroutine_name if in_subroutine and subroutine_name == test_subroutine_name: in_subroutine = False registry[module_name][scheme_name][ subroutine_name].append(current_line_number) # Avoid problems by enforcing end statements to carry a descriptor (subroutine, module, ...) elif in_subroutine and len( words) == 1 and words[0].lower() == 'end': raise Exception('Encountered closing statement "end" without descriptor (subroutine, module, ...): ' +\ 'line {0}="{1}" in file {2}'.format(original_line_numbers[current_line_number], line, filename)) line_counter += 1 # Check that for each registered subroutine the start and end lines were found for scheme_name in registry[module_name].keys(): for subroutine_name in registry[module_name][scheme_name].keys(): if not len(registry[module_name][scheme_name] [subroutine_name]) == 2: raise Exception( 'Error parsing start and end lines for subroutine {0} in module {1}' .format(subroutine_name, module_name)) logging.debug('Parsing file {0} with registry {1}'.format( filename, registry)) for scheme_name in registry[module_name].keys(): for subroutine_name in registry[module_name][scheme_name].keys(): # Record the order of variables in the call list to each subroutine in a list if not module_name in arguments.keys(): arguments[module_name] = {} if not scheme_name in arguments[module_name].keys(): arguments[module_name][scheme_name] = {} if not subroutine_name in arguments[module_name][ scheme_name].keys(): arguments[module_name][scheme_name][subroutine_name] = [] # Find the argument table corresponding to each subroutine by searching # "upward" from the subroutine definition line for the "arg_table_SubroutineName" section table_found = False header_line_number = None for line_number in range( registry[module_name][scheme_name][subroutine_name][0], -1, -1): line = lines[line_number] words = line.split() for word in words: if (len(words) > 2 and words[0] in ['!!', '!>'] and '\section' in words[1] and 'arg_table_{0}'.format(subroutine_name) in words[2]): table_found = True header_line_number = line_number + 1 table_name = subroutine_name break else: for word in words: if 'arg_table_{0}'.format( subroutine_name) in word: raise Exception( "Malformatted table found in {0} / {1} / {2} / {3}" .format(filename, module_name, scheme_name, subroutine_name)) if table_found: break # If an argument table is found, parse it if table_found: if 'htmlinclude' in lines[header_line_number].lower(): words = lines[header_line_number].split() if words[0] == '!!' and words[ 1] == '\\htmlinclude' and len(words) == 3: new_metadata = True filename_parts = filename.split('.') metadata_filename = '.'.join( filename_parts[0:len(filename_parts) - 1]) + '.meta' this_metadata = read_new_metadata( metadata_filename, module_name, table_name, scheme_name=scheme_name, subroutine_name=subroutine_name) for var_name in this_metadata.keys(): # Add standard_name to argument list for this subroutine arguments[module_name][scheme_name][ subroutine_name].append(var_name) # For all instances of this var (can be only one) in this subroutine's metadata, # add to global metadata and check for compatibility with existing variables for var in this_metadata[var_name]: if not var_name in metadata.keys(): metadata[var_name] = [var] else: for existing_var in metadata[var_name]: if not existing_var.compatible( var): raise Exception('New entry for variable {0}'.format(var_name) + \ ' in argument table of subroutine {0}'.format(subroutine_name) +\ ' is incompatible with existing entry:\n' +\ ' existing: {0}\n'.format(existing_var.print_debug()) +\ ' vs. new: {0}'.format(var.print_debug())) metadata[var_name].append(var) # Next line must denote the end of table, # i.e. look for a line containing only '!!' line_number = header_line_number + 1 nextline = lines[line_number] nextwords = nextline.split() if len(nextwords) == 1 and nextwords[0].strip( ) == '!!': end_of_table = True else: raise Exception( 'Encountered invalid format "{0}" of new metadata table hook in table {1}' .format(line, table_name)) line_number += 1 continue # Separate the table headers table_header = lines[header_line_number].split('|') # Check for blank table if len(table_header) <= 1: logging.debug( 'Skipping blank table {0}'.format(table_name)) table_found = False continue # Extract table header table_header = [x.strip() for x in table_header[1:-1]] # Check that only valid table headers are used for item in table_header: if not item in VALID_ITEMS['header']: raise Exception( 'Invalid column header {0} in argument table {1}' .format(item, table_name)) # Locate mandatory column 'standard_name' try: standard_name_index = table_header.index( 'standard_name') except ValueError: raise Exception( 'Mandatory column standard_name not found in argument table {0}' .format(table_name)) # DH* warn or raise error for old metadata format logging.warn( "Old metadata table found for table {}".format( table_name)) #raise Exception("Old metadata table found for table {}".format(table_name)) # *DH # Get all of the variable information in table end_of_table = False line_number = header_line_number + 2 while not end_of_table: line = lines[line_number] words = line.split('|') if len(words) == 1: if words[0].strip() == '!!': end_of_table = True else: raise Exception( 'Encountered invalid line "{0}" in argument table {1}' .format(line, table_name)) else: var_items = [x.strip() for x in words[1:-1]] if not len(var_items) == len(table_header): raise Exception( 'Error parsing variable entry "{0}" in argument table {1}' .format(var_items, table_name)) var_name = var_items[standard_name_index] # Column standard_name cannot be left blank in scheme_tables if not var_name: raise Exception( 'Encountered line "{0}" without standard name in argument table {1}' .format(line, table_name)) # Enforce CF standards: no dashes, no dots (underscores instead) if "-" in var_name: raise Exception( "Invalid character '-' found in standard name {0} in table {1}" .format(var_name, table_name)) elif "." in var_name: raise Exception( "Invalid character '.' found in standard name {0} in table {1}" .format(var_name, table_name)) # # Add standard_name to argument list for this subroutine arguments[module_name][scheme_name][ subroutine_name].append(var_name) var = Var.from_table(table_header, var_items) # Check for incompatible definitions with CCPP mandatory variables if var_name in CCPP_MANDATORY_VARIABLES.keys( ) and not CCPP_MANDATORY_VARIABLES[ var_name].compatible(var): raise Exception('Entry for variable {0}'.format(var_name) + \ ' in argument table of subroutine {0}'.format(subroutine_name) +\ ' is incompatible with mandatory variable:\n' +\ ' existing: {0}\n'.format(CCPP_MANDATORY_VARIABLES[var_name].print_debug()) +\ ' vs. new: {0}'.format(var.print_debug())) # Record the location of this variable: module, scheme, table container = encode_container( module_name, scheme_name, table_name) var.container = container # Add variable to metadata dictionary if not var_name in metadata.keys(): metadata[var_name] = [var] else: for existing_var in metadata[var_name]: if not existing_var.compatible(var): raise Exception('New entry for variable {0}'.format(var_name) + \ ' in argument table of subroutine {0}'.format(subroutine_name) +\ ' is incompatible with existing entry:\n' +\ ' existing: {0}\n'.format(existing_var.print_debug()) +\ ' vs. new: {0}'.format(var.print_debug())) metadata[var_name].append(var) line_number += 1 # After parsing entire metadata table for the subroutine, check that all mandatory CCPP variables are present for var_name in CCPP_MANDATORY_VARIABLES.keys(): if not var_name in arguments[module_name][scheme_name][ subroutine_name]: raise Exception( 'Mandatory CCPP variable {0} not declared in metadata table of subroutine {1}' .format(var_name, subroutine_name)) # For CCPP-compliant files (i.e. files with metadata tables, perform additional checks) if len(metadata.keys()) > 0: # Check that all subroutine "root" names in the current module are equal to scheme_name # and that there are exactly three subroutines for scheme X: X_init, X_run, X_finalize message = '' abort = False for scheme_name in registry[module_name].keys(): # Pre-generate error message message += 'Check that all subroutines in module {0} have the same root name:\n'.format( module_name) message += ' i.e. scheme_A_init, scheme_A_run, scheme_A_finalize\n' message += 'Here is a list of the subroutine names for scheme {0}:\n'.format( scheme_name) message += '{0}\n\n'.format(', '.join( sorted(registry[module_name][scheme_name].keys()))) if (not len(registry[module_name][scheme_name].keys()) == 3): logging.exception(message) abort = True else: for suffix in subroutine_suffices: subroutine_name = '{0}_{1}'.format(scheme_name, suffix) if not subroutine_name in registry[module_name][ scheme_name].keys(): logging.exception(message) abort = True if abort: raise Exception(message) # Debugging output to screen and to XML if debug and len(metadata.keys()) > 0: # To screen logging.debug('Module name: {0}'.format(module_name)) for scheme_name in registry[module_name].keys(): logging.debug('Scheme name: {0}'.format(scheme_name)) for subroutine_name in registry[module_name][scheme_name].keys( ): container = encode_container(module_name, scheme_name, subroutine_name) vars_in_subroutine = [] for var_name in metadata.keys(): for var in metadata[var_name]: if var.container == container: vars_in_subroutine.append(var_name) logging.debug('Variables in subroutine {0}: {1}'.format( subroutine_name, ', '.join(vars_in_subroutine))) # Standard output to screen elif len(metadata.keys()) > 0: for scheme_name in registry[module_name].keys(): logging.info('Parsed tables in scheme {0}'.format(scheme_name)) # End of loop over all module_names return (metadata, arguments)
def write(self, metadata_request, metadata_define, arguments): # Create an inverse lookup table of local variable names defined (by the host model) and standard names standard_name_by_local_name_define = {} for standard_name in metadata_define.keys(): standard_name_by_local_name_define[metadata_define[standard_name][0].local_name] = standard_name # First get target names of standard CCPP variables for subcycling and error handling ccpp_loop_counter_target_name = metadata_request[CCPP_LOOP_COUNTER][0].target ccpp_error_flag_target_name = metadata_request[CCPP_ERROR_FLAG_VARIABLE][0].target ccpp_error_msg_target_name = metadata_request[CCPP_ERROR_MSG_VARIABLE][0].target # module_use = '' self._module = 'ccpp_{suite}_{name}_cap'.format(name=self._name, suite=self._suite) self._filename = '{module_name}.F90'.format(module_name=self._module) self._subroutines = [] local_subs = '' # for ccpp_stage in CCPP_STAGES: # The special init and finalize routines are only run in that stage if self._init and not ccpp_stage == 'init': continue elif self._finalize and not ccpp_stage == 'finalize': continue # For mapping local variable names to standard names local_vars = {} # For mapping temporary variable names (for unit conversions, etc) to local variable names tmpvar_cnt = 0 tmpvars = {} # body = '' var_defs = '' for subcycle in self._subcycles: if subcycle.loop > 1 and ccpp_stage == 'run': body += ''' associate(cnt => {loop_var_name}) do cnt=1,{loop_cnt}\n\n'''.format(loop_var_name=ccpp_loop_counter_target_name,loop_cnt=subcycle.loop) for scheme_name in subcycle.schemes: # actions_before and actions_after capture operations such # as unit conversions, transformations that have to happen # before and/or after the call to the subroutine (scheme) actions_before = '' actions_after = '' # module_name = scheme_name subroutine_name = scheme_name + '_' + ccpp_stage container = encode_container(module_name, scheme_name, subroutine_name) # Skip entirely empty routines if not arguments[module_name][scheme_name][subroutine_name]: continue error_check = '' args = '' length = 0 # Extract all variables needed (including indices for components/slices of arrays) for var_standard_name in arguments[module_name][scheme_name][subroutine_name]: # Pick the correct variable for this module/scheme/subroutine # from the list of requested variable for var in metadata_request[var_standard_name]: if container == var.container: break if not var_standard_name in local_vars.keys(): if not var_standard_name in metadata_define.keys(): raise Exception('Variable {standard_name} not defined in host model metadata'.format( standard_name=var_standard_name)) var_local_name_define = metadata_define[var_standard_name][0].local_name # Break apart var_local_name_define into the different components (members of DDTs) # to determine all variables that are required (parent_local_name_define, parent_local_names_define_indices) = \ extract_parents_and_indices_from_local_name(var_local_name_define) # Check for each of the derived parent local names as defined by the host model # if they are registered (i.e. if there is a standard name for it). Note that # the output of extract_parents_and_indices_from_local_name is stripped of any # array subset information, i.e. a local name 'Atm(:)%...' will produce a # parent local name 'Atm'. Since the rank of tha parent variable is not known # at this point and since the local name in the host model metadata table could # contain '(:)', '(:,:)', ... (up to the rank of the array), we search for the # maximum number of dimensions allowed by the Fortran standard. for local_name_define in [parent_local_name_define] + parent_local_names_define_indices: parent_standard_name = None parent_var = None for i in xrange(FORTRAN_ARRAY_MAX_DIMS+1): if i==0: dims_string = '' else: # (:) for i==1, (:,:) for i==2, ... dims_string = '(' + ','.join([':' for j in xrange(i)]) + ')' if local_name_define+dims_string in standard_name_by_local_name_define.keys(): parent_standard_name = standard_name_by_local_name_define[local_name_define+dims_string] parent_var = metadata_define[parent_standard_name][0] break if not parent_var: raise Exception('Parent variable {parent} of {child} with standard name '.format( parent=local_name_define, child=var_local_name_define)+\ '{standard_name} not defined in host model metadata'.format( standard_name=var_standard_name)) # Reset local name for entire array to a notation without (:), (:,:), etc.; # this is needed for the var.print_def_intent() routine to work correctly parent_var.local_name = local_name_define # Add variable to dictionary of parent variables, if not already there. # Set or update intent, depending on whether the variable is an index # in var_local_name_define or the actual parent of that variable. if not parent_standard_name in self.parents[ccpp_stage].keys(): self.parents[ccpp_stage][parent_standard_name] = copy.deepcopy(parent_var) # Copy the intent of the actual variable being processed if local_name_define == parent_local_name_define: self.parents[ccpp_stage][parent_standard_name].intent = var.intent # It's an index for the actual variable being processed --> intent(in) else: self.parents[ccpp_stage][parent_standard_name].intent = 'in' elif self.parents[ccpp_stage][parent_standard_name].intent == 'in': # Adjust the intent if the actual variable is not intent(in) if local_name_define == parent_local_name_define and not var.intent == 'in': self.parents[ccpp_stage][parent_standard_name].intent = 'inout' # It's an index for the actual variable being processed, intent is ok #else: # # nothing to do elif self.parents[ccpp_stage][parent_standard_name].intent == 'out': # Adjust the intent if the actual variable is not intent(out) if local_name_define == parent_local_name_define and not var.intent == 'out': self.parents[ccpp_stage][parent_standard_name].intent = 'inout' # Adjust the intent, because the variable is also used as index variable else: self.parents[ccpp_stage][parent_standard_name].intent = 'inout' # Record this information in the local_vars dictionary local_vars[var_standard_name] = { 'name' : metadata_define[var_standard_name][0].local_name, 'kind' : metadata_define[var_standard_name][0].kind, 'parent_standard_name' : parent_standard_name } else: parent_standard_name = local_vars[var_standard_name]['parent_standard_name'] # Update intent information if necessary if self.parents[ccpp_stage][parent_standard_name].intent == 'in' and not var.intent == 'in': self.parents[ccpp_stage][parent_standard_name].intent = 'inout' elif self.parents[ccpp_stage][parent_standard_name].intent == 'out' and not var.intent == 'out': self.parents[ccpp_stage][parent_standard_name].intent = 'inout' # Add necessary actions before/after while populating the subroutine's argument list kind_string = '_' + local_vars[var_standard_name]['kind'] if local_vars[var_standard_name]['kind'] else '' if var.actions['out']: if local_vars[var_standard_name]['name'] in tmpvars.keys(): # If the variable already has a local variable (tmpvar), reuse it tmpvar = tmpvars[local_vars[var_standard_name]['name']] else: # Add a local variable (tmpvar) for this variable tmpvar_cnt += 1 tmpvar = copy.deepcopy(var) tmpvar.local_name = 'tmpvar{0}'.format(tmpvar_cnt) tmpvars[local_vars[var_standard_name]['name']] = tmpvar if var.rank: # Add allocate statement if the variable has a rank > 0 actions_before += ' allocate({t}, source={v})\n'.format(t=tmpvar.local_name, v=local_vars[var_standard_name]['name']) if var.actions['in']: # Add unit conversion before entering the subroutine actions_before += ' {t} = {c}\n'.format(t=tmpvar.local_name, c=var.actions['in'].format(var=local_vars[var_standard_name]['name'], kind=kind_string)) # Add unit conversion after returning from the subroutine actions_after += ' {v} = {c}\n'.format(v=local_vars[var_standard_name]['name'], c=var.actions['out'].format(var=tmpvar.local_name, kind=kind_string)) if var.rank: # Add deallocate statement if the variable has a rank > 0 actions_after += ' deallocate({t})\n'.format(t=tmpvar.local_name) # Add to argument list arg = '{local_name}={var_name},'.format(local_name=var.local_name, var_name=tmpvar.local_name) elif var.actions['in']: # Add to argument list, call action in argument list action = var.actions['in'].format(var=local_vars[var_standard_name]['name'], kind=kind_string) arg = '{local_name}={action},'.format(local_name=var.local_name, action=action) else: # Add to argument list arg = '{local_name}={var_name},'.format(local_name=var.local_name, var_name=local_vars[var_standard_name]['name']) args += arg length += len(arg) # Split args so that lines don't exceed 260 characters (for PGI) if length > 70 and not var_standard_name == arguments[module_name][scheme_name][subroutine_name][-1]: args += ' &\n ' length = 0 args = args.rstrip(',') subroutine_call = ''' {actions_before} call {subroutine_name}({args}) {actions_after} '''.format(subroutine_name=subroutine_name, args=args, actions_before=actions_before.rstrip('\n'), actions_after=actions_after.rstrip('\n')) error_check = '''if ({target_name_flag}/=0) then write({target_name_msg},'(a)') "An error occured in {subroutine_name}" ierr={target_name_flag} return end if '''.format(target_name_flag=ccpp_error_flag_target_name, target_name_msg=ccpp_error_msg_target_name, subroutine_name=subroutine_name) body += ''' {subroutine_call} {error_check} '''.format(subroutine_call=subroutine_call, error_check=error_check) module_use += ' use {m}, only: {s}\n'.format(m=module_name, s=subroutine_name) if subcycle.loop > 1 and ccpp_stage == 'run': body += ''' end do end associate ''' # Get list of arguments, module use statement and variable definitions for this subroutine (=stage for the group) (self.arguments[ccpp_stage], sub_module_use, sub_var_defs) = create_arguments_module_use_var_defs( self.parents[ccpp_stage], metadata_define, tmpvars.values()) sub_argument_list = create_argument_list_wrapped(self.arguments[ccpp_stage]) subroutine = self._suite + '_' + self._name + '_' + ccpp_stage + '_cap' self._subroutines.append(subroutine) # Test and set blocks for initialization status initialized_test_block = Group.initialized_test_blocks[ccpp_stage].format( target_name_flag=ccpp_error_flag_target_name, target_name_msg=ccpp_error_msg_target_name, name=self._name) initialized_set_block = Group.initialized_set_blocks[ccpp_stage].format( target_name_flag=ccpp_error_flag_target_name, target_name_msg=ccpp_error_msg_target_name, name=self._name) # Create subroutine local_subs += Group.sub.format(subroutine=subroutine, argument_list=sub_argument_list, module_use='\n '.join(sub_module_use), initialized_test_block=initialized_test_block, initialized_set_block=initialized_set_block, var_defs='\n '.join(sub_var_defs), body=body) # Write output to stdout or file if (self.filename is not sys.stdout): f = open(self.filename, 'w') else: f = sys.stdout f.write(Group.header.format(group=self._name, module=self._module, module_use=module_use, subroutines=', &\n '.join(self._subroutines))) f.write(local_subs) f.write(Group.footer.format(module=self._module)) if (f is not sys.stdout): f.close() return
def parse_variable_tables(filename): """Parses metadata tables on the host model side that define the available variables. Metadata tables can refer to variables inside a module or as part of a derived datatype, which itself is defined inside a module (depending on the location of the metadata table). Each variable (standard_name) can exist only once, i.e. each entry (list of variables) in the metadata dictionary contains only one element (variable = instance of class Var defined in mkcap.py)""" # Set debug to true if logging level is debug debug = logging.getLogger().getEffectiveLevel() == logging.DEBUG # Final metadata container for all variables in file metadata = collections.OrderedDict() # Registry of modules and derived data types in file registry = collections.OrderedDict() # Read all lines of the file at once with (open(filename, 'r')) as file: file_lines = file.readlines() lines = [] buffer = '' for i in xrange(len(file_lines)): line = file_lines[i].rstrip('\n').strip() # Skip empty lines if line == '' or line == '&': continue # Remove line continuations: concatenate with following lines if line.endswith('&'): buffer += file_lines[i].rstrip('\n').replace('&', ' ') continue # Write out line with buffer and reset buffer lines.append(buffer + file_lines[i].rstrip('\n').replace('&', ' ')) buffer = '' del file_lines # Find all modules within the file, and save the start and end lines module_lines = {} line_counter = 0 for line in lines: words = line.split() if len(words) > 1 and words[0].lower() == 'module' and not words[1].lower() == 'procedure': module_name = words[1].strip() if module_name in registry.keys(): raise Exception('Duplicate module name {0}'.format(module_name)) registry[module_name] = {} module_lines[module_name] = { 'startline' : line_counter } elif len(words) > 1 and words[0].lower() == 'end' and words[1].lower() == 'module': try: test_module_name = words[2] except IndexError: logging.warning('Encountered closing statement "end module" without module name; assume module_name is {0}'.format(module_name)) test_module_name = module_name if not module_name == test_module_name: raise Exception('Module names in opening/closing statement do not match: {0} vs {1}'.format(module_name, test_module_name)) module_lines[module_name]['endline'] = line_counter line_counter += 1 # Parse each module in the file separately for module_name in registry.keys(): startline = module_lines[module_name]['startline'] endline = module_lines[module_name]['endline'] line_counter = 0 in_type = False for line in lines[startline:endline]: current_line_number = startline + line_counter words = line.split() for j in range(len(words)): # Check for the word 'type', that it is the first word in the line, # and that a name exists afterwards. It is assumed that definitions # (not usage) of derived types cannot be nested - reasonable for Fortran. if words[j].lower() == 'type' and j == 0 and len(words) > 1 and not '(' in words[j+1]: if in_type: raise Exception('Nested definitions of derived types not supported') in_type = True type_name = words[j+1].split('(')[0].strip() if type_name in registry[module_name].keys(): raise Exception('Duplicate derived type name {0} in module {1}'.format( type_name, module_name)) registry[module_name][type_name] = [current_line_number] elif words[j].lower() == 'type' and j == 1 and words[j-1].lower() == 'end': if not in_type: raise Exception('Encountered "end_type" without corresponding "type" statement') try: test_type_name = words[j+1] except IndexError: logging.warning('Encountered closing statement "end type" without type name; assume type_name is {0}'.format(type_name)) test_type_name = type_name if not type_name == test_type_name: raise Exception('Type names in opening/closing statement do not match: {0} vs {1}'.format(type_name, test_type_name)) in_type = False registry[module_name][type_name].append(current_line_number) line_counter += 1 logging.debug('Parsing file {0} with registry {1}'.format(filename, registry)) # Variables can either be defined at module-level or in derived types - alongside with their tables line_counter = 0 in_table = False in_type = False for line in lines[startline:endline]: current_line_number = startline + line_counter # Check for beginning of new table words = line.split() # This is case sensitive if len(words) > 2 and words[0] in ['!!', '!>'] and '\section' in words[1] and 'arg_table_' in words[2]: if in_table: raise Exception('Encountered table start for table {0} while still in table {1}'.format(words[2].replace('arg_table_',''), table_name)) table_name = words[2].replace('arg_table_','') if not (table_name == module_name or table_name in registry[module_name].keys()): raise Exception('Encountered table with name {0} without corresponding module or type name'.format(table_name)) in_table = True header_line_number = current_line_number + 1 line_counter += 1 continue elif (words[0].startswith('!!') or words[0].startswith('!>')) and '\section' in words[0]: raise Exception("Malformatted table found in {0} / {1} / {2}".format(filename, module_name, table_name)) # If an argument table is found, parse it if in_table: words = line.split('|') # Separate the table headers if current_line_number == header_line_number: # Check for blank table if len(words) <= 1: logging.debug('Skipping blank table {0}'.format(table_name)) in_table = False line_counter += 1 continue table_header = [x.strip() for x in words[1:-1]] # Check that only valid table headers are used for item in table_header: if not item in VALID_ITEMS['header']: raise Exception('Invalid column header {0} in argument table {1}'.format(item, table_name)) # Locate mandatory column 'standard_name' try: standard_name_index = table_header.index('standard_name') except ValueError: raise Exception('Mandatory column standard_name not found in argument table {0}'.format(table_name)) line_counter += 1 continue elif current_line_number == header_line_number + 1: # Skip over separator line line_counter += 1 continue else: if len(words) == 1: # End of table if words[0].strip() == '!!': in_table = False line_counter += 1 continue else: raise Exception('Encountered invalid line "{0}" in argument table {1}'.format(line, table_name)) else: var_items = [x.strip() for x in words[1:-1]] if not len(var_items) == len(table_header): raise Exception('Error parsing variable entry "{0}" in argument table {1}'.format(var_items, table_name)) var_name = var_items[standard_name_index] # Skip variables without a standard_name (i.e. empty cell in column standard_name) if var_name: var = Var.from_table(table_header,var_items) if table_name == module_name: container = encode_container(module_name) else: container = encode_container(module_name, table_name) var.container = container # Check for incompatible definitions with CCPP mandatory variables if var_name in CCPP_MANDATORY_VARIABLES.keys() and not CCPP_MANDATORY_VARIABLES[var_name].compatible(var): raise Exception('Entry for variable {0}'.format(var_name) + \ ' in argument table {0}'.format(table_name) +\ ' is incompatible with mandatory variable:\n' +\ ' existing: {0}\n'.format(CCPP_MANDATORY_VARIABLES[var_name].print_debug()) +\ ' vs. new: {0}'.format(var.print_debug())) # Add variable to metadata dictionary if not var_name in metadata.keys(): metadata[var_name] = [var] else: for existing_var in metadata[var_name]: if not existing_var.compatible(var): raise Exception('New entry for variable {0}'.format(var_name) + \ ' in argument table {0}'.format(table_name) +\ ' is incompatible with existing entry:\n' +\ ' existing: {0}\n'.format(existing_var.print_debug()) +\ ' vs. new: {0}'.format(var.print_debug())) metadata[var_name].append(var) #else: # logging.debug('Skipping variable entry "{0}" without a standard_name'.format(var_items)) line_counter += 1 # Informative output to screen if debug and len(metadata.keys()) > 0: for module_name in registry.keys(): logging.debug('Module name: {0}'.format(module_name)) container = encode_container(module_name) vars_in_module = [] for var_name in metadata.keys(): for var in metadata[var_name]: if var.container == container: vars_in_module.append(var_name) logging.debug('Module variables: {0}'.format(', '.join(vars_in_module))) for type_name in registry[module_name].keys(): container = encode_container(module_name, type_name) vars_in_type = [] for var_name in metadata.keys(): for var in metadata[var_name]: if var.container == container: vars_in_type.append(var_name) logging.debug('Variables in derived type {0}: {1}'.format(type_name, ', '.join(vars_in_type))) if debug and len(metadata.keys()) > 0: # Write out the XML for debugging purposes top = ET.Element('definition') top.set('module', module_name) container = encode_container(module_name) for var_name in metadata.keys(): for var in metadata[var_name]: if var.container == container: sub_var = var.to_xml(ET.SubElement(top, 'variable')) for type_name in registry[module_name].keys(): container = encode_container(module_name, type_name) sub_type = ET.SubElement(top, 'type') sub_type.set('type_name', type_name) for var_name in metadata.keys(): for var in metadata[var_name]: if var.container == container: sub_var = var.to_xml(ET.SubElement(sub_type, 'variable')) indent(top) tree = ET.ElementTree(top) xmlfile = module_name + '.xml' tree.write(xmlfile, xml_declaration=True, encoding='utf-8', method="xml") logging.info('Parsed variable definition tables in module {0}; output => {1}'.format(module_name, xmlfile)) elif len(metadata.keys()) > 0: logging.info('Parsed variable definition tables in module {0}'.format(module_name)) return metadata
def read_new_metadata(filename, module_name, table_name, scheme_name=None, subroutine_name=None): """Read metadata in new format and convert output to ccpp_prebuild metadata dictionary""" if not os.path.isfile(filename): raise Exception("New metadata file {0} not found".format(filename)) # Save metadata, because this routine new_metadata # is called once for every table in that file if filename in NEW_METADATA_SAVE.keys(): new_metadata_headers = NEW_METADATA_SAVE[filename] else: new_metadata_headers = MetadataHeader.parse_metadata_file(filename) NEW_METADATA_SAVE[filename] = new_metadata_headers # Convert new metadata for requested table to old metadata dictionary metadata = collections.OrderedDict() for new_metadata_header in new_metadata_headers: if not scheme_name: if not new_metadata_header.title == table_name: # Skip this table, since it is not requested right now continue if new_metadata_header.title == module_name: container = encode_container(module_name) else: container = encode_container(module_name, new_metadata_header.title) else: if not new_metadata_header.title == table_name: # Skip this table, since it is not requested right now continue container = encode_container(module_name, scheme_name, table_name) for new_var in new_metadata_header.variable_list(): standard_name = new_var.get_prop_value('standard_name') # DH* 2020-05-26 # Legacy extension for inconsistent metadata (use of horizontal_dimension versus horizontal_loop_extent). # Since horizontal_dimension and horizontal_loop_extent have the same attributes (otherwise it doesn't # make sense), we swap the standard name and add a note to the long name legacy_note = '' if standard_name == 'horizontal_loop_extent' and scheme_name and \ (table_name.endswith("_init") or table_name.endswith("finalize")): logging.warn("Legacy extension - replacing variable 'horizontal_loop_extent'" + \ " with 'horizontal_dimension' in table {}".format(table_name)) standard_name = 'horizontal_dimension' legacy_note = ' replaced by horizontal dimension (legacy extension)' elif standard_name == 'horizontal_dimension' and scheme_name and table_name.endswith( "_run"): logging.warn("Legacy extension - replacing variable 'horizontal_dimension' " + \ "with 'horizontal_loop_extent' in table {}".format(table_name)) standard_name = 'horizontal_loop_extent' legacy_note = ' replaced by horizontal loop extent (legacy extension)' # Adjust dimensions dimensions = new_var.get_prop_value('dimensions') if scheme_name and (table_name.endswith("_init") or table_name.endswith("finalize")) \ and 'horizontal_loop_extent' in dimensions: logging.warn("Legacy extension - replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \ "for variable {} in table {}".format(standard_name,table_name)) dimensions = [ 'horizontal_dimension' if x == 'horizontal_loop_extent' else x for x in dimensions ] elif scheme_name and table_name.endswith( "_run") and 'horizontal_dimension' in dimensions: logging.warn("Legacy extension - replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \ "for variable {} in table {}".format(standard_name,table_name)) dimensions = [ 'horizontal_loop_extent' if x == 'horizontal_dimension' else x for x in dimensions ] # *DH 2020-05-26 if new_var.get_prop_value('active').lower() == '.true.': active = 'T' elif new_var.get_prop_value('active').lower() == '.false.': active = 'F' else: # Replace multiple whitespaces and use lowercase throughout active = ' '.join( new_var.get_prop_value('active').lower().split()) var = Var( standard_name=standard_name, long_name=new_var.get_prop_value('long_name') + legacy_note, units=new_var.get_prop_value('units'), local_name=new_var.get_prop_value('local_name'), type=new_var.get_prop_value('type'), dimensions=dimensions, container=container, kind=new_var.get_prop_value('kind'), intent=new_var.get_prop_value('intent'), optional='T' if new_var.get_prop_value('optional') else 'F', active=active, ) # Check for duplicates in same table if standard_name in metadata.keys(): raise Exception( "Error, multiple definitions of standard name {0} in new metadata table {1}" .format(standard_name, table_name)) metadata[standard_name] = [var] return metadata
def write(self, metadata_request, metadata_define, arguments, ccpp_field_maps, module_use_cap): # First get target names of standard CCPP variables for subcycling and error handling ccpp_loop_counter_target_name = metadata_request[CCPP_LOOP_COUNTER][ 0].target ccpp_error_flag_target_name = metadata_request[ CCPP_ERROR_FLAG_VARIABLE][0].target ccpp_error_msg_target_name = metadata_request[CCPP_ERROR_MSG_VARIABLE][ 0].target # module_use = module_use_cap self._module = 'ccpp_group_{name}_cap'.format(name=self._name) self._filename = '{module_name}.F90'.format(module_name=self._module) self._subroutines = [] local_subs = '' # for ccpp_stage in CCPP_STAGES: if self._init and not ccpp_stage == 'init': continue elif self._finalize and not ccpp_stage == 'finalize': continue # For creating and mappig local variable names to standard names local_vars = {} local_var_cnt = 0 local_var_cnt_max = 9999 local_var_template = 'v{x:04d}' # Add the predefined variables for subcycling and error handling (special, no need to retrieve from cdata via ccpp_field_get) local_vars[CCPP_LOOP_COUNTER] = ccpp_loop_counter_target_name local_vars[CCPP_ERROR_FLAG_VARIABLE] = ccpp_error_flag_target_name local_vars[CCPP_ERROR_MSG_VARIABLE] = ccpp_error_msg_target_name # body = '' var_defs = '' var_gets = '' for subcycle in self._subcycles: if subcycle.loop > 1 and ccpp_stage == 'run': body += ''' associate(cnt => {loop_var_name}) do cnt=1,{loop_cnt}\n\n'''.format( loop_var_name=ccpp_loop_counter_target_name, loop_cnt=subcycle.loop) for scheme_name in subcycle.schemes: module_name = scheme_name subroutine_name = scheme_name + '_' + ccpp_stage container = encode_container(module_name, scheme_name, subroutine_name) # Skip entirely empty routines if not arguments[module_name][scheme_name][subroutine_name]: continue error_check = '' args = '' length = 0 for var_name in arguments[module_name][scheme_name][ subroutine_name]: for var in metadata_request[var_name]: if container == var.container: break if not var_name in local_vars: local_var_cnt += 1 if local_var_cnt > local_var_cnt_max: raise Exception( "local_var_cnt exceeding local_var_cnt_max, increase limit!" ) tmpvar = copy.deepcopy(var) tmpvar.local_name = local_var_template.format( x=local_var_cnt) #var_name.replace('-','_') var_defs += ' ' + tmpvar.print_def() + '\n' # Use the index lookup from ccpp_field_maps for the group's # physics set and given variable name (standard name) var_gets += tmpvar.print_get(index=ccpp_field_maps[ self._pset][var_name]) + '\n' # Add to list of local variables with the auto-generated local variable name local_vars[var_name] = tmpvar.local_name del tmpvar # Add to argument list arg = '{local_name}={var_name},'.format( local_name=var.local_name, var_name=local_vars[var_name] ) #var_name.replace('-','_')) args += arg length += len(arg) # Split args so that lines don't exceed 260 characters (for PGI) if length > 70 and not var_name == arguments[ module_name][scheme_name][subroutine_name][-1]: args += ' &\n ' length = 0 args = args.rstrip(',') subroutine_call = 'call {subroutine_name}({args})'.format( subroutine_name=subroutine_name, args=args) error_check = '''if ({target_name_flag}/=0) then write({target_name_msg},'(a)') "An error occured in {subroutine_name}" ierr={target_name_flag} return end if '''.format(target_name_flag=ccpp_error_flag_target_name, target_name_msg=ccpp_error_msg_target_name, subroutine_name=subroutine_name) body += ''' {subroutine_call} {error_check} '''.format(subroutine_call=subroutine_call, error_check=error_check) module_use += ' use {m}, only: {s}\n'.format( m=module_name, s=subroutine_name) if subcycle.loop > 1 and ccpp_stage == 'run': body += ''' end do end associate ''' subroutine = self._name + '_' + ccpp_stage + '_cap' self._subroutines.append(subroutine) # Test and set blocks for initialization status initialized_test_block = Group.initialized_test_blocks[ ccpp_stage].format( target_name_flag=ccpp_error_flag_target_name, target_name_msg=ccpp_error_msg_target_name, name=self._name) initialized_set_block = Group.initialized_set_blocks[ ccpp_stage].format( target_name_flag=ccpp_error_flag_target_name, target_name_msg=ccpp_error_msg_target_name, name=self._name) # Create subroutine local_subs += Group.sub.format( subroutine=subroutine, initialized_test_block=initialized_test_block, initialized_set_block=initialized_set_block, var_defs=var_defs, var_gets=var_gets, body=body) # Write output to stdout or file if (self.filename is not sys.stdout): f = open(self.filename, 'w') else: f = sys.stdout f.write( Group.header.format(group=self._name, module=self._module, module_use=module_use, subroutines=','.join(self._subroutines))) f.write(local_subs) f.write(Group.footer.format(module=self._module)) if (f is not sys.stdout): f.close() return
def read_new_metadata(filename, module_name, table_name, scheme_name=None, subroutine_name=None): """Read metadata in new format and convert output to ccpp_prebuild metadata dictionary""" if not os.path.isfile(filename): raise Exception("New metadata file {0} not found".format(filename)) # Save metadata, because this routine new_metadata # is called once for every table in that file if filename in NEW_METADATA_SAVE.keys(): new_metadata_headers = NEW_METADATA_SAVE[filename] else: new_metadata_headers = parse_metadata_file( filename, known_ddts=registered_fortran_ddt_names(), logger=logging.getLogger(__name__)) NEW_METADATA_SAVE[filename] = new_metadata_headers # Record dependencies for the metadata table (only applies to schemes) dependencies = [] # Convert new metadata for requested table to old metadata dictionary metadata = collections.OrderedDict() for new_metadata_header in new_metadata_headers: for metadata_section in new_metadata_header.sections(): # Module or DDT tables if not scheme_name: # Module property tables if not metadata_section.title == table_name: # Skip this table, since it is not requested right now continue # Distinguish between module argument tables and DDT argument tables if metadata_section.title == module_name: container = encode_container(module_name) else: container = encode_container(module_name, metadata_section.title) # Add to dependencies if new_metadata_header.relative_path: dependencies += [ os.path.join(new_metadata_header.relative_path, x) for x in new_metadata_header.dependencies ] else: dependencies += new_metadata_header.dependencies else: # Scheme property tables if not metadata_section.title == table_name: # Skip this table, since it is not requested right now continue container = encode_container(module_name, scheme_name, table_name) # Add to dependencies if new_metadata_header.relative_path: dependencies += [ os.path.join(new_metadata_header.relative_path, x) for x in new_metadata_header.dependencies ] else: dependencies += new_metadata_header.dependencies for new_var in metadata_section.variable_list(): standard_name = new_var.get_prop_value('standard_name') # DH* 2020-05-26 # Legacy extension for inconsistent metadata (use of horizontal_dimension versus horizontal_loop_extent). # Since horizontal_dimension and horizontal_loop_extent have the same attributes (otherwise it doesn't # make sense), we swap the standard name and add a note to the long name - 2021-05-26: this is now an error. legacy_note = '' if standard_name == 'horizontal_loop_extent' and scheme_name and \ (table_name.endswith("_init") or table_name.endswith("_finalize")): #logging.warn("Legacy extension - replacing variable 'horizontal_loop_extent'" + \ # " with 'horizontal_dimension' in table {}".format(table_name)) #standard_name = 'horizontal_dimension' #legacy_note = ' replaced by horizontal dimension (legacy extension)' raise Exception("Legacy extension DISABLED: replacing variable 'horizontal_loop_extent'" + \ " with 'horizontal_dimension' in table {}".format(table_name)) elif standard_name == 'horizontal_dimension' and scheme_name and table_name.endswith( "_run"): #logging.warn("Legacy extension - replacing variable 'horizontal_dimension'" + \ # " with 'horizontal_loop_extent' in table {}".format(table_name)) #standard_name = 'horizontal_loop_extent' #legacy_note = ' replaced by horizontal loop extent (legacy extension)' raise Exception("Legacy extension DISABLED: replacing variable 'horizontal_dimension'" + \ " with 'horizontal_loop_extent' in table {}".format(table_name)) # Adjust dimensions dimensions = new_var.get_prop_value('dimensions') if scheme_name and (table_name.endswith("_init") or table_name.endswith("_finalize")) \ and 'horizontal_loop_extent' in dimensions: #logging.warn("Legacy extension - replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \ # "for variable {} in table {}".format(standard_name,table_name)) #dimensions = ['horizontal_dimension' if x=='horizontal_loop_extent' else x for x in dimensions] raise Exception("Legacy extension DISABLED: replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \ "for variable {} in table {}".format(standard_name,table_name)) elif scheme_name and table_name.endswith( "_run") and 'horizontal_dimension' in dimensions: #logging.warn("Legacy extension - replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \ # "for variable {} in table {}".format(standard_name,table_name)) #dimensions = ['horizontal_loop_extent' if x=='horizontal_dimension' else x for x in dimensions] raise Exception("Legacy extension DISABLED: replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \ "for variable {} in table {}".format(standard_name,table_name)) elif not scheme_name and 'horizontal_dimension' in dimensions: raise Exception("Legacy extension DISABLED: replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \ "for variable {} in table {}".format(standard_name,table_name)) # *DH 2020-05-26 if not new_var.get_prop_value('active'): # If it doesn't have an active attribute, then the variable is always active (default) active = 'T' elif new_var.get_prop_value('active').lower() == '.true.': active = 'T' elif new_var.get_prop_value( 'active') and new_var.get_prop_value( 'active').lower() == '.false.': active = 'F' else: # Replace multiple whitespaces, preserve case active = ' '.join(new_var.get_prop_value('active').split()) # DH* 20210812 # Workaround for Fortran DDTs incorrectly having the type of # the DDT copied into the kind attribute in parse_metadata_file if new_var.is_ddt() and new_var.get_prop_value('kind'): kind = '' else: kind = new_var.get_prop_value('kind') #kind = new_var.get_prop_value('kind') # *DH 20210812 var = Var( standard_name=standard_name, long_name=new_var.get_prop_value('long_name') + legacy_note, units=new_var.get_prop_value('units'), local_name=new_var.get_prop_value('local_name'), type=new_var.get_prop_value('type').lower(), dimensions=dimensions, container=container, kind=kind, intent=new_var.get_prop_value('intent'), optional='T' if new_var.get_prop_value('optional') else 'F', active=active, ) # Check for duplicates in same table if standard_name in metadata.keys(): raise Exception( "Error, multiple definitions of standard name {} in new metadata table {}" .format(standard_name, table_name)) metadata[standard_name] = [var] return (metadata, dependencies)