def parse_host_model_files(host_filenames, preproc_defs, host_name, logger):
###############################################################################
"""
Gather information from host files (e.g., DDTs, registry) and
return a host model object with the information.
"""
header_dict = {}
table_dict = {}
known_ddts = list()
for filename in host_filenames:
logger.info('Reading host model data from {}'.format(filename))
# parse metadata file
mtables = parse_metadata_file(filename, known_ddts, logger)
fort_file = find_associated_fortran_file(filename)
ftables = parse_fortran_file(fort_file,
preproc_defs=preproc_defs,
logger=logger)
# Check Fortran against metadata (will raise an exception on error)
mheaders = list()
for sect in [x.sections() for x in mtables]:
mheaders.extend(sect)
# end for
fheaders = list()
for sect in [x.sections() for x in ftables]:
fheaders.extend(sect)
# end for
check_fortran_against_metadata(mheaders, fheaders, filename, fort_file,
logger)
# Check for duplicate tables, then add to dict
for table in mtables:
if table.table_name in table_dict:
duplicate_item_error(table.table_name, filename,
table.table_type,
table_dict[header.title])
else:
table_dict[table.table_name] = table
# end if
# end for
# Check for duplicate headers, then add to dict
for header in mheaders:
if header.title in header_dict:
duplicate_item_error(header.title, filename,
header.header_type,
header_dict[header.title])
else:
header_dict[header.title] = header
if header.header_type == 'ddt':
known_ddts.append(header.title)
# end if
# end for
# end for
if not host_name:
host_name = None
# end if
host_model = HostModel(table_dict, host_name, logger)
return host_model
def parse_scheme_files(scheme_filenames, preproc_defs, logger):
###############################################################################
"""
Gather information from scheme files (e.g., init, run, and finalize
methods) and return resulting dictionary.
"""
table_dict = {} # Duplicate check and for dependencies processing
header_dict = {} # To check for duplicates
known_ddts = list()
for filename in scheme_filenames:
logger.info('Reading CCPP schemes from {}'.format(filename))
# parse metadata file
mtables = parse_metadata_file(filename, known_ddts, logger)
fort_file = find_associated_fortran_file(filename)
ftables = parse_fortran_file(fort_file,
preproc_defs=preproc_defs,
logger=logger)
# Check Fortran against metadata (will raise an exception on error)
mheaders = list()
for sect in [x.sections() for x in mtables]:
mheaders.extend(sect)
# end for
fheaders = list()
for sect in [x.sections() for x in ftables]:
fheaders.extend(sect)
# end for
check_fortran_against_metadata(mheaders, fheaders, filename, fort_file,
logger)
# Check for duplicate tables, then add to dict
for table in mtables:
if table.table_name in table_dict:
duplicate_item_error(table.table_name, filename,
table.table_type,
table_dict[header.title])
else:
table_dict[table.table_name] = table
# end if
# end for
# Check for duplicate headers, then add to dict
for header in mheaders:
if header.title in header_dict:
duplicate_item_error(header.title, filename,
header.header_type,
header_dict[header.title])
else:
header_dict[header.title] = header
if header.header_type == 'ddt':
known_ddts.append(header.title)
# end if
# end if
# end for
# end for
return header_dict.values(), table_dict
def parse_fortran_files(filenames, run_env, output_dir, sep, logger):
###############################################################################
"""
Parse each file in <filenames> and produce a prototype metadata file
with a metadata table for each arg_table entry in the file.
"""
meta_filenames = list()
for filename in filenames:
logger.info('Looking for arg_tables from {}'.format(filename))
reset_standard_name_counter()
ftables = parse_fortran_file(filename, run_env)
# Create metadata filename
filepath = '.'.join(os.path.basename(filename).split('.')[0:-1])
fname = filepath + '.meta'
mfilename = os.path.join(output_dir, fname)
write_metadata_file(mfilename, ftables, sep)
meta_filenames.append(mfilename)
return meta_filenames
def convert_file(filename_in, filename_out, metadata_filename_out, logger=None):
"""Convert a file's old metadata to the new format
Note that only the bare minimum error checking is done.
"""
current_module = None
# First, suck in the old file
do_convert = True
if not os.path.exists(filename_in):
raise IOError("convert_file: file, '{}', does not exist".format(filename_in))
# End if
if os.path.exists(filename_out):
raise IOError("convert_file: file, '{}', already exists".format(filename_out))
# End if
# End if
# Read all lines of the file at once
with open(filename_in, 'r') as file:
fin_lines = file.readlines()
for index in xrange(len(fin_lines)):
fin_lines[index] = fin_lines[index].rstrip('\n')
# End for
# End with
max_line = len(fin_lines) - 1
mdconfig = list()
in_preamble = True
in_type = False
with open(filename_out, 'w') as file:
line, lindex = next_line(fin_lines, max_line)
while line is not None:
# Check for a module line
current_module = parse_module_line(line, current_module)
# Maintain a status of being in a DDT definition
if (not in_type) and type_re.match(line):
in_type = True
elif in_type and end_type_re.match(line):
in_type = False
# End if
# Check for end of preamble
if (not in_type) and (line.lstrip()[0:8].lower() == 'contains'):
in_preamble = False
# End if
# Check for beginning of new table
words = line.split()
# This is case sensitive
if len(words) > 2 and words[0] in ['!!', '!>'] and '\section' in words[1] and 'arg_table_' in words[2]:
# We have a new table, parse the header
table_name = words[2].replace('arg_table_','')
##XXgoldyXX: Uncomment this after conversion is over
# logger.info('Found old metadata table, {}, on line {}'.format(table_name, lindex+1))
# The header line is not modified
file.write(line+"\n")
# Write an include line for the metadata table
file.write('!! \htmlinclude {}.html\n'.format(table_name))
# Create the table start section
mdtable = MetadataTable(table_name, current_module)
mdconfig.append(mdtable)
line, lindex = next_line(fin_lines, max_line, cindex=lindex)
words = line.split('|')
header_locs = {}
dim_names = [__not_found__]*15
# Do not work on a blank table
if len(words) > 1:
table_header = [x.strip() for x in words[1:-1]]
for ind in xrange(len(table_header)):
header_locs[table_header[ind]] = ind
# End for
# Find the local_name index (exception if not found)
local_name_ind = header_locs['local_name']
# The table header line is not output
line, lindex = next_line(fin_lines, max_line, cindex=lindex)
# This is the delimiter line, do not output
# Parse the entries
while len(words) > 1:
line, lindex = next_line(fin_lines, max_line, cindex=lindex)
words = line.split('|')
if len(words) <= 1:
# End of table, just write and continue
file.write(line+'\n')
continue
# End if
entries = [x.strip() for x in words[1:-1]]
# Okay, one check
if len(entries) != len(header_locs):
raise ValueError("Malformed table entry")
# End if
# First output the local name
var_name = entries[local_name_ind]
# Strip old-style DDT references
var_name = var_name[var_name.rfind('%')+1:]
mdobj = MetadataEntry(var_name)
mdtable[var_name] = mdobj
# Now, create the rest of the entries
for ind in xrange(len(entries)):
attr_name = table_header[ind]
entry = entries[ind]
if attr_name == 'local_name':
# Already handled this
continue
elif attr_name == 'rank':
attr_name = 'dimensions'
rank = int(entry)
entry = '(' + ','.join(dim_names[0:rank]) + ')'
elif attr_name == 'standard_name':
# The standard name needs to be lowercase
std_name = entry.lower()
# Standard names cannot have dashes
std_name = std_name.replace('-', '_')
# Standard names cannot have periods
std_name = std_name.replace('.', '_')
entries[ind] = std_name
entry = std_name
elif attr_name == 'intent':
if in_preamble:
entry = ''
elif entry.lower() == 'none':
if logger is None:
raise ValueError("{} has intent = none in {}".format(var_name, table_name))
else:
logger.warning("{} has intent = none in {}".format(var_name, table_name))
# End if
# End if
# No else needed
# End if
# Add attribute
if (len(entry) > 0) or (attr_name in required_attrs):
mdobj[attr_name] = entry
# End if
# End for (done with entry)
# End while (done with table)
else:
# Just write the line (should be a table ending)
if line.strip() != '!!':
raise ValueError("All tables must end with !! line")
# End if
file.write(line+'\n')
# End if (blank table)
else:
# Not a table, just write and continue
file.write(line+'\n')
# End if
# Always load a new line
line, lindex = next_line(fin_lines, max_line, cindex=lindex)
# End while
# End with (file)
# Read in the Fortran source to find dimension information
mheaders = parse_fortran_file(filename_out, preproc_defs={'CCPP':1, 'HYBRID':1}, logger=logger)
# Find and replace dimension information for each metadata header
for table in mdconfig:
# Find matching Fortran header
mheader = None
for mh in mheaders:
if mh.title == table.name:
mheader = mh
break
# End if
# End for
if mheader is None:
if logger is not None:
logger.warning('WARNING: Cannot find {} in {} for dimension translation'.format(table.name, filename_out))
# End if
continue # Skip this table
# End for
# The Fortran table does not have valid standard names so
# organize them by local name.
header_vars = {}
for var in mheader.variable_list():
header_vars[var.get_prop_value('local_name').lower()] = var
# End for
for lname in table.keys():
var = table[lname]
if lname.lower() in header_vars:
fvar = header_vars[lname.lower()]
else:
if logger is not None:
logger.warning('WARNING: Cannot find variable {} in {} ({}), skipping dimension translation'.format(lname, table.name, filename_out))
# End if
fvar = None
# End if
dimstring = var['dimensions'].strip().lstrip('(').rstrip(')')
if ':' in dimstring: # i.e., this not a scalar variable
dims = dimstring.split(',')
rank = len(dims)
if fvar is None:
fdims = None
else:
fdims = fvar.get_dimensions()
# End if
if fdims is None:
# We are dealing with an invalid variable, deal with it
dims = [__not_found__]*rank
else:
dims = list()
for fdim in fdims:
fdlist = fdim.split(':')
fdlen = len(fdlist)
dlist = ['']*fdlen
for index in xrange(fdlen):
# For each dimension component, find the standard_name
# First, see if it is blank, then integer, then var
fd = fdlist[index]
if len(fd) == 0:
fdnum = ''
else:
fdnum = None
# End if
if len(fd) > 0:
try:
test = int(fd)
fdnum = fd
except Exception as e:
pass
# End try
# End if
if (len(fd) > 0) and (fdnum is None):
# Try to find the standard name for this item
if table.has(fd):
fdvar = table.get(fd)
fdnum = fdvar['standard_name']
else:
if logger is not None:
logger.warning('WARNING: No local variable found for dimension, {} in {}'.format(fd, lname))
# End if
fdnum = 'XXnot_foundXX'
# End if
# End if
dlist[index] = fdnum
# End for
# Reassemble dim
dims.append(':'.join(dlist))
# End for
# End if
# Reset dimensions
var['dimensions'] = '(' + ','.join(dims) + ')'
# End if
# End for
# End for
# Write out finalized metadata file
with open(metadata_filename_out, 'w') as mdfile:
spacer = ""
for table in mdconfig:
if len(spacer) > 0:
mdfile.write(spacer)
# End if
table.write(mdfile)
spacer = '\n'+72*'#'+'\n'