def read_new_metadata(filename,
module_name,
table_name,
scheme_name=None,
subroutine_name=None):
"""Read metadata in new format and convert output to ccpp_prebuild metadata dictionary"""
if not os.path.isfile(filename):
raise Exception("New metadata file {0} not found".format(filename))
# Save metadata, because this routine new_metadata
# is called once for every table in that file
if filename in NEW_METADATA_SAVE.keys():
new_metadata_headers = NEW_METADATA_SAVE[filename]
else:
new_metadata_headers = MetadataHeader.parse_metadata_file(filename)
NEW_METADATA_SAVE[filename] = new_metadata_headers
# Convert new metadata for requested table to old metadata dictionary
metadata = collections.OrderedDict()
for new_metadata_header in new_metadata_headers:
if not scheme_name:
if not new_metadata_header.title == table_name:
# Skip this table, since it is not requested right now
continue
if new_metadata_header.title == module_name:
container = encode_container(module_name)
else:
container = encode_container(module_name,
new_metadata_header.title)
else:
if not new_metadata_header.title == table_name:
# Skip this table, since it is not requested right now
continue
container = encode_container(module_name, scheme_name, table_name)
for new_var in new_metadata_header.variable_list():
standard_name = new_var.get_prop_value('standard_name')
rank = len(new_var.get_prop_value('dimensions'))
var = Var(
standard_name=standard_name,
long_name=new_var.get_prop_value('long_name'),
units=new_var.get_prop_value('units'),
local_name=new_var.get_prop_value('local_name'),
type=new_var.get_prop_value('type'),
container=container,
kind=new_var.get_prop_value('kind'),
intent=new_var.get_prop_value('intent'),
optional='T' if new_var.get_prop_value('optional') else 'F',
)
# Set rank using integer-setter method
var.rank = rank
# Check for duplicates in same table
if standard_name in metadata.keys():
raise Exception(
"Error, multiple definitions of standard name {0} in new metadata table {1}"
.format(standard_name, table_name))
metadata[standard_name] = [var]
return metadata
def generate_scheme_caps(metadata, arguments, caps_dir, module_use_template_scheme_cap):
"""Generate scheme caps for all schemes parsed."""
success = True
# Change to physics directory
os.chdir(caps_dir)
# List of filenames of scheme caps
scheme_caps = []
for module_name in arguments.keys():
for scheme_name in arguments[module_name].keys():
for subroutine_name in arguments[module_name][scheme_name].keys():
# Skip subroutines without argument table or with empty argument table
if not arguments[module_name][scheme_name][subroutine_name]:
continue
# Create cap
cap = Cap()
cap.filename = "{0}_cap.F90".format(scheme_name)
scheme_caps.append(cap.filename)
# Parse all subroutines and their arguments to generate the cap
capdata = collections.OrderedDict()
for subroutine_name in arguments[module_name][scheme_name].keys():
capdata[subroutine_name] = []
for var_name in arguments[module_name][scheme_name][subroutine_name]:
container = encode_container(module_name, scheme_name, subroutine_name)
for var in metadata[var_name]:
if var.container == container:
capdata[subroutine_name].append(var)
break
# If required (not at the moment), add module use statements to module_use_template_scheme_cap
module_use_statement = module_use_template_scheme_cap
# Write cap
cap.write(module_name, module_use_statement, capdata)
#
os.chdir(BASEDIR)
return (success, scheme_caps)
def generate_scheme_caps(metadata_define, metadata_request, arguments,
pset_schemes, ccpp_field_maps, caps_dir):
"""Generate scheme caps for all schemes parsed."""
success = True
# Change to caps directory
os.chdir(caps_dir)
# List of filenames of scheme caps
scheme_caps = []
for module_name in arguments.keys():
for scheme_name in arguments[module_name].keys():
for subroutine_name in arguments[module_name][scheme_name].keys():
# Skip subroutines without argument table or with empty argument table
if not arguments[module_name][scheme_name][subroutine_name]:
continue
# Create cap
cap = Cap()
cap.filename = "{0}_cap.F90".format(scheme_name)
scheme_caps.append(cap.filename)
# Parse all subroutines and their arguments to generate the cap
capdata = collections.OrderedDict()
for subroutine_name in arguments[module_name][scheme_name].keys():
capdata[subroutine_name] = []
for var_name in arguments[module_name][scheme_name][
subroutine_name]:
container = encode_container(module_name, scheme_name,
subroutine_name)
for var in metadata_request[var_name]:
if var.container == container:
capdata[subroutine_name].append(var)
break
# Write cap using the unique physics set for the scheme
pset = pset_schemes[scheme_name][0]
cap.write(module_name, capdata, ccpp_field_maps[pset],
metadata_define)
#
os.chdir(BASEDIR)
return (success, scheme_caps)
def parse_scheme_tables(filepath, filename):
"""Parses metadata tables for a physics scheme that requests/requires variables as
input arguments. Metadata tables can only describe variables required by a subroutine
'subroutine_name' of scheme 'scheme_name' inside a module 'module_name'. Each variable
(standard_name) can exist only once, i.e. each entry (list of variables) in the metadata
dictionary contains only one element (variable = instance of class Var defined in
mkcap.py). The metadata dictionaries of the individual schemes are merged afterwards
(called from ccpp_prebuild.py) using merge_metadata_dicts, where multiple instances
of variables are compared for compatibility and collected in a list (entry in the
merged metadata dictionary). The merged metadata dictionary of all schemes (which
contains only compatible variable instances in the list referred to by standard_name)
is then compared to the unique definition in the metadata dictionary of the variables
provided by the host model using compare_metadata in ccpp_prebuild.py."""
# Set debug to true if logging level is debug
debug = logging.getLogger().getEffectiveLevel() == logging.DEBUG
# Final metadata container for all variables in file
metadata = collections.OrderedDict()
# Registry of modules and derived data types in file
registry = collections.OrderedDict()
# Argument lists of each subroutine in the file
arguments = collections.OrderedDict()
# List of Dependencies for this scheme
dependencies = collections.OrderedDict()
# Read all lines of the file at once
with (open(filename, 'r')) as file:
try:
file_lines = file.readlines()
except UnicodeDecodeError:
raise Exception(
"Decoding error while trying to read file {}, check that the file only contains ASCII characters"
.format(filename))
lines = []
original_line_numbers = []
buffer = ''
for i in range(len(file_lines)):
line = file_lines[i].rstrip('\n').strip()
# Skip empty lines
if line == '' or line == '&':
continue
# Remove line continuations: concatenate with following lines
if line.endswith('&'):
buffer += file_lines[i].rstrip('\n').replace('&', ' ')
continue
# Write out line with buffer and reset buffer
lines.append(buffer + file_lines[i].rstrip('\n').replace('&', ' '))
original_line_numbers.append(i + 1)
buffer = ''
del file_lines
# Find all modules within the file, and save the start and end lines
module_lines = collections.OrderedDict()
line_counter = 0
for line in lines:
# For the purpose of identifying module constructs, remove any trailing comments from line
if '!' in line and not line.startswith('!'):
line = line[:line.find('!')]
words = line.split()
if len(words) > 1 and words[0].lower(
) == 'module' and not words[1].lower() == 'procedure':
module_name = words[1].strip()
if module_name in registry.keys():
raise Exception(
'Duplicate module name {0}'.format(module_name))
registry[module_name] = collections.OrderedDict()
module_lines[module_name] = {'startline': line_counter}
elif len(words) > 1 and words[0].lower() == 'end' and words[1].lower(
) == 'module':
try:
test_module_name = words[2]
except IndexError:
logging.warning(
'Warning, encountered closing statement "end module" without module name; assume module_name is {0}'
.format(module_name))
test_module_name = module_name
if not module_name == test_module_name:
raise Exception(
'Module names in opening/closing statement do not match: {0} vs {1}'
.format(module_name, test_module_name))
module_lines[module_name]['endline'] = line_counter
line_counter += 1
# Parse each module in the file separately
for module_name in registry.keys():
startline = module_lines[module_name]['startline']
endline = module_lines[module_name]['endline']
line_counter = 0
in_subroutine = False
for line in lines[startline:endline]:
# For the purpose of identifying scheme constructs, remove any trailing comments from line
if '!' in line and not line.startswith('!'):
line = line[:line.find('!')]
current_line_number = startline + line_counter
words = line.split()
for j in range(len(words)):
# Check for the word 'subroutine', that it is the first word in the line,
# and that a name exists afterwards. Nested subroutines are ignored.
if words[j].lower(
) == 'subroutine' and j == 0 and len(words) > 1:
if in_subroutine:
logging.debug(
'Warning, ignoring nested subroutine in module {0} and subroutine {1}'
.format(module_name, subroutine_name))
continue
subroutine_name = words[j + 1].split('(')[0].strip()
# Consider the last substring separated by a '_' of the subroutine name as a 'postfix'
if subroutine_name.find('_') >= 0:
scheme_name = None
subroutine_suffix = None
for ccpp_stage in CCPP_STAGES:
pattern = '^(.*)_{}$'.format(ccpp_stage)
match = re.match(pattern, subroutine_name)
if match:
scheme_name = match.group(1)
subroutine_suffix = ccpp_stage
break
if match:
if not scheme_name == module_name:
raise Exception(
'Scheme name differs from module name: module_name="{0}" vs. scheme_name="{1}"'
.format(module_name, scheme_name))
if not scheme_name in registry[module_name].keys():
registry[module_name][
scheme_name] = collections.OrderedDict()
if subroutine_name in registry[module_name][
scheme_name].keys():
raise Exception(
'Duplicate subroutine name {0} in module {1}'
.format(subroutine_name, module_name))
registry[module_name][scheme_name][
subroutine_name] = [current_line_number]
in_subroutine = True
elif words[j].lower() == 'subroutine' and j == 1 and words[
j - 1].lower() == 'end':
try:
test_subroutine_name = words[j + 1]
except IndexError:
logging.warning('Warning, encountered closing statement "end subroutine" without subroutine name; ' +\
' assume subroutine_name is {0}'.format(subroutine_name))
test_subroutine_name = subroutine_name
if in_subroutine and subroutine_name == test_subroutine_name:
in_subroutine = False
registry[module_name][scheme_name][
subroutine_name].append(current_line_number)
# Avoid problems by enforcing end statements to carry a descriptor (subroutine, module, ...)
elif in_subroutine and len(
words) == 1 and words[0].lower() == 'end':
raise Exception('Encountered closing statement "end" without descriptor (subroutine, module, ...): ' +\
'line {0}="{1}" in file {2}'.format(original_line_numbers[current_line_number], line, filename))
line_counter += 1
# Check that for each registered subroutine the start and end lines were found
for scheme_name in registry[module_name].keys():
for subroutine_name in registry[module_name][scheme_name].keys():
if not len(registry[module_name][scheme_name]
[subroutine_name]) == 2:
raise Exception(
'Error parsing start and end lines for subroutine {0} in module {1}'
.format(subroutine_name, module_name))
logging.debug('Parsing file {0} with registry {1}'.format(
filename, registry))
for scheme_name in registry[module_name].keys():
# Record the dependencies for the scheme
if not scheme_name in dependencies.keys():
dependencies[scheme_name] = []
for subroutine_name in registry[module_name][scheme_name].keys():
# Record the order of variables in the call list to each subroutine in a list
if not scheme_name in arguments.keys():
arguments[scheme_name] = collections.OrderedDict()
if not subroutine_name in arguments[scheme_name].keys():
arguments[scheme_name][subroutine_name] = []
# Find the argument table corresponding to each subroutine by searching
# "upward" from the subroutine definition line for the "arg_table_SubroutineName" section
table_found = False
header_line_number = None
for line_number in range(
registry[module_name][scheme_name][subroutine_name][0],
-1, -1):
line = lines[line_number]
words = line.split()
for word in words:
if (len(words) > 2 and words[0] in ['!!', '!>']
and '\section' in words[1]
and 'arg_table_{0}'.format(subroutine_name)
in words[2]):
table_found = True
header_line_number = line_number + 1
table_name = subroutine_name
break
else:
for word in words:
if 'arg_table_{0}'.format(
subroutine_name) in word:
raise Exception(
"Malformatted table found in {0} / {1} / {2} / {3}"
.format(filename, module_name,
scheme_name, subroutine_name))
if table_found:
break
# If an argument table is found, parse it
if table_found:
if 'htmlinclude' in lines[header_line_number].lower():
words = lines[header_line_number].split()
if words[0] == '!!' and words[
1] == '\\htmlinclude' and len(words) == 3:
filename_parts = filename.split('.')
metadata_filename = '.'.join(
filename_parts[0:len(filename_parts) -
1]) + '.meta'
(this_metadata,
these_dependencies) = read_new_metadata(
metadata_filename,
module_name,
table_name,
scheme_name=scheme_name,
subroutine_name=subroutine_name)
if these_dependencies:
# Remove duplicates when combining lists
dependencies[scheme_name] = list(
set(dependencies[scheme_name] +
these_dependencies))
for var_name in this_metadata.keys():
# Add standard_name to argument list for this subroutine
arguments[scheme_name][subroutine_name].append(
var_name)
# For all instances of this var (can be only one) in this subroutine's metadata,
# add to global metadata and check for compatibility with existing variables
for var in this_metadata[var_name]:
if not var_name in metadata.keys():
metadata[var_name] = [var]
else:
for existing_var in metadata[var_name]:
if not existing_var.compatible(
var):
raise Exception('New entry for variable {0}'.format(var_name) + \
' in argument table of subroutine {0}'.format(subroutine_name) +\
' is incompatible with existing entry:\n' +\
' existing: {0}\n'.format(existing_var.print_debug()) +\
' vs. new: {0}'.format(var.print_debug()))
metadata[var_name].append(var)
else:
raise Exception(
"Invalid definition of new metadata format in file {}, \htmlinclude must be preceeded by '!! ' : {}"
.format(filename, lines[header_line_number]))
# Next line must denote the end of table,
# i.e. look for a line containing only '!!'
line_number = header_line_number + 1
nextline = lines[line_number]
nextwords = nextline.split()
if len(nextwords) == 1 and nextwords[0].strip(
) == '!!':
end_of_table = True
else:
raise Exception(
'Encountered invalid format "{0}" of new metadata table hook in table {1}'
.format(line, table_name))
line_number += 1
else:
words = lines[header_line_number].split()
if len(words) == 1 and words[0].strip() == '!!':
logging.info(
"Legacy extension - skip empty table for {}".
format(table_name))
end_of_table = True
line_number += 1
else:
raise Exception(
"Invalid definition of metadata in file {0}: {1}"
.format(filename, words))
# After parsing entire metadata table for the subroutine, check that all
# mandatory CCPP variables are present - skip empty tables.
if arguments[scheme_name][subroutine_name]:
for var_name in CCPP_MANDATORY_VARIABLES.keys():
if not var_name in arguments[scheme_name][
subroutine_name]:
raise Exception(
'Mandatory CCPP variable {0} not declared in metadata table of subroutine {1}'
.format(var_name, subroutine_name))
# Sort the dependencies to avoid differences in the auto-generated code
dependencies[scheme_name].sort()
# Debugging output to screen and to XML
if debug and len(metadata.keys()) > 0:
# To screen
logging.debug('Module name: {}'.format(module_name))
for scheme_name in registry[module_name].keys():
logging.debug('Scheme name: {}'.format(scheme_name))
logging.debug('Scheme dependencies: {}'.format(
dependencies[scheme_name]))
for subroutine_name in registry[module_name][scheme_name].keys(
):
container = encode_container(module_name, scheme_name,
subroutine_name)
vars_in_subroutine = []
for var_name in metadata.keys():
for var in metadata[var_name]:
if var.container == container:
vars_in_subroutine.append(var_name)
logging.debug('Variables in subroutine {}: {}'.format(
subroutine_name, ', '.join(vars_in_subroutine)))
# Standard output to screen
elif len(metadata.keys()) > 0:
for scheme_name in registry[module_name].keys():
if dependencies[scheme_name]:
logging.info(
'Parsed tables in scheme {} with dependencies {}'.
format(scheme_name, dependencies[scheme_name]))
else:
logging.info(
'Parsed tables in scheme {}'.format(scheme_name))
# End of loop over all module_names
# Add absolute path to dependencies
for scheme_name in dependencies.keys():
if dependencies[scheme_name]:
dependencies[scheme_name] = [
os.path.join(filepath, x) for x in dependencies[scheme_name]
]
for dependency in dependencies[scheme_name]:
if not os.path.isfile(dependency):
raise Exception(
"Dependency {} for scheme table {} does not exit".
format(dependency, scheme_name))
return (metadata, arguments, dependencies)
def parse_variable_tables(filepath, filename):
"""Parses metadata tables on the host model side that define the available variables.
Metadata tables can refer to variables inside a module or as part of a derived
datatype, which itself is defined inside a module (depending on the location of the
metadata table). Each variable (standard_name) can exist only once, i.e. each entry
(list of variables) in the metadata dictionary contains only one element
(variable = instance of class Var defined in mkcap.py)"""
# Set debug to true if logging level is debug
debug = logging.getLogger().getEffectiveLevel() == logging.DEBUG
# Final metadata container for all variables in file
metadata = collections.OrderedDict()
# Registry of modules and derived data types in file
registry = collections.OrderedDict()
# List of dependencies for this scheme
dependencies = collections.OrderedDict()
# Read all lines of the file at once
with (open(filename, 'r')) as file:
try:
file_lines = file.readlines()
except UnicodeDecodeError:
raise Exception(
"Decoding error while trying to read file {}, check that the file only contains ASCII characters"
.format(filename))
lines = []
buffer = ''
for i in range(len(file_lines)):
line = file_lines[i].rstrip('\n').strip()
# Skip empty lines
if line == '' or line == '&':
continue
# Remove line continuations: concatenate with following lines
if line.endswith('&'):
buffer += file_lines[i].rstrip('\n').replace('&', ' ')
continue
# Write out line with buffer and reset buffer
lines.append(buffer + file_lines[i].rstrip('\n').replace('&', ' '))
buffer = ''
del file_lines
# Find all modules within the file, and save the start and end lines
module_lines = collections.OrderedDict()
line_counter = 0
for line in lines:
words = line.split()
if len(words) > 1 and words[0].lower() in [
'module', 'program'
] and not words[1].lower() == 'procedure':
module_name = words[1].strip()
if module_name in registry.keys():
raise Exception(
'Duplicate module name {0}'.format(module_name))
registry[module_name] = collections.OrderedDict()
module_lines[module_name] = {'startline': line_counter}
elif len(words) > 1 and words[0].lower() == 'end' and words[1].lower(
) in ['module', 'program']:
try:
test_module_name = words[2]
except IndexError:
logging.warning(
'Encountered closing statement "end module" without module name; assume module_name is {0}'
.format(module_name))
test_module_name = module_name
if not module_name == test_module_name:
raise Exception(
'Module names in opening/closing statement do not match: {0} vs {1}'
.format(module_name, test_module_name))
module_lines[module_name]['endline'] = line_counter
line_counter += 1
# Parse each module in the file separately
for module_name in registry.keys():
startline = module_lines[module_name]['startline']
endline = module_lines[module_name]['endline']
line_counter = 0
in_type = False
for line in lines[startline:endline]:
# For the purpose of identifying module, type and scheme constructs, remove any trailing comments from line
if '!' in line and not line.startswith('!'):
line = line[:line.find('!')]
current_line_number = startline + line_counter
words = line.split()
for j in range(len(words)):
# Check for the word 'type', that it is the first word in the line,
# and that a name exists afterwards. It is assumed that definitions
# (not usage) of derived types cannot be nested - reasonable for Fortran.
# The following if / elif / else statements filter lines that do not
# contain a type definition.
#
# Ignore words containing type that are not 'type', 'type,', 'type::';
# this includes variable declarations of a user defined type, e.g. 'type(mytype) ::'
if not (words[j].lower()=='type' or \
words[j].lower().startswith('type,') or \
words[j].lower().startswith('type::')):
continue
# Ignore variable declarations of a user defined type with a space
# between 'type' and '(', e.g. 'type (mytype) ::'
elif j == 0 and len(words) > 1 and words[j +
1].startswith('('):
continue
# Ignore lines starting with 'type is' or 'type is(' (select type statements)
elif (words[j].lower() == 'type' and j==0 and j<len(words)-1 and \
(words[j+1].lower() == 'is' or words[j+1].lower().startswith('is('))):
continue
# Detect 'end type TYPENAME' and (fallback) unlabeled 'end type' statements
elif words[j].lower() == 'type' and j == 1 and words[
j - 1].lower() == 'end':
if not in_type:
raise Exception(
'Encountered "end_type" without corresponding "type" statement'
)
try:
test_type_name = words[j + 1]
except IndexError:
logging.warning(
'Encountered closing statement "end type" without type name; assume type_name is {0}'
.format(type_name))
test_type_name = type_name
if not type_name == test_type_name:
raise Exception(
'Type names in opening/closing statement do not match: {0} vs {1}'
.format(type_name, test_type_name))
in_type = False
registry[module_name][type_name].append(
current_line_number)
# If type is not the first word, ignore the word
elif j > 0:
continue
# Detect type definition using Ftype_type_decl class, routine
# type_def_line and extract type_name
else:
type_declaration = Ftype_type_decl.type_def_line(
line.strip())
if in_type:
raise Exception(
'Nested definitions of derived types not supported'
)
in_type = True
type_name = type_declaration[0]
if type_name in registry[module_name].keys():
raise Exception(
'Duplicate derived type name {0} in module {1}'.
format(type_name, module_name))
registry[module_name][type_name] = [current_line_number]
# Done with user defined type detection
line_counter += 1
logging.debug('Parsing file {0} with registry {1}'.format(
filename, registry))
# Variables can either be defined at module-level or in derived types - alongside with their tables
line_counter = 0
in_table = False
in_type = False
for line in lines[startline:endline]:
current_line_number = startline + line_counter
# Check for beginning of new table
words = line.split()
# This is case sensitive
if len(words) > 2 and words[0] in [
'!!', '!>'
] and '\section' in words[1] and 'arg_table_' in words[2]:
if in_table:
raise Exception(
'Encountered table start for table {0} while still in table {1}'
.format(words[2].replace('arg_table_', ''),
table_name))
table_name = words[2].replace('arg_table_', '')
if not (table_name == module_name
or table_name in registry[module_name].keys()):
raise Exception(
'Encountered table with name {0} without corresponding module or type name'
.format(table_name))
in_table = True
if not table_name in dependencies.keys():
dependencies[table_name] = []
header_line_number = current_line_number + 1
line_counter += 1
continue
elif (words[0].startswith('!!')
or words[0].startswith('!>')) and '\section' in words[0]:
raise Exception(
"Malformatted table found in {0} / {1} / {2}".format(
filename, module_name, table_name))
# If an argument table is found, parse it
if in_table:
words = line.split('|')
# Separate the table headers
if current_line_number == header_line_number:
if 'htmlinclude' in line.lower():
words = line.split()
if words[0] == '!!' and words[
1] == '\\htmlinclude' and len(words) == 3:
filename_parts = filename.split('.')
metadata_filename = '.'.join(
filename_parts[0:len(filename_parts) -
1]) + '.meta'
(this_metadata,
these_dependencies) = read_new_metadata(
metadata_filename, module_name, table_name)
if these_dependencies:
# Remove duplicates when combining lists
dependencies[table_name] = list(
set(dependencies[table_name] +
these_dependencies))
for var_name in this_metadata.keys():
for var in this_metadata[var_name]:
if var_name in CCPP_MANDATORY_VARIABLES.keys(
) and not CCPP_MANDATORY_VARIABLES[
var_name].compatible(var):
raise Exception('Entry for variable {0}'.format(var_name) + \
' in argument table {0}'.format(table_name) +\
' is incompatible with mandatory variable:\n' +\
' existing: {0}\n'.format(CCPP_MANDATORY_VARIABLES[var_name].print_debug()) +\
' vs. new: {0}'.format(var.print_debug()))
# Add variable to metadata dictionary
if not var_name in metadata.keys():
metadata[var_name] = [var]
else:
for existing_var in metadata[var_name]:
if not existing_var.compatible(
var):
raise Exception('New entry for variable {0}'.format(var_name) + \
' in argument table {0}'.format(table_name) +\
' is incompatible with existing entry:\n' +\
' existing: {0}\n'.format(existing_var.print_debug()) +\
' vs. new: {0}'.format(var.print_debug()))
metadata[var_name].append(var)
else:
raise Exception(
"Invalid definition of new metadata format in file {}, \htmlinclude must be preceeded by '!! ' : {}"
.format(filename, line))
line_counter += 1
continue
# Check for blank table
if len(words) <= 1:
logging.debug(
'Skipping blank table {0}'.format(table_name))
in_table = False
line_counter += 1
continue
table_header = [x.strip() for x in words[1:-1]]
# Check that only valid table headers are used
for item in table_header:
if not item in VALID_ITEMS['header']:
raise Exception(
'Invalid column header {0} in argument table {1}'
.format(item, table_name))
# Locate mandatory column 'standard_name'
try:
standard_name_index = table_header.index(
'standard_name')
except ValueError:
raise Exception(
'Mandatory column standard_name not found in argument table {0}'
.format(table_name))
line_counter += 1
# DH* warn or raise error for old metadata format
logging.warn(
"Old metadata table found for table {}".format(
table_name))
#raise Exception("Old metadata table found for table {}".format(table_name))
# *DH
continue
else:
if len(words) == 1:
# End of table
if words[0].strip() == '!!':
if not current_line_number == header_line_number + 1:
raise Exception(
"Invalid definition of new metadata format in file {0}"
.format(filename))
in_table = False
line_counter += 1
continue
else:
raise Exception(
'Encountered invalid line "{0}" in argument table {1}'
.format(line, table_name))
else:
raise Exception(
"Invalid definition of metadata in file {0}: {1}".
format(filename, words))
line_counter += 1
# Informative output to screen
if debug and len(metadata.keys()) > 0:
for module_name in registry.keys():
logging.debug('Module name: {0}'.format(module_name))
container = encode_container(module_name)
vars_in_module = []
for var_name in metadata.keys():
for var in metadata[var_name]:
if var.container == container:
vars_in_module.append(var_name)
logging.debug('Module variables: {0}'.format(
', '.join(vars_in_module)))
for type_name in registry[module_name].keys():
container = encode_container(module_name, type_name)
vars_in_type = []
for var_name in metadata.keys():
for var in metadata[var_name]:
if var.container == container:
vars_in_type.append(var_name)
logging.debug('Variables in derived type {0}: {1}'.format(
type_name, ', '.join(vars_in_type)))
if len(metadata.keys()) > 0:
logging.info(
'Parsed variable definition tables in module {0}'.format(
module_name))
# Add absolute path to dependencies
for table_name in dependencies.keys():
if dependencies[table_name]:
dependencies[table_name] = [
os.path.join(filepath, x) for x in dependencies[table_name]
]
for dependency in dependencies[table_name]:
if not os.path.isfile(dependency):
raise Exception(
"Dependency {} for variable table {} does not exit".
format(dependency, table_name))
return (metadata, dependencies)
def read_new_metadata(filename,
module_name,
table_name,
scheme_name=None,
subroutine_name=None):
"""Read metadata in new format and convert output to ccpp_prebuild metadata dictionary"""
if not os.path.isfile(filename):
raise Exception("New metadata file {0} not found".format(filename))
# Save metadata, because this routine new_metadata
# is called once for every table in that file
if filename in NEW_METADATA_SAVE.keys():
new_metadata_headers = NEW_METADATA_SAVE[filename]
else:
new_metadata_headers = MetadataHeader.parse_metadata_file(filename)
NEW_METADATA_SAVE[filename] = new_metadata_headers
# Record dependencies for the metadata table (only applies to schemes)
has_property_table = False
dependencies = []
# Convert new metadata for requested table to old metadata dictionary
metadata = collections.OrderedDict()
for new_metadata_header in new_metadata_headers:
# Module or DDT tables
if not scheme_name:
# Module property tables
if new_metadata_header.property_table and new_metadata_header.title == module_name:
# If this is a ccpp-table-properties table for a module, it can only contain dependencies;
# ensure that for module tables, the header type is "module"
if not new_metadata_header.header_type == 'module':
raise Exception(
"Unsupported header_type '{}' for table properties for modules"
.format(new_metadata_header.header_type))
dependencies += new_metadata_header.dependencies
has_property_table = True
continue
# DDT property tables
elif new_metadata_header.property_table:
# If this is a ccpp-table-properties table for a DDT, it can only contain dependencies;
# ensure that for DDT tables, the header type is "ddt"
if not new_metadata_header.header_type == 'ddt':
raise Exception(
"Unsupported header_type '{}' for table properties for DDTs"
.format(new_metadata_header.header_type))
dependencies += new_metadata_header.dependencies
has_property_table = True
continue
# Module or DDT argument tables
else:
if not new_metadata_header.title == table_name:
# Skip this table, since it is not requested right now
continue
# Distinguish between module argument tables and DDT argument tables
if new_metadata_header.title == module_name:
container = encode_container(module_name)
else:
container = encode_container(module_name,
new_metadata_header.title)
else:
# Scheme property tables
if new_metadata_header.property_table and new_metadata_header.title == scheme_name:
# If this is a ccpp-table-properties table for a scheme, it can only contain dependencies;
# ensure that for scheme tables, the header type is "scheme"
if not new_metadata_header.header_type == 'scheme':
raise Exception(
"Unsupported header_type '{}' for table properties for schemes"
.format(new_metadata_header.header_type))
dependencies += new_metadata_header.dependencies
has_property_table = True
continue
# Scheme argument tables
else:
if not new_metadata_header.title == table_name:
# Skip this table, since it is not requested right now
continue
container = encode_container(module_name, scheme_name,
table_name)
for new_var in new_metadata_header.variable_list():
standard_name = new_var.get_prop_value('standard_name')
# DH* 2020-05-26
# Legacy extension for inconsistent metadata (use of horizontal_dimension versus horizontal_loop_extent).
# Since horizontal_dimension and horizontal_loop_extent have the same attributes (otherwise it doesn't
# make sense), we swap the standard name and add a note to the long name
legacy_note = ''
if standard_name == 'horizontal_loop_extent' and scheme_name and \
(table_name.endswith("_init") or table_name.endswith("_finalize")):
logging.warn("Legacy extension - replacing variable 'horizontal_loop_extent'" + \
" with 'horizontal_dimension' in table {}".format(table_name))
standard_name = 'horizontal_dimension'
legacy_note = ' replaced by horizontal dimension (legacy extension)'
elif standard_name == 'horizontal_dimension' and scheme_name and table_name.endswith(
"_run"):
logging.warn("Legacy extension - replacing variable 'horizontal_dimension' " + \
"with 'horizontal_loop_extent' in table {}".format(table_name))
standard_name = 'horizontal_loop_extent'
legacy_note = ' replaced by horizontal loop extent (legacy extension)'
# Adjust dimensions
dimensions = new_var.get_prop_value('dimensions')
if scheme_name and (table_name.endswith("_init") or table_name.endswith("_finalize")) \
and 'horizontal_loop_extent' in dimensions:
logging.warn("Legacy extension - replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \
"for variable {} in table {}".format(standard_name,table_name))
dimensions = [
'horizontal_dimension'
if x == 'horizontal_loop_extent' else x for x in dimensions
]
elif scheme_name and table_name.endswith(
"_run") and 'horizontal_dimension' in dimensions:
logging.warn("Legacy extension - replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \
"for variable {} in table {}".format(standard_name,table_name))
dimensions = [
'horizontal_loop_extent'
if x == 'horizontal_dimension' else x for x in dimensions
]
# *DH 2020-05-26
if new_var.get_prop_value('active').lower() == '.true.':
active = 'T'
elif new_var.get_prop_value('active').lower() == '.false.':
active = 'F'
else:
# Replace multiple whitespaces, preserve case
active = ' '.join(new_var.get_prop_value('active').split())
var = Var(
standard_name=standard_name,
long_name=new_var.get_prop_value('long_name') + legacy_note,
units=new_var.get_prop_value('units'),
local_name=new_var.get_prop_value('local_name'),
type=new_var.get_prop_value('type'),
dimensions=dimensions,
container=container,
kind=new_var.get_prop_value('kind'),
intent=new_var.get_prop_value('intent'),
optional='T' if new_var.get_prop_value('optional') else 'F',
active=active,
)
# Check for duplicates in same table
if standard_name in metadata.keys():
raise Exception(
"Error, multiple definitions of standard name {} in new metadata table {}"
.format(standard_name, table_name))
metadata[standard_name] = [var]
# CCPP property tables are mandatory
if not has_property_table:
if scheme_name:
raise Exception("Metadata file {} for scheme {} does not have a [ccpp-table-properties] section,".format(filename, scheme_name) + \
" or the 'name = ...' attribute in the [ccpp-table-properties] is wrong")
else:
raise Exception("Metadata file {} for table {} does not have a [ccpp-table-properties] section,".format(filename, table_name) + \
" or the 'name = ...' attribute in the [ccpp-table-properties] is wrong")
return (metadata, dependencies)
def parse_scheme_tables(filename):
"""Parses metadata tables for a physics scheme that requests/requires variables as
input arguments. Metadata tables can only describe variables required by a subroutine
'subroutine_name' of scheme 'scheme_name' inside a module 'module_name'. Each variable
(standard_name) can exist only once, i.e. each entry (list of variables) in the metadata
dictionary contains only one element (variable = instance of class Var defined in
mkcap.py). The metadata dictionaries of the individual schemes are merged afterwards
(called from ccpp_prebuild.py) using merge_metadata_dicts, where multiple instances
of variables are compared for compatibility and collected in a list (entry in the
merged metadata dictionary). The merged metadata dictionary of all schemes (which
contains only compatible variable instances in the list referred to by standard_name)
is then compared to the unique definition in the metadata dictionary of the variables
provided by the host model using compare_metadata in ccpp_prebuild.py."""
# Set debug to true if logging level is debug
debug = logging.getLogger().getEffectiveLevel() == logging.DEBUG
# Valid suffices for physics scheme routines
subroutine_suffices = ['init', 'run', 'finalize']
# Final metadata container for all variables in file
metadata = collections.OrderedDict()
# Registry of modules and derived data types in file
#registry = {}
registry = collections.OrderedDict()
# Argument lists of each subroutine in the file
arguments = collections.OrderedDict()
# Read all lines of the file at once
with (open(filename, 'r')) as file:
file_lines = file.readlines()
lines = []
original_line_numbers = []
buffer = ''
for i in range(len(file_lines)):
line = file_lines[i].rstrip('\n').strip()
# Skip empty lines
if line == '' or line == '&':
continue
# Remove line continuations: concatenate with following lines
if line.endswith('&'):
buffer += file_lines[i].rstrip('\n').replace('&', ' ')
continue
# Write out line with buffer and reset buffer
lines.append(buffer + file_lines[i].rstrip('\n').replace('&', ' '))
original_line_numbers.append(i + 1)
buffer = ''
del file_lines
# Find all modules within the file, and save the start and end lines
module_lines = {}
line_counter = 0
for line in lines:
# For the purpose of identifying module constructs, remove any trailing comments from line
if '!' in line and not line.startswith('!'):
line = line[:line.find('!')]
words = line.split()
if len(words) > 1 and words[0].lower(
) == 'module' and not words[1].lower() == 'procedure':
module_name = words[1].strip()
if module_name in registry.keys():
raise Exception(
'Duplicate module name {0}'.format(module_name))
registry[module_name] = {}
module_lines[module_name] = {'startline': line_counter}
elif len(words) > 1 and words[0].lower() == 'end' and words[1].lower(
) == 'module':
try:
test_module_name = words[2]
except IndexError:
logging.warning(
'Warning, encountered closing statement "end module" without module name; assume module_name is {0}'
.format(module_name))
test_module_name = module_name
if not module_name == test_module_name:
raise Exception(
'Module names in opening/closing statement do not match: {0} vs {1}'
.format(module_name, test_module_name))
module_lines[module_name]['endline'] = line_counter
line_counter += 1
# Parse each module in the file separately
for module_name in registry.keys():
startline = module_lines[module_name]['startline']
endline = module_lines[module_name]['endline']
line_counter = 0
in_subroutine = False
for line in lines[startline:endline]:
# For the purpose of identifying scheme constructs, remove any trailing comments from line
if '!' in line and not line.startswith('!'):
line = line[:line.find('!')]
current_line_number = startline + line_counter
words = line.split()
for j in range(len(words)):
# Check for the word 'subroutine', that it is the first word in the line,
# and that a name exists afterwards. Nested subroutines are ignored.
if words[j].lower(
) == 'subroutine' and j == 0 and len(words) > 1:
if in_subroutine:
logging.debug(
'Warning, ignoring nested subroutine in module {0} and subroutine {1}'
.format(module_name, subroutine_name))
continue
subroutine_name = words[j + 1].split('(')[0].strip()
# Consider the last substring separated by a '_' of the subroutine name as a 'postfix'
if subroutine_name.find('_') >= 0:
subroutine_suffix = subroutine_name.split('_')[-1]
if subroutine_suffix in subroutine_suffices:
scheme_name = subroutine_name[0:subroutine_name.
rfind('_')]
if not scheme_name == module_name:
raise Exception(
'Scheme name differs from module name: module_name="{0}" vs. scheme_name="{1}"'
.format(module_name, scheme_name))
if not scheme_name in registry[module_name].keys():
registry[module_name][scheme_name] = {}
if subroutine_name in registry[module_name][
scheme_name].keys():
raise Exception(
'Duplicate subroutine name {0} in module {1}'
.format(subroutine_name, module_name))
registry[module_name][scheme_name][
subroutine_name] = [current_line_number]
in_subroutine = True
elif words[j].lower() == 'subroutine' and j == 1 and words[
j - 1].lower() == 'end':
try:
test_subroutine_name = words[j + 1]
except IndexError:
logging.warning('Warning, encountered closing statement "end subroutine" without subroutine name; ' +\
' assume subroutine_name is {0}'.format(subroutine_name))
test_subroutine_name = subroutine_name
if in_subroutine and subroutine_name == test_subroutine_name:
in_subroutine = False
registry[module_name][scheme_name][
subroutine_name].append(current_line_number)
# Avoid problems by enforcing end statements to carry a descriptor (subroutine, module, ...)
elif in_subroutine and len(
words) == 1 and words[0].lower() == 'end':
raise Exception('Encountered closing statement "end" without descriptor (subroutine, module, ...): ' +\
'line {0}="{1}" in file {2}'.format(original_line_numbers[current_line_number], line, filename))
line_counter += 1
# Check that for each registered subroutine the start and end lines were found
for scheme_name in registry[module_name].keys():
for subroutine_name in registry[module_name][scheme_name].keys():
if not len(registry[module_name][scheme_name]
[subroutine_name]) == 2:
raise Exception(
'Error parsing start and end lines for subroutine {0} in module {1}'
.format(subroutine_name, module_name))
logging.debug('Parsing file {0} with registry {1}'.format(
filename, registry))
for scheme_name in registry[module_name].keys():
for subroutine_name in registry[module_name][scheme_name].keys():
# Record the order of variables in the call list to each subroutine in a list
if not module_name in arguments.keys():
arguments[module_name] = {}
if not scheme_name in arguments[module_name].keys():
arguments[module_name][scheme_name] = {}
if not subroutine_name in arguments[module_name][
scheme_name].keys():
arguments[module_name][scheme_name][subroutine_name] = []
# Find the argument table corresponding to each subroutine by searching
# "upward" from the subroutine definition line for the "arg_table_SubroutineName" section
table_found = False
header_line_number = None
for line_number in range(
registry[module_name][scheme_name][subroutine_name][0],
-1, -1):
line = lines[line_number]
words = line.split()
for word in words:
if (len(words) > 2 and words[0] in ['!!', '!>']
and '\section' in words[1]
and 'arg_table_{0}'.format(subroutine_name)
in words[2]):
table_found = True
header_line_number = line_number + 1
table_name = subroutine_name
break
else:
for word in words:
if 'arg_table_{0}'.format(
subroutine_name) in word:
raise Exception(
"Malformatted table found in {0} / {1} / {2} / {3}"
.format(filename, module_name,
scheme_name, subroutine_name))
if table_found:
break
# If an argument table is found, parse it
if table_found:
if 'htmlinclude' in lines[header_line_number].lower():
words = lines[header_line_number].split()
if words[0] == '!!' and words[
1] == '\\htmlinclude' and len(words) == 3:
new_metadata = True
filename_parts = filename.split('.')
metadata_filename = '.'.join(
filename_parts[0:len(filename_parts) -
1]) + '.meta'
this_metadata = read_new_metadata(
metadata_filename,
module_name,
table_name,
scheme_name=scheme_name,
subroutine_name=subroutine_name)
for var_name in this_metadata.keys():
# Add standard_name to argument list for this subroutine
arguments[module_name][scheme_name][
subroutine_name].append(var_name)
# For all instances of this var (can be only one) in this subroutine's metadata,
# add to global metadata and check for compatibility with existing variables
for var in this_metadata[var_name]:
if not var_name in metadata.keys():
metadata[var_name] = [var]
else:
for existing_var in metadata[var_name]:
if not existing_var.compatible(
var):
raise Exception('New entry for variable {0}'.format(var_name) + \
' in argument table of subroutine {0}'.format(subroutine_name) +\
' is incompatible with existing entry:\n' +\
' existing: {0}\n'.format(existing_var.print_debug()) +\
' vs. new: {0}'.format(var.print_debug()))
metadata[var_name].append(var)
# Next line must denote the end of table,
# i.e. look for a line containing only '!!'
line_number = header_line_number + 1
nextline = lines[line_number]
nextwords = nextline.split()
if len(nextwords) == 1 and nextwords[0].strip(
) == '!!':
end_of_table = True
else:
raise Exception(
'Encountered invalid format "{0}" of new metadata table hook in table {1}'
.format(line, table_name))
line_number += 1
continue
# Separate the table headers
table_header = lines[header_line_number].split('|')
# Check for blank table
if len(table_header) <= 1:
logging.debug(
'Skipping blank table {0}'.format(table_name))
table_found = False
continue
# Extract table header
table_header = [x.strip() for x in table_header[1:-1]]
# Check that only valid table headers are used
for item in table_header:
if not item in VALID_ITEMS['header']:
raise Exception(
'Invalid column header {0} in argument table {1}'
.format(item, table_name))
# Locate mandatory column 'standard_name'
try:
standard_name_index = table_header.index(
'standard_name')
except ValueError:
raise Exception(
'Mandatory column standard_name not found in argument table {0}'
.format(table_name))
# DH* warn or raise error for old metadata format
logging.warn(
"Old metadata table found for table {}".format(
table_name))
#raise Exception("Old metadata table found for table {}".format(table_name))
# *DH
# Get all of the variable information in table
end_of_table = False
line_number = header_line_number + 2
while not end_of_table:
line = lines[line_number]
words = line.split('|')
if len(words) == 1:
if words[0].strip() == '!!':
end_of_table = True
else:
raise Exception(
'Encountered invalid line "{0}" in argument table {1}'
.format(line, table_name))
else:
var_items = [x.strip() for x in words[1:-1]]
if not len(var_items) == len(table_header):
raise Exception(
'Error parsing variable entry "{0}" in argument table {1}'
.format(var_items, table_name))
var_name = var_items[standard_name_index]
# Column standard_name cannot be left blank in scheme_tables
if not var_name:
raise Exception(
'Encountered line "{0}" without standard name in argument table {1}'
.format(line, table_name))
# Enforce CF standards: no dashes, no dots (underscores instead)
if "-" in var_name:
raise Exception(
"Invalid character '-' found in standard name {0} in table {1}"
.format(var_name, table_name))
elif "." in var_name:
raise Exception(
"Invalid character '.' found in standard name {0} in table {1}"
.format(var_name, table_name))
#
# Add standard_name to argument list for this subroutine
arguments[module_name][scheme_name][
subroutine_name].append(var_name)
var = Var.from_table(table_header, var_items)
# Check for incompatible definitions with CCPP mandatory variables
if var_name in CCPP_MANDATORY_VARIABLES.keys(
) and not CCPP_MANDATORY_VARIABLES[
var_name].compatible(var):
raise Exception('Entry for variable {0}'.format(var_name) + \
' in argument table of subroutine {0}'.format(subroutine_name) +\
' is incompatible with mandatory variable:\n' +\
' existing: {0}\n'.format(CCPP_MANDATORY_VARIABLES[var_name].print_debug()) +\
' vs. new: {0}'.format(var.print_debug()))
# Record the location of this variable: module, scheme, table
container = encode_container(
module_name, scheme_name, table_name)
var.container = container
# Add variable to metadata dictionary
if not var_name in metadata.keys():
metadata[var_name] = [var]
else:
for existing_var in metadata[var_name]:
if not existing_var.compatible(var):
raise Exception('New entry for variable {0}'.format(var_name) + \
' in argument table of subroutine {0}'.format(subroutine_name) +\
' is incompatible with existing entry:\n' +\
' existing: {0}\n'.format(existing_var.print_debug()) +\
' vs. new: {0}'.format(var.print_debug()))
metadata[var_name].append(var)
line_number += 1
# After parsing entire metadata table for the subroutine, check that all mandatory CCPP variables are present
for var_name in CCPP_MANDATORY_VARIABLES.keys():
if not var_name in arguments[module_name][scheme_name][
subroutine_name]:
raise Exception(
'Mandatory CCPP variable {0} not declared in metadata table of subroutine {1}'
.format(var_name, subroutine_name))
# For CCPP-compliant files (i.e. files with metadata tables, perform additional checks)
if len(metadata.keys()) > 0:
# Check that all subroutine "root" names in the current module are equal to scheme_name
# and that there are exactly three subroutines for scheme X: X_init, X_run, X_finalize
message = ''
abort = False
for scheme_name in registry[module_name].keys():
# Pre-generate error message
message += 'Check that all subroutines in module {0} have the same root name:\n'.format(
module_name)
message += ' i.e. scheme_A_init, scheme_A_run, scheme_A_finalize\n'
message += 'Here is a list of the subroutine names for scheme {0}:\n'.format(
scheme_name)
message += '{0}\n\n'.format(', '.join(
sorted(registry[module_name][scheme_name].keys())))
if (not len(registry[module_name][scheme_name].keys()) == 3):
logging.exception(message)
abort = True
else:
for suffix in subroutine_suffices:
subroutine_name = '{0}_{1}'.format(scheme_name, suffix)
if not subroutine_name in registry[module_name][
scheme_name].keys():
logging.exception(message)
abort = True
if abort:
raise Exception(message)
# Debugging output to screen and to XML
if debug and len(metadata.keys()) > 0:
# To screen
logging.debug('Module name: {0}'.format(module_name))
for scheme_name in registry[module_name].keys():
logging.debug('Scheme name: {0}'.format(scheme_name))
for subroutine_name in registry[module_name][scheme_name].keys(
):
container = encode_container(module_name, scheme_name,
subroutine_name)
vars_in_subroutine = []
for var_name in metadata.keys():
for var in metadata[var_name]:
if var.container == container:
vars_in_subroutine.append(var_name)
logging.debug('Variables in subroutine {0}: {1}'.format(
subroutine_name, ', '.join(vars_in_subroutine)))
# Standard output to screen
elif len(metadata.keys()) > 0:
for scheme_name in registry[module_name].keys():
logging.info('Parsed tables in scheme {0}'.format(scheme_name))
# End of loop over all module_names
return (metadata, arguments)
def write(self, metadata_request, metadata_define, arguments):
# Create an inverse lookup table of local variable names defined (by the host model) and standard names
standard_name_by_local_name_define = {}
for standard_name in metadata_define.keys():
standard_name_by_local_name_define[metadata_define[standard_name][0].local_name] = standard_name
# First get target names of standard CCPP variables for subcycling and error handling
ccpp_loop_counter_target_name = metadata_request[CCPP_LOOP_COUNTER][0].target
ccpp_error_flag_target_name = metadata_request[CCPP_ERROR_FLAG_VARIABLE][0].target
ccpp_error_msg_target_name = metadata_request[CCPP_ERROR_MSG_VARIABLE][0].target
#
module_use = ''
self._module = 'ccpp_{suite}_{name}_cap'.format(name=self._name, suite=self._suite)
self._filename = '{module_name}.F90'.format(module_name=self._module)
self._subroutines = []
local_subs = ''
#
for ccpp_stage in CCPP_STAGES:
# The special init and finalize routines are only run in that stage
if self._init and not ccpp_stage == 'init':
continue
elif self._finalize and not ccpp_stage == 'finalize':
continue
# For mapping local variable names to standard names
local_vars = {}
# For mapping temporary variable names (for unit conversions, etc) to local variable names
tmpvar_cnt = 0
tmpvars = {}
#
body = ''
var_defs = ''
for subcycle in self._subcycles:
if subcycle.loop > 1 and ccpp_stage == 'run':
body += '''
associate(cnt => {loop_var_name})
do cnt=1,{loop_cnt}\n\n'''.format(loop_var_name=ccpp_loop_counter_target_name,loop_cnt=subcycle.loop)
for scheme_name in subcycle.schemes:
# actions_before and actions_after capture operations such
# as unit conversions, transformations that have to happen
# before and/or after the call to the subroutine (scheme)
actions_before = ''
actions_after = ''
#
module_name = scheme_name
subroutine_name = scheme_name + '_' + ccpp_stage
container = encode_container(module_name, scheme_name, subroutine_name)
# Skip entirely empty routines
if not arguments[module_name][scheme_name][subroutine_name]:
continue
error_check = ''
args = ''
length = 0
# Extract all variables needed (including indices for components/slices of arrays)
for var_standard_name in arguments[module_name][scheme_name][subroutine_name]:
# Pick the correct variable for this module/scheme/subroutine
# from the list of requested variable
for var in metadata_request[var_standard_name]:
if container == var.container:
break
if not var_standard_name in local_vars.keys():
if not var_standard_name in metadata_define.keys():
raise Exception('Variable {standard_name} not defined in host model metadata'.format(
standard_name=var_standard_name))
var_local_name_define = metadata_define[var_standard_name][0].local_name
# Break apart var_local_name_define into the different components (members of DDTs)
# to determine all variables that are required
(parent_local_name_define, parent_local_names_define_indices) = \
extract_parents_and_indices_from_local_name(var_local_name_define)
# Check for each of the derived parent local names as defined by the host model
# if they are registered (i.e. if there is a standard name for it). Note that
# the output of extract_parents_and_indices_from_local_name is stripped of any
# array subset information, i.e. a local name 'Atm(:)%...' will produce a
# parent local name 'Atm'. Since the rank of tha parent variable is not known
# at this point and since the local name in the host model metadata table could
# contain '(:)', '(:,:)', ... (up to the rank of the array), we search for the
# maximum number of dimensions allowed by the Fortran standard.
for local_name_define in [parent_local_name_define] + parent_local_names_define_indices:
parent_standard_name = None
parent_var = None
for i in xrange(FORTRAN_ARRAY_MAX_DIMS+1):
if i==0:
dims_string = ''
else:
# (:) for i==1, (:,:) for i==2, ...
dims_string = '(' + ','.join([':' for j in xrange(i)]) + ')'
if local_name_define+dims_string in standard_name_by_local_name_define.keys():
parent_standard_name = standard_name_by_local_name_define[local_name_define+dims_string]
parent_var = metadata_define[parent_standard_name][0]
break
if not parent_var:
raise Exception('Parent variable {parent} of {child} with standard name '.format(
parent=local_name_define, child=var_local_name_define)+\
'{standard_name} not defined in host model metadata'.format(
standard_name=var_standard_name))
# Reset local name for entire array to a notation without (:), (:,:), etc.;
# this is needed for the var.print_def_intent() routine to work correctly
parent_var.local_name = local_name_define
# Add variable to dictionary of parent variables, if not already there.
# Set or update intent, depending on whether the variable is an index
# in var_local_name_define or the actual parent of that variable.
if not parent_standard_name in self.parents[ccpp_stage].keys():
self.parents[ccpp_stage][parent_standard_name] = copy.deepcopy(parent_var)
# Copy the intent of the actual variable being processed
if local_name_define == parent_local_name_define:
self.parents[ccpp_stage][parent_standard_name].intent = var.intent
# It's an index for the actual variable being processed --> intent(in)
else:
self.parents[ccpp_stage][parent_standard_name].intent = 'in'
elif self.parents[ccpp_stage][parent_standard_name].intent == 'in':
# Adjust the intent if the actual variable is not intent(in)
if local_name_define == parent_local_name_define and not var.intent == 'in':
self.parents[ccpp_stage][parent_standard_name].intent = 'inout'
# It's an index for the actual variable being processed, intent is ok
#else:
# # nothing to do
elif self.parents[ccpp_stage][parent_standard_name].intent == 'out':
# Adjust the intent if the actual variable is not intent(out)
if local_name_define == parent_local_name_define and not var.intent == 'out':
self.parents[ccpp_stage][parent_standard_name].intent = 'inout'
# Adjust the intent, because the variable is also used as index variable
else:
self.parents[ccpp_stage][parent_standard_name].intent = 'inout'
# Record this information in the local_vars dictionary
local_vars[var_standard_name] = {
'name' : metadata_define[var_standard_name][0].local_name,
'kind' : metadata_define[var_standard_name][0].kind,
'parent_standard_name' : parent_standard_name
}
else:
parent_standard_name = local_vars[var_standard_name]['parent_standard_name']
# Update intent information if necessary
if self.parents[ccpp_stage][parent_standard_name].intent == 'in' and not var.intent == 'in':
self.parents[ccpp_stage][parent_standard_name].intent = 'inout'
elif self.parents[ccpp_stage][parent_standard_name].intent == 'out' and not var.intent == 'out':
self.parents[ccpp_stage][parent_standard_name].intent = 'inout'
# Add necessary actions before/after while populating the subroutine's argument list
kind_string = '_' + local_vars[var_standard_name]['kind'] if local_vars[var_standard_name]['kind'] else ''
if var.actions['out']:
if local_vars[var_standard_name]['name'] in tmpvars.keys():
# If the variable already has a local variable (tmpvar), reuse it
tmpvar = tmpvars[local_vars[var_standard_name]['name']]
else:
# Add a local variable (tmpvar) for this variable
tmpvar_cnt += 1
tmpvar = copy.deepcopy(var)
tmpvar.local_name = 'tmpvar{0}'.format(tmpvar_cnt)
tmpvars[local_vars[var_standard_name]['name']] = tmpvar
if var.rank:
# Add allocate statement if the variable has a rank > 0
actions_before += ' allocate({t}, source={v})\n'.format(t=tmpvar.local_name,
v=local_vars[var_standard_name]['name'])
if var.actions['in']:
# Add unit conversion before entering the subroutine
actions_before += ' {t} = {c}\n'.format(t=tmpvar.local_name,
c=var.actions['in'].format(var=local_vars[var_standard_name]['name'],
kind=kind_string))
# Add unit conversion after returning from the subroutine
actions_after += ' {v} = {c}\n'.format(v=local_vars[var_standard_name]['name'],
c=var.actions['out'].format(var=tmpvar.local_name,
kind=kind_string))
if var.rank:
# Add deallocate statement if the variable has a rank > 0
actions_after += ' deallocate({t})\n'.format(t=tmpvar.local_name)
# Add to argument list
arg = '{local_name}={var_name},'.format(local_name=var.local_name,
var_name=tmpvar.local_name)
elif var.actions['in']:
# Add to argument list, call action in argument list
action = var.actions['in'].format(var=local_vars[var_standard_name]['name'],
kind=kind_string)
arg = '{local_name}={action},'.format(local_name=var.local_name, action=action)
else:
# Add to argument list
arg = '{local_name}={var_name},'.format(local_name=var.local_name,
var_name=local_vars[var_standard_name]['name'])
args += arg
length += len(arg)
# Split args so that lines don't exceed 260 characters (for PGI)
if length > 70 and not var_standard_name == arguments[module_name][scheme_name][subroutine_name][-1]:
args += ' &\n '
length = 0
args = args.rstrip(',')
subroutine_call = '''
{actions_before}
call {subroutine_name}({args})
{actions_after}
'''.format(subroutine_name=subroutine_name, args=args, actions_before=actions_before.rstrip('\n'), actions_after=actions_after.rstrip('\n'))
error_check = '''if ({target_name_flag}/=0) then
write({target_name_msg},'(a)') "An error occured in {subroutine_name}"
ierr={target_name_flag}
return
end if
'''.format(target_name_flag=ccpp_error_flag_target_name, target_name_msg=ccpp_error_msg_target_name, subroutine_name=subroutine_name)
body += '''
{subroutine_call}
{error_check}
'''.format(subroutine_call=subroutine_call, error_check=error_check)
module_use += ' use {m}, only: {s}\n'.format(m=module_name, s=subroutine_name)
if subcycle.loop > 1 and ccpp_stage == 'run':
body += '''
end do
end associate
'''
# Get list of arguments, module use statement and variable definitions for this subroutine (=stage for the group)
(self.arguments[ccpp_stage], sub_module_use, sub_var_defs) = create_arguments_module_use_var_defs(
self.parents[ccpp_stage], metadata_define, tmpvars.values())
sub_argument_list = create_argument_list_wrapped(self.arguments[ccpp_stage])
subroutine = self._suite + '_' + self._name + '_' + ccpp_stage + '_cap'
self._subroutines.append(subroutine)
# Test and set blocks for initialization status
initialized_test_block = Group.initialized_test_blocks[ccpp_stage].format(
target_name_flag=ccpp_error_flag_target_name,
target_name_msg=ccpp_error_msg_target_name,
name=self._name)
initialized_set_block = Group.initialized_set_blocks[ccpp_stage].format(
target_name_flag=ccpp_error_flag_target_name,
target_name_msg=ccpp_error_msg_target_name,
name=self._name)
# Create subroutine
local_subs += Group.sub.format(subroutine=subroutine,
argument_list=sub_argument_list,
module_use='\n '.join(sub_module_use),
initialized_test_block=initialized_test_block,
initialized_set_block=initialized_set_block,
var_defs='\n '.join(sub_var_defs),
body=body)
# Write output to stdout or file
if (self.filename is not sys.stdout):
f = open(self.filename, 'w')
else:
f = sys.stdout
f.write(Group.header.format(group=self._name,
module=self._module,
module_use=module_use,
subroutines=', &\n '.join(self._subroutines)))
f.write(local_subs)
f.write(Group.footer.format(module=self._module))
if (f is not sys.stdout):
f.close()
return
def parse_variable_tables(filename):
"""Parses metadata tables on the host model side that define the available variables.
Metadata tables can refer to variables inside a module or as part of a derived
datatype, which itself is defined inside a module (depending on the location of the
metadata table). Each variable (standard_name) can exist only once, i.e. each entry
(list of variables) in the metadata dictionary contains only one element
(variable = instance of class Var defined in mkcap.py)"""
# Set debug to true if logging level is debug
debug = logging.getLogger().getEffectiveLevel() == logging.DEBUG
# Final metadata container for all variables in file
metadata = collections.OrderedDict()
# Registry of modules and derived data types in file
registry = collections.OrderedDict()
# Read all lines of the file at once
with (open(filename, 'r')) as file:
file_lines = file.readlines()
lines = []
buffer = ''
for i in xrange(len(file_lines)):
line = file_lines[i].rstrip('\n').strip()
# Skip empty lines
if line == '' or line == '&':
continue
# Remove line continuations: concatenate with following lines
if line.endswith('&'):
buffer += file_lines[i].rstrip('\n').replace('&', ' ')
continue
# Write out line with buffer and reset buffer
lines.append(buffer + file_lines[i].rstrip('\n').replace('&', ' '))
buffer = ''
del file_lines
# Find all modules within the file, and save the start and end lines
module_lines = {}
line_counter = 0
for line in lines:
words = line.split()
if len(words) > 1 and words[0].lower() == 'module' and not words[1].lower() == 'procedure':
module_name = words[1].strip()
if module_name in registry.keys():
raise Exception('Duplicate module name {0}'.format(module_name))
registry[module_name] = {}
module_lines[module_name] = { 'startline' : line_counter }
elif len(words) > 1 and words[0].lower() == 'end' and words[1].lower() == 'module':
try:
test_module_name = words[2]
except IndexError:
logging.warning('Encountered closing statement "end module" without module name; assume module_name is {0}'.format(module_name))
test_module_name = module_name
if not module_name == test_module_name:
raise Exception('Module names in opening/closing statement do not match: {0} vs {1}'.format(module_name, test_module_name))
module_lines[module_name]['endline'] = line_counter
line_counter += 1
# Parse each module in the file separately
for module_name in registry.keys():
startline = module_lines[module_name]['startline']
endline = module_lines[module_name]['endline']
line_counter = 0
in_type = False
for line in lines[startline:endline]:
current_line_number = startline + line_counter
words = line.split()
for j in range(len(words)):
# Check for the word 'type', that it is the first word in the line,
# and that a name exists afterwards. It is assumed that definitions
# (not usage) of derived types cannot be nested - reasonable for Fortran.
if words[j].lower() == 'type' and j == 0 and len(words) > 1 and not '(' in words[j+1]:
if in_type:
raise Exception('Nested definitions of derived types not supported')
in_type = True
type_name = words[j+1].split('(')[0].strip()
if type_name in registry[module_name].keys():
raise Exception('Duplicate derived type name {0} in module {1}'.format(
type_name, module_name))
registry[module_name][type_name] = [current_line_number]
elif words[j].lower() == 'type' and j == 1 and words[j-1].lower() == 'end':
if not in_type:
raise Exception('Encountered "end_type" without corresponding "type" statement')
try:
test_type_name = words[j+1]
except IndexError:
logging.warning('Encountered closing statement "end type" without type name; assume type_name is {0}'.format(type_name))
test_type_name = type_name
if not type_name == test_type_name:
raise Exception('Type names in opening/closing statement do not match: {0} vs {1}'.format(type_name, test_type_name))
in_type = False
registry[module_name][type_name].append(current_line_number)
line_counter += 1
logging.debug('Parsing file {0} with registry {1}'.format(filename, registry))
# Variables can either be defined at module-level or in derived types - alongside with their tables
line_counter = 0
in_table = False
in_type = False
for line in lines[startline:endline]:
current_line_number = startline + line_counter
# Check for beginning of new table
words = line.split()
# This is case sensitive
if len(words) > 2 and words[0] in ['!!', '!>'] and '\section' in words[1] and 'arg_table_' in words[2]:
if in_table:
raise Exception('Encountered table start for table {0} while still in table {1}'.format(words[2].replace('arg_table_',''), table_name))
table_name = words[2].replace('arg_table_','')
if not (table_name == module_name or table_name in registry[module_name].keys()):
raise Exception('Encountered table with name {0} without corresponding module or type name'.format(table_name))
in_table = True
header_line_number = current_line_number + 1
line_counter += 1
continue
elif (words[0].startswith('!!') or words[0].startswith('!>')) and '\section' in words[0]:
raise Exception("Malformatted table found in {0} / {1} / {2}".format(filename, module_name, table_name))
# If an argument table is found, parse it
if in_table:
words = line.split('|')
# Separate the table headers
if current_line_number == header_line_number:
# Check for blank table
if len(words) <= 1:
logging.debug('Skipping blank table {0}'.format(table_name))
in_table = False
line_counter += 1
continue
table_header = [x.strip() for x in words[1:-1]]
# Check that only valid table headers are used
for item in table_header:
if not item in VALID_ITEMS['header']:
raise Exception('Invalid column header {0} in argument table {1}'.format(item, table_name))
# Locate mandatory column 'standard_name'
try:
standard_name_index = table_header.index('standard_name')
except ValueError:
raise Exception('Mandatory column standard_name not found in argument table {0}'.format(table_name))
line_counter += 1
continue
elif current_line_number == header_line_number + 1:
# Skip over separator line
line_counter += 1
continue
else:
if len(words) == 1:
# End of table
if words[0].strip() == '!!':
in_table = False
line_counter += 1
continue
else:
raise Exception('Encountered invalid line "{0}" in argument table {1}'.format(line, table_name))
else:
var_items = [x.strip() for x in words[1:-1]]
if not len(var_items) == len(table_header):
raise Exception('Error parsing variable entry "{0}" in argument table {1}'.format(var_items, table_name))
var_name = var_items[standard_name_index]
# Skip variables without a standard_name (i.e. empty cell in column standard_name)
if var_name:
var = Var.from_table(table_header,var_items)
if table_name == module_name:
container = encode_container(module_name)
else:
container = encode_container(module_name, table_name)
var.container = container
# Check for incompatible definitions with CCPP mandatory variables
if var_name in CCPP_MANDATORY_VARIABLES.keys() and not CCPP_MANDATORY_VARIABLES[var_name].compatible(var):
raise Exception('Entry for variable {0}'.format(var_name) + \
' in argument table {0}'.format(table_name) +\
' is incompatible with mandatory variable:\n' +\
' existing: {0}\n'.format(CCPP_MANDATORY_VARIABLES[var_name].print_debug()) +\
' vs. new: {0}'.format(var.print_debug()))
# Add variable to metadata dictionary
if not var_name in metadata.keys():
metadata[var_name] = [var]
else:
for existing_var in metadata[var_name]:
if not existing_var.compatible(var):
raise Exception('New entry for variable {0}'.format(var_name) + \
' in argument table {0}'.format(table_name) +\
' is incompatible with existing entry:\n' +\
' existing: {0}\n'.format(existing_var.print_debug()) +\
' vs. new: {0}'.format(var.print_debug()))
metadata[var_name].append(var)
#else:
# logging.debug('Skipping variable entry "{0}" without a standard_name'.format(var_items))
line_counter += 1
# Informative output to screen
if debug and len(metadata.keys()) > 0:
for module_name in registry.keys():
logging.debug('Module name: {0}'.format(module_name))
container = encode_container(module_name)
vars_in_module = []
for var_name in metadata.keys():
for var in metadata[var_name]:
if var.container == container:
vars_in_module.append(var_name)
logging.debug('Module variables: {0}'.format(', '.join(vars_in_module)))
for type_name in registry[module_name].keys():
container = encode_container(module_name, type_name)
vars_in_type = []
for var_name in metadata.keys():
for var in metadata[var_name]:
if var.container == container:
vars_in_type.append(var_name)
logging.debug('Variables in derived type {0}: {1}'.format(type_name, ', '.join(vars_in_type)))
if debug and len(metadata.keys()) > 0:
# Write out the XML for debugging purposes
top = ET.Element('definition')
top.set('module', module_name)
container = encode_container(module_name)
for var_name in metadata.keys():
for var in metadata[var_name]:
if var.container == container:
sub_var = var.to_xml(ET.SubElement(top, 'variable'))
for type_name in registry[module_name].keys():
container = encode_container(module_name, type_name)
sub_type = ET.SubElement(top, 'type')
sub_type.set('type_name', type_name)
for var_name in metadata.keys():
for var in metadata[var_name]:
if var.container == container:
sub_var = var.to_xml(ET.SubElement(sub_type, 'variable'))
indent(top)
tree = ET.ElementTree(top)
xmlfile = module_name + '.xml'
tree.write(xmlfile, xml_declaration=True, encoding='utf-8', method="xml")
logging.info('Parsed variable definition tables in module {0}; output => {1}'.format(module_name, xmlfile))
elif len(metadata.keys()) > 0:
logging.info('Parsed variable definition tables in module {0}'.format(module_name))
return metadata
def read_new_metadata(filename,
module_name,
table_name,
scheme_name=None,
subroutine_name=None):
"""Read metadata in new format and convert output to ccpp_prebuild metadata dictionary"""
if not os.path.isfile(filename):
raise Exception("New metadata file {0} not found".format(filename))
# Save metadata, because this routine new_metadata
# is called once for every table in that file
if filename in NEW_METADATA_SAVE.keys():
new_metadata_headers = NEW_METADATA_SAVE[filename]
else:
new_metadata_headers = MetadataHeader.parse_metadata_file(filename)
NEW_METADATA_SAVE[filename] = new_metadata_headers
# Convert new metadata for requested table to old metadata dictionary
metadata = collections.OrderedDict()
for new_metadata_header in new_metadata_headers:
if not scheme_name:
if not new_metadata_header.title == table_name:
# Skip this table, since it is not requested right now
continue
if new_metadata_header.title == module_name:
container = encode_container(module_name)
else:
container = encode_container(module_name,
new_metadata_header.title)
else:
if not new_metadata_header.title == table_name:
# Skip this table, since it is not requested right now
continue
container = encode_container(module_name, scheme_name, table_name)
for new_var in new_metadata_header.variable_list():
standard_name = new_var.get_prop_value('standard_name')
# DH* 2020-05-26
# Legacy extension for inconsistent metadata (use of horizontal_dimension versus horizontal_loop_extent).
# Since horizontal_dimension and horizontal_loop_extent have the same attributes (otherwise it doesn't
# make sense), we swap the standard name and add a note to the long name
legacy_note = ''
if standard_name == 'horizontal_loop_extent' and scheme_name and \
(table_name.endswith("_init") or table_name.endswith("finalize")):
logging.warn("Legacy extension - replacing variable 'horizontal_loop_extent'" + \
" with 'horizontal_dimension' in table {}".format(table_name))
standard_name = 'horizontal_dimension'
legacy_note = ' replaced by horizontal dimension (legacy extension)'
elif standard_name == 'horizontal_dimension' and scheme_name and table_name.endswith(
"_run"):
logging.warn("Legacy extension - replacing variable 'horizontal_dimension' " + \
"with 'horizontal_loop_extent' in table {}".format(table_name))
standard_name = 'horizontal_loop_extent'
legacy_note = ' replaced by horizontal loop extent (legacy extension)'
# Adjust dimensions
dimensions = new_var.get_prop_value('dimensions')
if scheme_name and (table_name.endswith("_init") or table_name.endswith("finalize")) \
and 'horizontal_loop_extent' in dimensions:
logging.warn("Legacy extension - replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \
"for variable {} in table {}".format(standard_name,table_name))
dimensions = [
'horizontal_dimension'
if x == 'horizontal_loop_extent' else x for x in dimensions
]
elif scheme_name and table_name.endswith(
"_run") and 'horizontal_dimension' in dimensions:
logging.warn("Legacy extension - replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \
"for variable {} in table {}".format(standard_name,table_name))
dimensions = [
'horizontal_loop_extent'
if x == 'horizontal_dimension' else x for x in dimensions
]
# *DH 2020-05-26
if new_var.get_prop_value('active').lower() == '.true.':
active = 'T'
elif new_var.get_prop_value('active').lower() == '.false.':
active = 'F'
else:
# Replace multiple whitespaces and use lowercase throughout
active = ' '.join(
new_var.get_prop_value('active').lower().split())
var = Var(
standard_name=standard_name,
long_name=new_var.get_prop_value('long_name') + legacy_note,
units=new_var.get_prop_value('units'),
local_name=new_var.get_prop_value('local_name'),
type=new_var.get_prop_value('type'),
dimensions=dimensions,
container=container,
kind=new_var.get_prop_value('kind'),
intent=new_var.get_prop_value('intent'),
optional='T' if new_var.get_prop_value('optional') else 'F',
active=active,
)
# Check for duplicates in same table
if standard_name in metadata.keys():
raise Exception(
"Error, multiple definitions of standard name {0} in new metadata table {1}"
.format(standard_name, table_name))
metadata[standard_name] = [var]
return metadata
def write(self, metadata_request, metadata_define, arguments,
ccpp_field_maps, module_use_cap):
# First get target names of standard CCPP variables for subcycling and error handling
ccpp_loop_counter_target_name = metadata_request[CCPP_LOOP_COUNTER][
0].target
ccpp_error_flag_target_name = metadata_request[
CCPP_ERROR_FLAG_VARIABLE][0].target
ccpp_error_msg_target_name = metadata_request[CCPP_ERROR_MSG_VARIABLE][
0].target
#
module_use = module_use_cap
self._module = 'ccpp_group_{name}_cap'.format(name=self._name)
self._filename = '{module_name}.F90'.format(module_name=self._module)
self._subroutines = []
local_subs = ''
#
for ccpp_stage in CCPP_STAGES:
if self._init and not ccpp_stage == 'init':
continue
elif self._finalize and not ccpp_stage == 'finalize':
continue
# For creating and mappig local variable names to standard names
local_vars = {}
local_var_cnt = 0
local_var_cnt_max = 9999
local_var_template = 'v{x:04d}'
# Add the predefined variables for subcycling and error handling (special, no need to retrieve from cdata via ccpp_field_get)
local_vars[CCPP_LOOP_COUNTER] = ccpp_loop_counter_target_name
local_vars[CCPP_ERROR_FLAG_VARIABLE] = ccpp_error_flag_target_name
local_vars[CCPP_ERROR_MSG_VARIABLE] = ccpp_error_msg_target_name
#
body = ''
var_defs = ''
var_gets = ''
for subcycle in self._subcycles:
if subcycle.loop > 1 and ccpp_stage == 'run':
body += '''
associate(cnt => {loop_var_name})
do cnt=1,{loop_cnt}\n\n'''.format(
loop_var_name=ccpp_loop_counter_target_name,
loop_cnt=subcycle.loop)
for scheme_name in subcycle.schemes:
module_name = scheme_name
subroutine_name = scheme_name + '_' + ccpp_stage
container = encode_container(module_name, scheme_name,
subroutine_name)
# Skip entirely empty routines
if not arguments[module_name][scheme_name][subroutine_name]:
continue
error_check = ''
args = ''
length = 0
for var_name in arguments[module_name][scheme_name][
subroutine_name]:
for var in metadata_request[var_name]:
if container == var.container:
break
if not var_name in local_vars:
local_var_cnt += 1
if local_var_cnt > local_var_cnt_max:
raise Exception(
"local_var_cnt exceeding local_var_cnt_max, increase limit!"
)
tmpvar = copy.deepcopy(var)
tmpvar.local_name = local_var_template.format(
x=local_var_cnt) #var_name.replace('-','_')
var_defs += ' ' + tmpvar.print_def() + '\n'
# Use the index lookup from ccpp_field_maps for the group's
# physics set and given variable name (standard name)
var_gets += tmpvar.print_get(index=ccpp_field_maps[
self._pset][var_name]) + '\n'
# Add to list of local variables with the auto-generated local variable name
local_vars[var_name] = tmpvar.local_name
del tmpvar
# Add to argument list
arg = '{local_name}={var_name},'.format(
local_name=var.local_name,
var_name=local_vars[var_name]
) #var_name.replace('-','_'))
args += arg
length += len(arg)
# Split args so that lines don't exceed 260 characters (for PGI)
if length > 70 and not var_name == arguments[
module_name][scheme_name][subroutine_name][-1]:
args += ' &\n '
length = 0
args = args.rstrip(',')
subroutine_call = 'call {subroutine_name}({args})'.format(
subroutine_name=subroutine_name, args=args)
error_check = '''if ({target_name_flag}/=0) then
write({target_name_msg},'(a)') "An error occured in {subroutine_name}"
ierr={target_name_flag}
return
end if
'''.format(target_name_flag=ccpp_error_flag_target_name,
target_name_msg=ccpp_error_msg_target_name,
subroutine_name=subroutine_name)
body += '''
{subroutine_call}
{error_check}
'''.format(subroutine_call=subroutine_call, error_check=error_check)
module_use += ' use {m}, only: {s}\n'.format(
m=module_name, s=subroutine_name)
if subcycle.loop > 1 and ccpp_stage == 'run':
body += '''
end do
end associate
'''
subroutine = self._name + '_' + ccpp_stage + '_cap'
self._subroutines.append(subroutine)
# Test and set blocks for initialization status
initialized_test_block = Group.initialized_test_blocks[
ccpp_stage].format(
target_name_flag=ccpp_error_flag_target_name,
target_name_msg=ccpp_error_msg_target_name,
name=self._name)
initialized_set_block = Group.initialized_set_blocks[
ccpp_stage].format(
target_name_flag=ccpp_error_flag_target_name,
target_name_msg=ccpp_error_msg_target_name,
name=self._name)
# Create subroutine
local_subs += Group.sub.format(
subroutine=subroutine,
initialized_test_block=initialized_test_block,
initialized_set_block=initialized_set_block,
var_defs=var_defs,
var_gets=var_gets,
body=body)
# Write output to stdout or file
if (self.filename is not sys.stdout):
f = open(self.filename, 'w')
else:
f = sys.stdout
f.write(
Group.header.format(group=self._name,
module=self._module,
module_use=module_use,
subroutines=','.join(self._subroutines)))
f.write(local_subs)
f.write(Group.footer.format(module=self._module))
if (f is not sys.stdout):
f.close()
return
def read_new_metadata(filename,
module_name,
table_name,
scheme_name=None,
subroutine_name=None):
"""Read metadata in new format and convert output to ccpp_prebuild metadata dictionary"""
if not os.path.isfile(filename):
raise Exception("New metadata file {0} not found".format(filename))
# Save metadata, because this routine new_metadata
# is called once for every table in that file
if filename in NEW_METADATA_SAVE.keys():
new_metadata_headers = NEW_METADATA_SAVE[filename]
else:
new_metadata_headers = parse_metadata_file(
filename,
known_ddts=registered_fortran_ddt_names(),
logger=logging.getLogger(__name__))
NEW_METADATA_SAVE[filename] = new_metadata_headers
# Record dependencies for the metadata table (only applies to schemes)
dependencies = []
# Convert new metadata for requested table to old metadata dictionary
metadata = collections.OrderedDict()
for new_metadata_header in new_metadata_headers:
for metadata_section in new_metadata_header.sections():
# Module or DDT tables
if not scheme_name:
# Module property tables
if not metadata_section.title == table_name:
# Skip this table, since it is not requested right now
continue
# Distinguish between module argument tables and DDT argument tables
if metadata_section.title == module_name:
container = encode_container(module_name)
else:
container = encode_container(module_name,
metadata_section.title)
# Add to dependencies
if new_metadata_header.relative_path:
dependencies += [
os.path.join(new_metadata_header.relative_path, x)
for x in new_metadata_header.dependencies
]
else:
dependencies += new_metadata_header.dependencies
else:
# Scheme property tables
if not metadata_section.title == table_name:
# Skip this table, since it is not requested right now
continue
container = encode_container(module_name, scheme_name,
table_name)
# Add to dependencies
if new_metadata_header.relative_path:
dependencies += [
os.path.join(new_metadata_header.relative_path, x)
for x in new_metadata_header.dependencies
]
else:
dependencies += new_metadata_header.dependencies
for new_var in metadata_section.variable_list():
standard_name = new_var.get_prop_value('standard_name')
# DH* 2020-05-26
# Legacy extension for inconsistent metadata (use of horizontal_dimension versus horizontal_loop_extent).
# Since horizontal_dimension and horizontal_loop_extent have the same attributes (otherwise it doesn't
# make sense), we swap the standard name and add a note to the long name - 2021-05-26: this is now an error.
legacy_note = ''
if standard_name == 'horizontal_loop_extent' and scheme_name and \
(table_name.endswith("_init") or table_name.endswith("_finalize")):
#logging.warn("Legacy extension - replacing variable 'horizontal_loop_extent'" + \
# " with 'horizontal_dimension' in table {}".format(table_name))
#standard_name = 'horizontal_dimension'
#legacy_note = ' replaced by horizontal dimension (legacy extension)'
raise Exception("Legacy extension DISABLED: replacing variable 'horizontal_loop_extent'" + \
" with 'horizontal_dimension' in table {}".format(table_name))
elif standard_name == 'horizontal_dimension' and scheme_name and table_name.endswith(
"_run"):
#logging.warn("Legacy extension - replacing variable 'horizontal_dimension'" + \
# " with 'horizontal_loop_extent' in table {}".format(table_name))
#standard_name = 'horizontal_loop_extent'
#legacy_note = ' replaced by horizontal loop extent (legacy extension)'
raise Exception("Legacy extension DISABLED: replacing variable 'horizontal_dimension'" + \
" with 'horizontal_loop_extent' in table {}".format(table_name))
# Adjust dimensions
dimensions = new_var.get_prop_value('dimensions')
if scheme_name and (table_name.endswith("_init") or table_name.endswith("_finalize")) \
and 'horizontal_loop_extent' in dimensions:
#logging.warn("Legacy extension - replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \
# "for variable {} in table {}".format(standard_name,table_name))
#dimensions = ['horizontal_dimension' if x=='horizontal_loop_extent' else x for x in dimensions]
raise Exception("Legacy extension DISABLED: replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \
"for variable {} in table {}".format(standard_name,table_name))
elif scheme_name and table_name.endswith(
"_run") and 'horizontal_dimension' in dimensions:
#logging.warn("Legacy extension - replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \
# "for variable {} in table {}".format(standard_name,table_name))
#dimensions = ['horizontal_loop_extent' if x=='horizontal_dimension' else x for x in dimensions]
raise Exception("Legacy extension DISABLED: replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \
"for variable {} in table {}".format(standard_name,table_name))
elif not scheme_name and 'horizontal_dimension' in dimensions:
raise Exception("Legacy extension DISABLED: replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \
"for variable {} in table {}".format(standard_name,table_name))
# *DH 2020-05-26
if not new_var.get_prop_value('active'):
# If it doesn't have an active attribute, then the variable is always active (default)
active = 'T'
elif new_var.get_prop_value('active').lower() == '.true.':
active = 'T'
elif new_var.get_prop_value(
'active') and new_var.get_prop_value(
'active').lower() == '.false.':
active = 'F'
else:
# Replace multiple whitespaces, preserve case
active = ' '.join(new_var.get_prop_value('active').split())
# DH* 20210812
# Workaround for Fortran DDTs incorrectly having the type of
# the DDT copied into the kind attribute in parse_metadata_file
if new_var.is_ddt() and new_var.get_prop_value('kind'):
kind = ''
else:
kind = new_var.get_prop_value('kind')
#kind = new_var.get_prop_value('kind')
# *DH 20210812
var = Var(
standard_name=standard_name,
long_name=new_var.get_prop_value('long_name') +
legacy_note,
units=new_var.get_prop_value('units'),
local_name=new_var.get_prop_value('local_name'),
type=new_var.get_prop_value('type').lower(),
dimensions=dimensions,
container=container,
kind=kind,
intent=new_var.get_prop_value('intent'),
optional='T'
if new_var.get_prop_value('optional') else 'F',
active=active,
)
# Check for duplicates in same table
if standard_name in metadata.keys():
raise Exception(
"Error, multiple definitions of standard name {} in new metadata table {}"
.format(standard_name, table_name))
metadata[standard_name] = [var]
return (metadata, dependencies)
