def parse_preamble_data(statements, pobj, spec_name, endmatch, logger): """Parse module variables or DDT definitions from a module preamble or parse program variables from the beginning of a program. """ inspec = True mheaders = list() var_dict = VarDictionary(spec_name) psrc = ParseSource(spec_name, 'MODULE', pobj) active_table = None while inspec and (statements is not None): while len(statements) > 0: statement = statements.pop(0) # End program or module pmatch = endmatch.match(statement) asmatch = arg_table_start_re.match(statement) type_def = fortran_type_definition(statement) if asmatch is not None: active_table = asmatch.group(1).lower() elif (pmatch is not None) or is_contains_statement( statement, inspec): # We are done with the specification inspec = False if len(var_dict.variable_list()) > 0: mheader = MetadataHeader(title=spec_name, type_in='MODULE', module=spec_name, var_dict=var_dict, logger=logger) mheaders.append(mheader) # End if break elif (type_def is not None) and (type_def[0].lower() == active_table): statements, ddt = parse_type_def(statements, type_def, spec_name, pobj, logger) mheaders.append(ddt) active_table = None else: # We should have a variable definition to add if ((not is_comment_statement(statement, logger)) and (not parse_use_statement({}, statement, pobj, logger)) and (active_table == spec_name)): vars = parse_fortran_var_decl(statement, psrc, logger=logger) for var in vars: var_dict.add_variable(var) # End for # End if # End if # End while if inspec and (len(statements) == 0): statements = read_statements(pobj) # End if # End while return statements, mheaders
def _new_var(self, standard_name, units, dimensions, vtype, vkind=''): """Create and return a new Var object with the requested properties""" context = ParseContext(linenum=self.__linenum, filename="foo.meta") source = ParseSource("foo", "host", context) prop_dict = {'local_name' : f"foo{self.__linenum}", 'standard_name' : standard_name, 'units' : units, 'dimensions' : f"({', '.join(dimensions)})", 'type' : vtype, 'kind' : vkind} self.__linenum += 5 return Var(prop_dict, source, self.__run_env)
def parse_type_def(statements, type_def, mod_name, pobj, logger): """Parse a type definition from <statements> and return the remaining statements along with a MetadataTable object representing the type's variables.""" psrc = ParseSource(mod_name, 'ddt', pobj) seen_contains = False mheader = None var_dict = VarDictionary(type_def[0]) inspec = True while inspec and (statements is not None): while len(statements) > 0: statement = statements.pop(0) # End program or module pmatch = _END_TYPE_RE.match(statement) if pmatch is not None: # We hit the end of the type, make a header mheader = MetadataTable(table_name_in=type_def[0], table_type_in='ddt', module=mod_name, var_dict=var_dict, logger=logger) inspec = False elif is_contains_statement(statement, inspec): seen_contains = True elif not seen_contains: # Comment of variable if ((not is_comment_statement(statement)) and (not parse_use_statement(statement, logger))): dvars = parse_fortran_var_decl(statement, psrc, logger=logger) for var in dvars: var_dict.add_variable(var) # End for # End if else: # We are just skipping lines until the end type pass # End if # End while if inspec and (len(statements) == 0): statements = read_statements(pobj) # End if # End while return statements, mheader
def parse_type_def(statements, type_def, mod_name, pobj, logger): psrc = ParseSource(mod_name, 'DDT', pobj) seen_contains = False mheader = None var_dict = VarDictionary(type_def[0]) inspec = True while inspec and (statements is not None): while len(statements) > 0: statement = statements.pop(0) # End program or module pmatch = end_type_re.match(statement) if pmatch is not None: # We hit the end of the type, make a header mheader = MetadataHeader(title=type_def[0], type_in='DDT', module=mod_name, var_dict=var_dict, logger=logger) inspec = False elif is_contains_statement(statement, inspec): seen_contains = True elif not seen_contains: # Comment of variable if ((not is_comment_statement(statement, logger)) and (not parse_use_statement({}, statement, pobj, logger))): vars = parse_fortran_var_decl(statement, psrc, logger=logger) for var in vars: var_dict.add_variable(var) # End for # End if else: # We are just skipping lines until the end type pass # End if # End while if inspec and (len(statements) == 0): statements = read_statements(pobj) # End if # End while return statements, mheader
from parse_tools import ParseObject, ParseSource, ParseContext ############################################################################### _HEADER = "cap for {host_model} calls to CCPP API" _SUBHEAD = ''' subroutine {host_model}_ccpp_physics_{stage}({api_vars}) ''' _SUBFOOT = ''' end subroutine {host_model}_ccpp_physics_{stage} ''' _API_SRC_NAME = "CCPP_API" _API_SOURCE = ParseSource(_API_SRC_NAME, "MODULE", ParseContext(filename="host_cap.F90")) _SUITE_NAME_VAR = Var({'local_name':'suite_name', 'standard_name':'suite_name', 'intent':'in', 'type':'character', 'kind':'len=*', 'units':'', 'protected':'True', 'dimensions':'()'}, _API_SOURCE) _SUITE_PART_VAR = Var({'local_name':'suite_part', 'standard_name':'suite_part', 'intent':'in', 'type':'character', 'kind':'len=*', 'units':'', 'protected':'True', 'dimensions':'()'}, _API_SOURCE) # Used to prevent loop substitution lookups _BLANK_DICT = VarDictionary(_API_SRC_NAME)
class VarDictionary(OrderedDict): """ A class to store and cross-check variables from one or more metadata headers. The class also serves as a scoping construct so that a variable can be found in an innermost available scope. The dictionary is organized by standard_name. It is an error to try to add a variable if its standard name is already in the dictionary. Scoping is a tree of VarDictionary objects. >>> VarDictionary('foo') VarDictionary(foo) >>> VarDictionary('bar', variables={}) VarDictionary(bar) >>> VarDictionary('baz', Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm/s', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()))) #doctest: +ELLIPSIS VarDictionary(baz, [('hi_mom', <__main__.Var hi_mom: foo at 0x...>)]) >>> print("{}".format(VarDictionary('baz', Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm/s', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()))))) VarDictionary(baz, ['hi_mom']) >>> VarDictionary('qux', [Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm/s', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()))]) #doctest: +ELLIPSIS VarDictionary(qux, [('hi_mom', <__main__.Var hi_mom: foo at 0x...>)]) >>> VarDictionary('boo').add_variable(Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm/s', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()))) >>> VarDictionary('who', variables=[Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm/s', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()))]).prop_list('local_name') ['foo'] >>> VarDictionary('glitch', Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm/s', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()))).add_variable(Var({'local_name' : 'bar', 'standard_name' : 'hi_mom', 'units' : 'm/s', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname2', 'DDT', ParseContext()))) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ParseSyntaxError: Invalid Duplicate standard name, 'hi_mom', at <standard input>: """ # Loop variables __ccpp_loop_vars__ = ['horizontal_loop_begin', 'horizontal_loop_end', 'thread_block_number', 'horizontal_loop_extent'] # Loop substitutions __ccpp_loop_subst__ = {'horizontal_loop_extent' : ('horizontal_loop_begin', 'horizontal_loop_end'), 'thread_block_begin:thread_block_end' : 'thread_block_number'} # Dimension substitutions __ccpp_dim_subst__ = {'horizontal_loop_extent' : 'horizontal_dimension'} # Variable representing the constant integer, 1 __var_one = Var({'local_name' : 'ccpp_one', 'constant' : 'True', 'standard_name' : 'ccpp_constant_one', 'units' : '1', 'dimensions' : '()', 'type' : 'integer'}, ParseSource('VarDictionary', 'REGISTRY', ParseContext())) def __init__(self, name, variables=None, parent_dict=None, logger=None): "Unlike dict, VarDictionary only takes a Var or Var list" super(VarDictionary, self).__init__() self._name = name self._logger = logger self._parent_dict = parent_dict if parent_dict is not None: parent_dict.add_sub_scope(self) # End if self._sub_dicts = list() if isinstance(variables, Var): self.add_variable(variables) elif isinstance(variables, list): for var in variables: self.add_variable(var) # End for elif isinstance(variables, VarDictionary): for stdname in variables.keys(): self[stdname] = variables[stdname] # End for elif isinstance(variables, dict): # variables will not be in 'order', but we accept them anyway for stdname in variables.keys(): self[stdname] = variables[stdname] # End for elif variables is not None: raise ParseInternalError('Illegal type for variables, {} in {}'.format(type(variables), self.name)) # End if @property def name(self): return self._name @property def parent(self): return self._parent_dict def include_var_in_list(self, var, std_vars, loop_vars, consts): '''Return True iff <var> is of a type allowed by the logicals, <std_vars> (not constants or loop_vars), <loop_vars> a variable ending in "_extent", "_begin", "_end", or <consts> a variable with the "constant" property. ''' const_val = var.get_prop_value('constant') const_var = Var.get_prop('constant').valid_value(const_val) include_var = consts and const_var if not include_var: standard_name = var.get_prop_value('standard_name') loop_var = VarDictionary.loop_var_match(standard_name) include_var = loop_var and loop_vars if not include_var: std_var = not (loop_var or const_var) include_var = std_vars and std_var # End if # End if return include_var def variable_list(self, recursive=False, std_vars=True, loop_vars=True, consts=True): "Return a list of all variables" if recursive and (self._parent_dict is not None): vlist = self._parent_dict.variable_list(recursive=recursive, std_vars=std_vars, loop_vars=loop_vars, consts=consts) else: vlist = list() # End if for sn in self.keys(): var = self[sn] if self.include_var_in_list(var, std_vars=std_vars, loop_vars=loop_vars, consts=consts): vlist.append(var) # End if # End for return vlist def add_variable(self, newvar, exists_ok=False): """Add a variable if it does not conflict with existing entries""" standard_name = newvar.get_prop_value('standard_name') if (standard_name in self) and (not exists_ok): # We already have a matching variable, error! if self._logger is not None: self._logger.error("Attempt to add duplicate variable, {} from {}".format(standard_name, newvar.source.name)) # End if raise ParseSyntaxError("Duplicate standard name in {}".format(self.name), token=standard_name, context=newvar._context) # End if cvar = self.find_variable(standard_name) if (cvar is not None) and (not cvar.compatible(newvar, self._logger)): if self._logger is not None: self._logger.error("Attempt to add incompatible variable, {} from {}".format(standard_name, newvar.source.name)) # End if errstr = "standard name, incompatible with {}" raise ParseSyntaxError(errstr.format(cvar.context), token=standard_name, context=newvar.source.context) # End if # If we make it to here without an exception, add the variable self[standard_name] = newvar def remove_variable(self, standard_name): """Remove <standard_name> from the dictionary. Ignore if <standard_name> is not in dict """ if standard_name in self: del self[standard_name] # End if def find_variable(self, standard_name, any_scope=True, loop_subst=False): """Return the variable matching <standard_name> or None If any_scope is True, search parent scopes if not in current scope. """ if standard_name in self: var = self[standard_name] elif any_scope and (self._parent_dict is not None): var = self._parent_dict.find_variable(standard_name, any_scope) else: var = None # End if if (var is None) and loop_subst: var = self.find_loop_subst(standard_name, any_scope=any_scope) # End if return var def add_sub_scope(self, sub_dict): 'Add a child dictionary to enable traversal' self._sub_dicts.append(sub_dict) def prop_list(self, prop_name, std_vars=True, loop_vars=True, consts=True): '''Return a list of the <prop_name> property for each variable. std_vars are variables which are neither constants nor loop variables. ''' plist = list() for standard_name in self.keys(): var = self.find_variable(standard_name, any_scope=False, loop_subst=False) if self.include_var_in_list(var, std_vars=std_vars, loop_vars=loop_vars, consts=consts): plist.append(self[standard_name].get_prop_value(prop_name)) # End if # End for return plist def declare_variables(self, outfile, indent, std_vars=True, loop_vars=True, consts=True): "Write out the declarations for this dictionary's variables" for standard_name in self.keys(): var = self.find_variable(standard_name, any_scope=False, loop_subst=False) if self.include_var_in_list(var, std_vars=std_vars, loop_vars=loop_vars, consts=consts): self[standard_name].write_def(outfile, indent, self) # End if # End for def merge(self, other_dict): "Add new entries from <other_dict>" for ovar in other_dict.variable_list(): self.add_variable(ovar) # End for def __str__(self): return "VarDictionary({}, {})".format(self.name, self.keys()) def __repr__(self): srepr = super(VarDictionary, self).__repr__() vstart = len("VarDictionary") + 1 if len(srepr) > vstart + 1: comma = ", " else: comma = "" # End if return "VarDictionary({}{}{}".format(self.name, comma, srepr[vstart:]) def __del__(self): try: for key in self.keys(): del self[key] # End for except Exception as e: pass # python does not guarantee object state during finalization # End try @classmethod def loop_var_match(cls, standard_name): 'Return True iff <standard_name> is a loop variable' return standard_name in cls.__ccpp_loop_vars__ @classmethod def loop_subst_match(cls, standard_name): 'Return a loop substitution match, if any, for <standard_name>' if standard_name in cls.__ccpp_loop_subst__: return cls.__ccpp_loop_subst__[standard_name] else: return None # End if def find_loop_subst(self, standard_name, any_scope=True, context=None): """If <standard_name> is of the form <standard_name>_extent and that variable is not in the dictionary, substitute a tuple of variables, (<standard_name>_begin, <standard_name>_end), if those variables are in the dictionary. If <standard_name>_extent *is* present, return that variable as a range, (__var_one, <standard_name>_extent) In other cases, return None """ loop_var = VarDictionary.loop_subst_match(standard_name) logger_str = None if loop_var is not None: # Let us see if we can fix a loop variable dict_var = self.find_variable(standard_name, any_scope=any_scope, loop_subst=False) if dict_var is not None: my_var = (VarDictionary.__var_one, dict_var) if self._logger is not None: logger_str = "loop_subst: found {}{}".format(standard_name, context_string(context)) # End if else: my_vars = [self.find_variable(x) for x in loop_var] if None not in my_vars: my_var = tuple(my_vars) if self._logger is not None: names = [x.get_prop_value('local_name') for x in my_vars] logger_str = "loop_subst: {} ==> (){}".format(standard_name, ', '.join(names), context_string(context)) # End if else: if self._logger is not None: logger_str = "loop_subst: {} ==> ({}, {}) FAILED{}".format(standard_name, beg_name, end_name, context_string(context)) # End if my_var = None # End if # End if else: if self._logger is not None: logger_str = "loop_subst: {} is not a loop variable{}".format(standard_name, context_string(context)) # End if my_var = None # End if if logger_str is not None: self._logger.debug(logger_str) # End if return my_var def find_dimension_subst(self, standard_name, any_scope=True, context=None): """If <standard_name> is of the form <standard_name>_loop_extent attempt to find a variable of the form <standard_name>_dimension and return that. If such a variable is not found, raise an exception. If <standard_name> is not of the form <standard_name>_extent, return None. """ loop_var = standard_name in VarDictionary.__ccpp_dim_subst__ logger_str = None if loop_var: # Let us see if we can replace the variable dim_name = VarDictionary.__ccpp_dim_subst__[standard_name] my_var = self.find_variable(dim_name, any_scope=any_scope) if my_var is None: raise CCPPError("Dimension variable, {} not found{}".format(dim_name, context_string(context))) # End if else: my_var = None # End if return my_var
def parse_scheme_metadata(statements, pobj, spec_name, table_name, logger): "Parse dummy argument information from a subroutine" psrc = None mheader = None var_dict = None scheme_name = None # Find the subroutine line, should be first executable statement inpreamble = False insub = True while insub and (statements is not None): while len(statements) > 0: statement = statements.pop(0) smatch = subroutine_re.match(statement) esmatch = end_subroutine_re.match(statement) pmatch = endmodule_re.match(statement) asmatch = arg_table_start_re.match(statement) if asmatch is not None: # We have run off the end of something, hope that is okay # Put this statement back for the caller to deal with statements.insert(0, statement) insub = False break elif (pmatch is not None): # We have run off the end of the module, hope that is okay pobj.leave_region('MODULE', region_name=spec_name) insub = False break elif smatch is not None: scheme_name = smatch.group(1) inpreamble = scheme_name == table_name if inpreamble: if smatch.group(2) is not None: scheme_args = [ x.strip().lower() for x in smatch.group(2).split(',') ] else: scheme_args = list() # End if scheme_set = set(scheme_args) var_dict = VarDictionary(scheme_name) psrc = ParseSource(scheme_name, 'SCHEME', pobj) # End if elif inpreamble: # Process a preamble statement (use or argument declaration) if esmatch is not None: inpreamble = False insub = False elif ((not is_comment_statement(statement, logger)) and (not parse_use_statement({}, statement, pobj, logger)) and ('intent' in statement.lower())): vars = parse_fortran_var_decl(statement, psrc, logger=logger) for var in vars: lname = var.get_prop_value('local_name').lower() if lname in scheme_set: scheme_set.remove(lname) else: raise ParseSyntaxError('dummy argument', token=lname, context=pobj) # End if var_dict.add_variable(var) # End for # End if # End if # End while if insub and (len(statements) == 0): statements = read_statements(pobj) # End if # End while if (scheme_name is not None) and (var_dict is not None): mheader = MetadataHeader(title=scheme_name, type_in='SCHEME', module=spec_name, var_dict=var_dict, logger=logger) # End if return statements, mheader
def parse_scheme_metadata(statements, pobj, spec_name, table_name, logger): "Parse dummy argument information from a subroutine" psrc = None mheader = None var_dict = None scheme_name = None # Find the subroutine line, should be first executable statement inpreamble = False insub = True if logger is not None: ctx = context_string(pobj, nodir=True) msg = "Parsing specification of {}{}" logger.debug(msg.format(table_name, ctx)) # End if ctx = context_string(pobj) # Save initial context with directory vdict = None # Initialized when we parse the subroutine arguments while insub and (statements is not None): while statements: statement = statements.pop(0) smatch = _SUBROUTINE_RE.match(statement) esmatch = _END_SUBROUTINE_RE.match(statement) pmatch = _ENDMODULE_RE.match(statement) asmatch = _ARG_TABLE_START_RE.match(statement) if asmatch is not None: # We have run off the end of something, hope that is okay # Put this statement back for the caller to deal with statements.insert(0, statement) insub = False break # End if if pmatch is not None: # We have run off the end of the module, hope that is okay pobj.leave_region('MODULE', region_name=spec_name) insub = False break # End if if smatch is not None: scheme_name = smatch.group(1) inpreamble = scheme_name.lower() == table_name.lower() if inpreamble: if smatch.group(2) is not None: smstr = smatch.group(2).strip() if len(smstr) > 0: smlist = smstr.strip().split(',') else: smlist = list() # End if scheme_args = [x.strip().lower() for x in smlist] else: scheme_args = list() # End if # Create a dict template with all the scheme's arguments # in the correct order vdict = OrderedDict() for arg in scheme_args: if len(arg) == 0: errmsg = 'Empty argument{}' raise ParseInternalError(errmsg.format(pobj)) # End if if arg in vdict: errmsg = 'Duplicate dummy argument, {}' raise ParseSyntaxError(errmsg.format(arg), context=pobj) # End if vdict[arg] = None # End for psrc = ParseSource(scheme_name, 'scheme', pobj) # End if elif inpreamble: # Process a preamble statement (use or argument declaration) if esmatch is not None: inpreamble = False insub = False elif ((not is_comment_statement(statement)) and (not parse_use_statement(statement, logger)) and is_dummy_argument_statement(statement)): dvars = parse_fortran_var_decl(statement, psrc, logger=logger) for var in dvars: lname = var.get_prop_value('local_name').lower() if lname in vdict: if vdict[lname] is not None: emsg = "Error: duplicate dummy argument, {}" raise ParseSyntaxError(emsg.format(lname), context=pobj) # End if vdict[lname] = var else: raise ParseSyntaxError('dummy argument', token=lname, context=pobj) # End if # End for # End if # End if # End while if insub and (len(statements) == 0): statements = read_statements(pobj) # End if # End while # Check for missing declarations missing = list() if vdict is None: errmsg = 'Subroutine, {}, not found{}' raise CCPPError(errmsg.format(scheme_name, ctx)) # End if for lname in vdict.keys(): if vdict[lname] is None: missing.append(lname) # End if # End for if len(missing) > 0: errmsg = 'Missing local_variables, {} in {}' raise CCPPError(errmsg.format(missing, scheme_name)) # End if var_dict = VarDictionary(scheme_name, variables=vdict) if (scheme_name is not None) and (var_dict is not None): mheader = MetadataTable(table_name_in=scheme_name, table_type_in='scheme', module=spec_name, var_dict=var_dict, logger=logger) # End if return statements, mheader
def parse_preamble_data(statements, pobj, spec_name, endmatch, logger): """Parse module variables or DDT definitions from a module preamble or parse program variables from the beginning of a program. """ inspec = True mheaders = list() var_dict = VarDictionary(spec_name) psrc = ParseSource(spec_name, 'MODULE', pobj) active_table = None if logger is not None: ctx = context_string(pobj, nodir=True) msg = "Parsing preamble variables of {}{}" logger.debug(msg.format(spec_name, ctx)) # End if while inspec and (statements is not None): while len(statements) > 0: statement = statements.pop(0) # End program or module pmatch = endmatch.match(statement) asmatch = _ARG_TABLE_START_RE.match(statement) type_def = fortran_type_definition(statement) if asmatch is not None: active_table = asmatch.group(1) elif (pmatch is not None) or is_contains_statement(statement, inspec): # We are done with the specification inspec = False # Put statement back so caller knows where we are statements.insert(0, statement) # Add the header (even if we found no variables) mheader = MetadataTable(table_name_in=spec_name, table_type_in='module', module=spec_name, var_dict=var_dict, logger=logger) mheaders.append(mheader) if logger is not None: ctx = context_string(pobj, nodir=True) msg = 'Adding header {}{}' logger.debug(msg.format(mheader.table_name, ctx)) break elif ((type_def is not None) and (type_def[0].lower() == active_table.lower())): # Put statement back so caller knows where we are statements.insert(0, statement) statements, ddt = parse_type_def(statements, type_def, spec_name, pobj, logger) if ddt is None: ctx = context_string(pobj, nodir=True) msg = "No DDT found at '{}'{}" raise CCPPError(msg.format(statement, ctx)) # End if mheaders.append(ddt) if logger is not None: ctx = context_string(pobj, nodir=True) msg = 'Adding DDT {}{}' logger.debug(msg.format(ddt.table_name, ctx)) # End if active_table = None elif active_table is not None: # We should have a variable definition to add if ((not is_comment_statement(statement)) and (not parse_use_statement(statement, logger)) and (active_table.lower() == spec_name.lower())): dvars = parse_fortran_var_decl(statement, psrc, logger=logger) for var in dvars: var_dict.add_variable(var) # End for # End if # End if (else we are not in an active table so just skip) # End while if inspec and (len(statements) == 0): statements = read_statements(pobj) # End if # End while return statements, mheaders