def _read_params(self, method): """Read in the parameters declared in a function, procedure or operator""" #look for parameters while not self._match_lookahead('symbol', ')'): #consume ) at end param_tok, other_tok = self._lookahead(2) if param_tok[0] == 'id': if other_tok[0] == 'id': #first one is a modifier modifier = self._next_token()[1] else: modifier = None param_toks = [self._match_token('id')] while self._match_lookahead('symbol', ',', True): #there is a list of parameters param_toks.append(self._match_token('id')) colon_tok = self._match_token('symbol', ':') the_type = self._read_type_usage() for param_tok in param_toks: param = method.create_parameter(param_tok[1]) param.data_type = the_type param.modifier = modifier logger.debug('Parser : Adding parameter %s (%s) to %s', param.name, param.data_type, method.name) if not self._match_lookahead('symbol', ';', True): break else: logger.error('Parser : Error in parameter list %s', self._tokeniser.line_details()) self._match_token('symbol', ')')
def _append_attribute(self, attr, val): logger.debug('Parser : Appending attribute %s with value %s to %s',attr,val, attr + 's') if (attr + 's') in self._attributes.keys(): self._attributes[attr + 's'].append(val) else: self._attributes[attr + 's'] = [val] self._ordered_attributes.append([attr + 's', self._attributes[attr + 's']])
def process_unit(self, token): tok = self._match_token('id') unit_name = tok[1] name = self._get_attribute('module', unit_name) self._add_attribute('name',name) logger.info('-' * 70) logger.info(' Parser : Processing unit: %s', unit_name) logger.info('-' * 70) #self._check_attributes(['class','static'],'unit ' + unit_name) self._check_meta_comments(1, 'unit ' + unit_name + '\'s declaration') #Create the unit unit = self._create_model_element(SGCodeModule) unit.module_kind = 'module' unit.is_static = True logger.info(' Parser : Creating class %s', unit.name) self._apply_attributes_to(unit) #unit name; tok = self._match_token('symbol', ';') #interface - ignore comments #self._skip_comments('unit ' + unit_name + '\'s interface') self._match_token('id', 'interface') #check 'uses' tok = self._lookahead()[0] if tok[0] == 'id' and tok[1] == 'uses': tok = self._next_token() self.process_uses_clause(tok) logger.info('-' * 70) while True: logger.debug('Parser : At start of loop to process elements in unit') #read comments if they follow this position, otherwise leave # attributes as they are if self._match_lookahead('meta comment'): self.process_meta_comments() #Process the body of the unit's interface tok = self._lookahead()[0] #check the next token if tok[0] == 'id' and tok[1] == 'implementation': logger.info(' Parser : Found end of unit\'s interface') logger.info('-' * 70) break #interface contains types, functions, procedures, var, const, operator tok = self._match_one_token([['id','type'], ['id','function'], ['id','procedure'], ['id','const'], ['id','operator'], ['id','var']]) self._block_header_processors[tok[1]](unit, tok); logger.info(' Parser : Finished processing unit. Resulting class is:\n%s', str(unit))
def process_block_types(self, block, token): '''Reads the types within a type declaration''' logger.info(' Parser : Processing types') logger.info('-' * 70) self.process_meta_comments() #following type... in pascal while True: type_name = self._match_token('id')[1] #read the types name self._match_token('operator','=') #read the = the_type = find_or_add_type(type_name) the_type.file_line_details = self._tokeniser.line_details() the_type.meta_comment_line_details = self._tokeniser.meta_comment_line_details() self._parse_type_declaration(the_type) if 'type' or 'class' or 'struct'in the_type.keys(): self._add_class(the_type) self.process_meta_comments() tok1, tok2 = self._lookahead(2) #looking for... type_name = if tok2[0] != 'operator' or tok2[1] != '=' or tok1[0] != 'id': logger.info('-' * 70) logger.debug('Parser : At end of block types') break
def process_operator_decl(self, block, token): '''process a operator read from a unit.''' op_kind = self._match_token('operator') open_tok = self._match_token('symbol', '(') #add to the class (e.g. Core module: which creates lib + other) self._add_attribute('name', op_kind[1]) #name comes from operator method = self._create_model_element(SGMethod) method.in_class = block method.is_operator = True self._read_params(method) result_tok = self._match_token('id') colon_tok = self._match_token('symbol', ':') the_type = self._read_type_usage() method.return_type = the_type logger.debug('Parser : Set return type of operator %s to %s', method.name, method.return_type) end_tok = self._match_token('symbol', ';') self._apply_attributes_to(method) method.complete_method_processing() block.add_member(method)
def check_arguments(self): ''' Ensure that the arguments in the call match the available parameters, if no arguments are provided copy across read in parameters ''' if self.method_called == None: logger.error( 'Method : Method %s does not call anything. Check attributes. %s', self.uname, self.file_line_details) assert False method_called = self.method_called #get called method args = self.args #the arguments self passes to the method logger.debug( 'Method : Checking arguments used by %s calling %s (%s)', self, method_called, args) if len(args) != len(method_called.params): logger.error('Method : Error in %s calling %s', self.uname, method_called.uname) assert False for arg in args: if isinstance(arg, SGParameter): if not self.has_parameter(arg.name): logger.error( "Cannot match parameter %s in call to %s from %s", str(arg), str(method_called), self) assert False
def process_unit(self, token): tok = self._match_token('id') unit_name = tok[1] name = self._get_attribute('module', unit_name) self._add_attribute('name', name) logger.info('-' * 70) logger.info(' Parser : Processing unit: %s', unit_name) logger.info('-' * 70) #self._check_attributes(['class','static'],'unit ' + unit_name) self._check_meta_comments(1, 'unit ' + unit_name + '\'s declaration') #Create the unit unit = self._create_model_element(SGCodeModule) unit.module_kind = 'module' unit.is_static = True logger.info(' Parser : Creating class %s', unit.name) self._apply_attributes_to(unit) #unit name; tok = self._match_token('symbol', ';') #interface - ignore comments #self._skip_comments('unit ' + unit_name + '\'s interface') self._match_token('id', 'interface') #check 'uses' tok = self._lookahead()[0] if tok[0] == 'id' and tok[1] == 'uses': tok = self._next_token() self.process_uses_clause(tok) logger.info('-' * 70) while True: logger.debug( 'Parser : At start of loop to process elements in unit') #read comments if they follow this position, otherwise leave # attributes as they are if self._match_lookahead('meta comment'): self.process_meta_comments() #Process the body of the unit's interface tok = self._lookahead()[0] #check the next token if tok[0] == 'id' and tok[1] == 'implementation': logger.info(' Parser : Found end of unit\'s interface') logger.info('-' * 70) break #interface contains types, functions, procedures, var, const, operator tok = self._match_one_token([['id', 'type'], ['id', 'function'], ['id', 'procedure'], ['id', 'const'], ['id', 'operator'], ['id', 'var']]) self._block_header_processors[tok[1]](unit, tok) logger.info( ' Parser : Finished processing unit. Resulting class is:\n%s', str(unit))
def process_block_types(self, block, token): '''Reads the types within a type declaration''' logger.info(' Parser : Processing types') logger.info('-' * 70) self.process_meta_comments() #following type... in pascal while True: type_name = self._match_token('id')[1] #read the types name self._match_token('operator', '=') #read the = the_type = find_or_add_type(type_name) the_type.file_line_details = self._tokeniser.line_details() the_type.meta_comment_line_details = self._tokeniser.meta_comment_line_details( ) self._parse_type_declaration(the_type) if 'type' or 'class' or 'struct' in the_type.keys(): self._add_class(the_type) self.process_meta_comments() tok1, tok2 = self._lookahead(2) #looking for... type_name = if tok2[0] != 'operator' or tok2[1] != '=' or tok1[0] != 'id': logger.info('-' * 70) logger.debug('Parser : At end of block types') break
def complete_method_processing(self): ''' This is called on methods that are read by the parser from the Pascal file. Set up the call from the library to this method if marked. Check the call's validity Steps: 1: Get other methods related to this one 2: Set parameters on library method (if called) ''' logger.info(' Method : Completing processing of %s', self) #This is 'the' method it has its params self.params = self.params #Find the length method if it exists if self.length_call != None: self.length_call = self.in_class.find_method(self.length_call) #Convert args to appropriate values... self._process_args() self._check_args_match_params() #Get other methods lib_method = self.method_called class_method = self.class_method #check rules if lib_method == None and not self.is_operator: logger.error('Method : Found method %s without lib', self) assert False if lib_method != None: #set up library method self.setup_lib_method(lib_method) logger.info(' Method : %s calls %s', self.name, lib_method.name) lib_method.called_by.append(self) #set up class method if class_method != None: logger.debug(' Method : %s is also %s', self.name, class_method) self._setup_class_method(class_method) logger.info(' Method : %s calls %s', class_method.name, lib_method.name) lib_method.called_by.append(class_method) #library is also called by class elif self.is_operator: assert self.other_class != None self.name = 'operator ' + self.name self.other_method = self.in_class.find_method(self['calls'].other) self.doc = self.other_method.doc self.method_called = self.other_method.method_called self.method_called.called_by.append(self) #library is also called by operator self.is_static = True self.in_class.operators[self.signature] = None self.in_class = self.other_class self.in_class.add_member(self) self.args = list(self.params) # operators must match directly
def _add_attribute(self, attr, val): logger.debug('Parser : Adding attribute %s with value %s', attr, val) if attr in self._attributes and self._attributes[attr] != None: logger.warning('Parser : Added attribute twice %s = %s', attr, val) self._attributes[attr] = val self._ordered_attributes.append([attr, val])
def _lookahead(self, count=1): logger.debug('Parser : Looking ahead %d', count) while len(self._lookahead_toks) < count: current_token = self._tokeniser.next_token() while current_token[0] == 'comment': current_token = self._tokeniser.next_token() self._lookahead_toks.append(current_token) return self._lookahead_toks
def _lookahead(self,count=1): logger.debug('Parser : Looking ahead %d', count) while len(self._lookahead_toks) < count: current_token = self._tokeniser.next_token() while current_token[0] == 'comment': current_token = self._tokeniser.next_token() self._lookahead_toks.append(current_token) return self._lookahead_toks
def _append_attribute(self, attr, val): logger.debug('Parser : Appending attribute %s with value %s to %s', attr, val, attr + 's') if (attr + 's') in self._attributes.keys(): self._attributes[attr + 's'].append(val) else: self._attributes[attr + 's'] = [val] self._ordered_attributes.append( [attr + 's', self._attributes[attr + 's']])
def _match_lookahead(self, token_kind, token_value = None, consume = False): logger.debug('Parser : Looking to find %s (%s)%s', token_kind, token_value if token_value != None else 'any', ' will consume' if consume else '') token = self._lookahead(1)[0] result = token[0] == token_kind and (token_value == None or token_value == token[1].lower()) if consume and result: self._match_token(token_kind, token_value) return result
def _match_lookahead(self, token_kind, token_value=None, consume=False): logger.debug('Parser : Looking to find %s (%s)%s', token_kind, token_value if token_value != None else 'any', ' will consume' if consume else '') token = self._lookahead(1)[0] result = token[0] == token_kind and (token_value == None or token_value == token[1].lower()) if consume and result: self._match_token(token_kind, token_value) return result
def _match_token(self, token_kind, token_value = None): tok = self._next_token() if tok[0] != token_kind or (token_value != None and token_value != tok[1].lower()): logger.error('Parse Error %s: found a %s (%s) expected %s (%s)', self._tokeniser.line_details(), tok[0], tok[1], token_kind, token_value) assert False logger.debug('Parser : Matched token %s (%s)', tok[0], tok[1]) return tok
def _next_token(self): current_token = None while current_token == None or current_token[0] == 'comment': if len(self._lookahead_toks) > 0: current_token = self._lookahead_toks[0] self._lookahead_toks = self._lookahead_toks[1:] else: current_token = self._tokeniser.next_token() if current_token[0] == 'comment': logger.debug('Parser : Skipping comment: %s', current_token[1]) return current_token
def _match_token(self, token_kind, token_value=None): tok = self._next_token() if tok[0] != token_kind or (token_value != None and token_value != tok[1].lower()): logger.error('Parse Error %s: found a %s (%s) expected %s (%s)', self._tokeniser.line_details(), tok[0], tok[1], token_kind, token_value) assert False logger.debug('Parser : Matched token %s (%s)', tok[0], tok[1]) return tok
def _setup_class_method(self, class_method): ''' Setup the class's method. The class method is a clone of the current method, in a class wrapper. This comes from @method or @overload in the pascal source. Steps: 1: clone self to class_method 2: add it to its class or property 3: alter args (add pointer field access) ''' self.clone_to(class_method) #copy self into other class_method.is_class_method = True #if the class method is actually a property... if self.is_getter or self.is_setter: self.create_and_add_property(class_method) else: class_method.in_class.add_member(class_method) #add to its class #static methods and constructors directly call the method # instance methods change the first argument for the pointer field if class_method.is_static or class_method.is_constructor: #use self's args - i.e. it calls the method in the same way args = list(self.args) else: #other is an instance (with ptr) if self.args == None or len(self.args) < self.self_pos: logger.error( 'Class method calling a method without parameter for self pointer... ' + self.name) assert False args = list(self.args) #change the old argument for the self pointer if class_method.in_class.is_pointer_wrapper: args[self.self_pos - 1] = 'self.pointer' elif class_method.in_class.wraps_array: args[self.self_pos - 1] = 'self.data' else: args[self.self_pos - 1] = 'self' #add in self's arguments (-1st which is pointer) #set class method to call the same method this does class_method.calls(self.method_called, args) logger.debug('Method : Setting up call: %s calls %s with args %s', class_method, class_method.method_called, class_method.args) class_method._check_args_match_params()
def _setup_class_method(self, class_method): ''' Setup the class's method. The class method is a clone of the current method, in a class wrapper. This comes from @method or @overload in the pascal source. Steps: 1: clone self to class_method 2: add it to its class or property 3: alter args (add pointer field access) ''' self.clone_to(class_method) #copy self into other class_method.is_class_method = True #if the class method is actually a property... if self.is_getter or self.is_setter: self.create_and_add_property(class_method) else: class_method.in_class.add_member(class_method) #add to its class #static methods and constructors directly call the method # instance methods change the first argument for the pointer field if class_method.is_static or class_method.is_constructor: #use self's args - i.e. it calls the method in the same way args = list(self.args) else: #other is an instance (with ptr) if self.args == None or len(self.args) < self.self_pos: logger.error('Class method calling a method without parameter for self pointer... ' + self.name) assert False args = list(self.args) #change the old argument for the self pointer if class_method.in_class.is_pointer_wrapper: args[self.self_pos - 1] = 'self.pointer' elif class_method.in_class.wraps_array: args[self.self_pos - 1] = 'self.data' else: args[self.self_pos - 1] = 'self' #add in self's arguments (-1st which is pointer) #set class method to call the same method this does class_method.calls(self.method_called, args) logger.debug('Method : Setting up call: %s calls %s with args %s', class_method, class_method.method_called, class_method.args) class_method._check_args_match_params()
def method_process_visitor(the_method, other): '''Process a method prior to rendering. This does all of the transformations that go into the functions and procedures of the SGSDK unit.''' # Change functions with string or array return type to have a result parameter if the_method.return_type != None and ( the_method.return_type.wraps_array or the_method.return_type.name.lower() in ['string']): #print 'PARS RUN PARAMS', [p.name for p in the_method.params] result_param = SGParameter('result') for param in the_method.params: if 'result' == param.name: logger.error('PARSER RUN: Error adding result parameter to %s', the_method.name) assert False _add_parameter(the_method, result_param) result_param.maps_result = True result_param.modifier = 'result' result_param.data_type = the_method.return_type the_method.return_type = None the_method.was_function = True #print 'PARS RUN PARAMS', [p.name for p in the_method.params] #Replace any variable length arrays orig_params = the_method.params for param in orig_params: #does it wrap a variable length array if param.data_type.array_wrapper: the_method.has_length_params = True logger.debug( 'PARSER RUN: Altering variable length array of %s\'s parameter %s', the_method.name, param.name) # old_type = param.data_type #param.data_type = param.data_type #.fields[0].data_type len_param = SGParameter('%s_len' % param.name) len_param.is_length_param = True len_param.data_type = find_or_add_type('Longint') len_param.modifier = param.modifier if param.modifier is [ 'out', 'var' ] else None len_param.length_of = param param.has_length_param = True param.length_idx = _add_parameter(the_method, len_param) return other
def process_uses_clause(self, token): '''Read the list of units referred to be the uses clause''' self._check_non_commented('uses clause') while True: tok = self._match_token('id') self._current_file.uses.append(find_or_add_file(tok[1])) logger.debug('Parser : Found using unit %s', tok[1]) #found a token/unit next_tok = self._match_token('symbol') if next_tok[1] == ';': break; #found end elif next_tok[1] != ',': logger.error('Parser Error %s: expected , or ; but found %s at %s', self._tokeniser.line_details(), next_tok[1]) sys.exit(-1)
def _match_one_token(self, token_kind_lst): matched = False tok = self._next_token() for token_kind,token_value in token_kind_lst: if tok[0] == token_kind and (token_value == None or token_value == tok[1]): matched = True logger.debug('Parser : Matched %s with %s', tok[0], tok[1]) break if not matched: logger.error('Parser Error %s: unexpected %s(%s) expected %s', self._tokeniser.line_details(), tok[0], tok[1], map(lambda n: '%s(%s)' % (n[0],n[1]),token_kind_lst)) assert False return tok
def _match_one_token(self, token_kind_lst): matched = False tok = self._next_token() for token_kind, token_value in token_kind_lst: if tok[0] == token_kind and (token_value == None or token_value == tok[1]): matched = True logger.debug('Parser : Matched %s with %s', tok[0], tok[1]) break if not matched: logger.error( 'Parser Error %s: unexpected %s(%s) expected %s', self._tokeniser.line_details(), tok[0], tok[1], map(lambda n: '%s(%s)' % (n[0], n[1]), token_kind_lst)) assert False return tok
def _check_args_match_params(self): ''' Ensure that the arguments in the call match the available parameters, if no arguments are provided copy across read in parameters ''' method_called = self.method_called #self.tags['calls'].other[0] #get called method args = self.args #self.tags['calls'].other[1] #the arguments self passes to the method logger.debug('Method : Checking arguments used by %s calling %s (%s)', self, method_called, args) if args == None: logger.error('Method : No arguments that map to parameters for %s', self) assert False else: for arg in args: if isinstance(arg, SGParameter) and not self.has_parameter(arg.name): logger.error("Cannot match parameter %s in call to %s from %s", str(arg), str(method_called), self) assert False
def tokenise(self, filename): '''Initialises the tokeniser with characters loaded from the specified filename. Call `next_token` process each token. ''' if isinstance(filename, list): logger.debug('Tokenising list') self.pas_lines = filename else: logger.debug('Tokenising %s', filename) self._filename = filename # print filename f = open(filename) self.pas_lines = f.readlines() f.close() self._char_no = -1 self._line_no = 0 #starts at first line self._token_val = 'none'
def _create_model_element(self, kind): name = self._get_attribute('name') if kind == SGCodeModule: result = find_or_add_class(name) if not result in self._current_file.members: self._current_file.members.append(result) result.file_line_details = self._tokeniser.line_details() result.meta_comment_line_details = self._tokeniser.meta_comment_line_details() elif kind == SGType: assert False #result = find_or_add_type(name) else: result = kind(name) result.file_line_details = self._tokeniser.line_details() result.meta_comment_line_details = self._tokeniser.meta_comment_line_details() result.in_file = self._current_file logger.debug('Parser : Creating model element: %s with kind:%s', name, kind) return result
def process_uses_clause(self, token): '''Read the list of units referred to be the uses clause''' self._check_non_commented('uses clause') while True: tok = self._match_token('id') self._current_file.uses.append(find_or_add_file(tok[1])) logger.debug('Parser : Found using unit %s', tok[1]) #found a token/unit next_tok = self._match_token('symbol') if next_tok[1] == ';': break #found end elif next_tok[1] != ',': logger.error( 'Parser Error %s: expected , or ; but found %s at %s', self._tokeniser.line_details(), next_tok[1]) sys.exit(-1)
def process_meta_comments(self): logger.debug('Parser : Starting to process meta comments: clearing old comments and attributes') #self._meta_comments = [] #self._attributes = {} tok = self._lookahead(1)[0] #_next_token() attrs_started = False while tok[0] in ['meta comment','attribute','comment']: if tok[0] == 'attribute': attrs_started = True if attrs_started and tok[0] == 'meta comment': tok = self._next_token() #actually read token if len(tok[1]) > 0: logger.error('Parser Error %s: Found additional meta comment after start of attributes', self._tokeniser.line_details()) assert False else: tok = self._next_token() #actually read token self._processors[tok[0]](tok) tok = self._lookahead(1)[0]
def method_process_visitor(the_method, other): '''Process a method prior to rendering. This does all of the transformations that go into the functions and procedures of the SGSDK unit.''' # Change functions with string or array return type to have a result parameter if the_method.return_type != None and (the_method.return_type.wraps_array or the_method.return_type.name.lower() in ['string']): #print 'PARS RUN PARAMS', [p.name for p in the_method.params] result_param = SGParameter('result') for param in the_method.params: if 'result' == param.name: logger.error('PARSER RUN: Error adding result parameter to %s', the_method.name) assert False _add_parameter(the_method, result_param) result_param.maps_result = True result_param.modifier = 'result' result_param.data_type = the_method.return_type the_method.return_type = None the_method.was_function = True #print 'PARS RUN PARAMS', [p.name for p in the_method.params] #Replace any variable length arrays orig_params = the_method.params for param in orig_params: #does it wrap a variable length array if param.data_type.array_wrapper: the_method.has_length_params = True logger.debug('PARSER RUN: Altering variable length array of %s\'s parameter %s', the_method.name, param.name) # old_type = param.data_type #param.data_type = param.data_type #.fields[0].data_type len_param = SGParameter('%s_len' % param.name) len_param.is_length_param = True len_param.data_type = find_or_add_type('Longint') len_param.modifier = param.modifier if param.modifier is ['out', 'var'] else None len_param.length_of = param param.has_length_param = True param.length_idx = _add_parameter(the_method, len_param) return other
def setup_lib_method(self, lib_method): ''' Setup the library method Set: return type parameters calls ''' if lib_method.file_line_details == None: lib_method.file_line_details = [] lib_method.file_line_details.append(self.file_line_details) if self.called_by_lib: #Set up the call from the library to this method logger.debug('Method : Setting %s in library to call %s', lib_method, self) lib_method.return_type = self.return_type #set return type lib_method.params = [] #add parameters for p in self.params: lib_method.params.append(p.clone()) lib_method.calls(self, self.args) #..calls this method at other end
def process_method_decl(self, block, token): '''process a method read from a unit. block is the block that contains the method, token is the first token. At the end of this the method has been read in and added to block. ''' name_tok = self._match_token('id') #name of function/procedure open_tok = self._match_token('symbol', '(') #add to the class (e.g. Core module: which creates lib + other) self._add_attribute('name', name_tok[1]) #name comes from function/procedure method = self._create_model_element(SGMethod) method.in_class = block self._read_params(method) if token[1] == 'function': #return return details colon_tok = self._match_token('symbol', ':') the_type = self._read_type_usage() method.return_type = the_type logger.debug('Parser : Set return type of method %s to %s', method.name, method.return_type) end_tok = self._match_token('symbol', ';') #check overload ; if self._match_lookahead('id', 'overload', True): self._match_token('symbol', ';') self._add_attribute('overload', True) #logger.info('Adding method %s.%s(%s)', block.name, method.name, method.param_string()) if self._get_attribute( 'called_by_lib', False) and self._get_attribute('method_called') == None: self._setup_lib_method(method.uname) self._apply_attributes_to(method) method.complete_method_processing() block.add_member(method)
def _create_model_element(self, kind): name = self._get_attribute('name') if kind == SGCodeModule: result = find_or_add_class(name) if not result in self._current_file.members: self._current_file.members.append(result) result.file_line_details = self._tokeniser.line_details() result.meta_comment_line_details = self._tokeniser.meta_comment_line_details( ) elif kind == SGType: assert False #result = find_or_add_type(name) else: result = kind(name) result.file_line_details = self._tokeniser.line_details() result.meta_comment_line_details = self._tokeniser.meta_comment_line_details( ) result.in_file = self._current_file logger.debug('Parser : Creating model element: %s with kind:%s', name, kind) return result
def check_arguments(self): ''' Ensure that the arguments in the call match the available parameters, if no arguments are provided copy across read in parameters ''' if self.method_called == None: logger.error('Method : Method %s does not call anything. Check attributes. %s', self.uname, self.file_line_details) assert False method_called = self.method_called #get called method args = self.args #the arguments self passes to the method logger.debug('Method : Checking arguments used by %s calling %s (%s)', self, method_called, args) if len(args) != len(method_called.params): logger.error('Method : Error in %s calling %s', self.uname, method_called.uname) assert False for arg in args: if isinstance(arg, SGParameter): if not self.has_parameter(arg.name): logger.error("Cannot match parameter %s in call to %s from %s", str(arg), str(method_called), self) assert False
def process_meta_comments(self): logger.debug( 'Parser : Starting to process meta comments: clearing old comments and attributes' ) #self._meta_comments = [] #self._attributes = {} tok = self._lookahead(1)[0] #_next_token() attrs_started = False while tok[0] in ['meta comment', 'attribute', 'comment']: if tok[0] == 'attribute': attrs_started = True if attrs_started and tok[0] == 'meta comment': tok = self._next_token() #actually read token if len(tok[1]) > 0: logger.error( 'Parser Error %s: Found additional meta comment after start of attributes', self._tokeniser.line_details()) assert False else: tok = self._next_token() #actually read token self._processors[tok[0]](tok) tok = self._lookahead(1)[0]
def process_method_decl(self, block, token): '''process a method read from a unit. block is the block that contains the method, token is the first token. At the end of this the method has been read in and added to block. ''' name_tok = self._match_token('id') #name of function/procedure open_tok = self._match_token('symbol', '(') #add to the class (e.g. Core module: which creates lib + other) self._add_attribute('name', name_tok[1]) #name comes from function/procedure method = self._create_model_element(SGMethod) method.in_class = block self._read_params(method) if token[1] == 'function': #return return details colon_tok = self._match_token('symbol', ':') the_type = self._read_type_usage() method.return_type = the_type logger.debug('Parser : Set return type of method %s to %s', method.name, method.return_type) end_tok = self._match_token('symbol', ';') #check overload ; if self._match_lookahead('id', 'overload', True): self._match_token('symbol', ';') self._add_attribute('overload', True) #logger.info('Adding method %s.%s(%s)', block.name, method.name, method.param_string()) if self._get_attribute('called_by_lib', False) and self._get_attribute('method_called') == None: self._setup_lib_method(method.uname) self._apply_attributes_to(method) method.complete_method_processing() block.add_member(method)
def _check_args_match_params(self): ''' Ensure that the arguments in the call match the available parameters, if no arguments are provided copy across read in parameters ''' method_called = self.method_called #self.tags['calls'].other[0] #get called method args = self.args #self.tags['calls'].other[1] #the arguments self passes to the method logger.debug( 'Method : Checking arguments used by %s calling %s (%s)', self, method_called, args) if args == None: logger.error( 'Method : No arguments that map to parameters for %s', self) assert False else: for arg in args: if isinstance( arg, SGParameter) and not self.has_parameter(arg.name): logger.error( "Cannot match parameter %s in call to %s from %s", str(arg), str(method_called), self) assert False
def process_attribute(self, token): logger.debug('Parser : Processing attribute: %s', token[1]) self._attribute_processors[token[1]](token)
def complete_method_processing(self): ''' This is called on methods that are read by the parser from the Pascal file. Set up the call from the library to this method if marked. Check the call's validity Steps: 1: Get other methods related to this one 2: Set parameters on library method (if called) ''' logger.info(' Method : Completing processing of %s', self) #This is 'the' method it has its params self.params = self.params #Find the length method if it exists if self.length_call != None: self.length_call = self.in_class.find_method(self.length_call) #Convert args to appropriate values... self._process_args() self._check_args_match_params() #Get other methods lib_method = self.method_called class_method = self.class_method #check rules if lib_method == None and not self.is_operator: logger.error('Method : Found method %s without lib', self) assert False if lib_method != None: #set up library method self.setup_lib_method(lib_method) logger.info(' Method : %s calls %s', self.name, lib_method.name) lib_method.called_by.append(self) #set up class method if class_method != None: logger.debug(' Method : %s is also %s', self.name, class_method) self._setup_class_method(class_method) logger.info(' Method : %s calls %s', class_method.name, lib_method.name) lib_method.called_by.append( class_method) #library is also called by class elif self.is_operator: assert self.other_class != None self.name = 'operator ' + self.name self.other_method = self.in_class.find_method(self['calls'].other) self.doc = self.other_method.doc self.method_called = self.other_method.method_called self.method_called.called_by.append( self) #library is also called by operator self.is_static = True self.in_class.operators[self.signature] = None self.in_class = self.other_class self.in_class.add_member(self) self.args = list(self.params) # operators must match directly
def find_method(self, uname, file_name): for key, method in self.methods.items(): if method.uname == 'sg_%s_%s' % (file_name, uname): logger.debug('Library : Found match for %s', method.uname) return method return None
def visit_all_units(file_visitor): logger.info('Processing files') files = [unit[0] for unit in all_units] + ['SGSDK'] for each_file in files: logger.debug('Visiting file %s', each_file) find_or_add_file(each_file).visit(file_visitor, None)
def next_token(self): '''Find and return a tuple with the next token details. Tuple contains the token (type, value) as strings. The token types and details are: number, # such as 1234, 123.45, -123, +123.4 comment, # single // or multi-line (* ... *), { ... } meta comment, # start with /// ... id, # identifier name starting with alpha, including # alpha-numeric characters and the _ character attribute, # name starting with @... inside meta comment block # follows the id name character rules operator, # one of + - / * ** := < > <= >= <> symbol, # one of ':;,.()' ''' def num_match(cha, tmp): '''Checks for a number in format ##, ##.#. Returns False when at the end of a number.''' if cha in '1234567890': return True elif cha == '.' and '.' not in tmp: return self._peek(1) in '1234567890' else: return False while (True): t = self._next_char() self._token_start = self._char_no # Ignore white space characters if t == ' ' or t == '\t': #ignore white space pass # Move to next line (if at end of line) elif t == '\n': self._advance_line() # Numbers (int or float style format elif t in '1234567890' or (t in '-+' and self._peek(1) in '1234567890'): #is digit or +/- result = ('number', self._read_matching(t, num_match)) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result # Comment, single line // or meta comment line /// elif t == '/' and self._peek(1) == '/': #start of comment if self._match_and_read('/'): if self._match_and_read('/'): kind = 'meta comment' self._meta_comment_start = self._char_no self._meta_comment_line = self._line_no comment = self.read_to_end_of_comment() else: kind = 'comment' comment = self.read_to_eol() result = (kind, comment) else: result = ('error', t) logger.debug('Tokenisier: read %s', result[0]) return result # Attribute identified by an @ symbol then a name elif t == '@': name = self._read_matching( '', lambda cha, tmp: cha.isalnum() or cha == '_') result = ('attribute', name) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result # Identifier (id) of alphanumeric characters including elif t.isalpha(): name = self._read_matching( t, lambda cha, tmp: cha.isalnum() or cha == '_') if name.lower() in ['true', 'false']: result = ('boolean', name) else: result = ('id', name) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result #Bound Comment elif t == '{' or (t == '(' and self._peek(1) == '*'): if t == '(' and self._match_and_read('*'): comment = self._read_until('', lambda temp: temp[-2:] == '*)') result = ('comment', comment[:-2]) elif t == '{': comment = self._read_until('', lambda temp: temp[-1:] == '}') result = ('comment', comment[:-1]) logger.log(logging.DEBUG, 'Tokenisier: read %s', result[0]) return result # Operator elif (t == ':' and self._peek(1) == '=') or t in '=+-*/><': if t == ':' and self._match_and_read('='): result = ('operator', ':=') elif t == '*' and self._match_and_read('*'): result = ('operator', '**') elif t == '<' and self._match_and_read('>'): result = ('operator', '<>') elif t in '<>' and self._match_and_read('='): result = ('operator', t + '=') else: result = ('operator', t) return result # Symbol elif t in '(),:;[].^': result = ('symbol', t) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result # Catch any single quotes inside a string value. elif t == "'": string = self._read_until( '', lambda temp: (temp[-1:] == "'") and (not self._match_and_read("'"))) result = ('string', string[:-1]) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result # Hmm.. unknown token. What did we forget? else: logger.error("Unknown token type: " + t) return ('error', t)
def _add_attribute(self, attr, val): logger.debug('Parser : Adding attribute %s with value %s',attr,val) if attr in self._attributes and self._attributes[attr] != None: logger.warning('Parser : Added attribute twice %s = %s', attr, val) self._attributes[attr] = val self._ordered_attributes.append([attr, val])
def next_token(self): '''Find and return a tuple with the next token details. Tuple contains the token (type, value) as strings. The token types and details are: number, # such as 1234, 123.45, -123, +123.4 comment, # single // or multi-line (* ... *), { ... } meta comment, # start with /// ... id, # identifier name starting with alpha, including # alpha-numeric characters and the _ character attribute, # name starting with @... inside meta comment block # follows the id name character rules operator, # one of + - / * ** := < > <= >= <> symbol, # one of ':;,.()' ''' def num_match(cha, tmp): '''Checks for a number in format ##, ##.#. Returns False when at the end of a number.''' if cha in '1234567890': return True elif cha == '.' and '.' not in tmp: return self._peek(1) in '1234567890' else: return False while (True): t = self._next_char(); self._token_start = self._char_no # Ignore white space characters if t == ' ' or t == '\t': #ignore white space pass # Move to next line (if at end of line) elif t == '\n': self._advance_line() # Numbers (int or float style format elif t in '1234567890' or (t in '-+' and self._peek(1) in '1234567890'): #is digit or +/- result = ('number', self._read_matching(t, num_match)) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result # Comment, single line // or meta comment line /// elif t == '/' and self._peek(1) == '/': #start of comment if self._match_and_read('/'): if self._match_and_read('/'): kind = 'meta comment' self._meta_comment_start = self._char_no self._meta_comment_line = self._line_no comment = self.read_to_end_of_comment() else: kind = 'comment' comment = self.read_to_eol() result = (kind, comment) else: result = ('error', t) logger.debug('Tokenisier: read %s', result[0]) return result # Attribute identified by an @ symbol then a name elif t == '@': name = self._read_matching('', lambda cha, tmp: cha.isalnum() or cha == '_') result = ('attribute', name) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result # Identifier (id) of alphanumeric characters including elif t.isalpha(): name = self._read_matching(t, lambda cha, tmp: cha.isalnum() or cha == '_') if name.lower() in ['true','false']: result = ('boolean', name) else: result = ('id', name) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result #Bound Comment elif t == '{' or (t == '(' and self._peek(1) == '*'): if t == '(' and self._match_and_read('*'): comment = self._read_until('', lambda temp: temp[-2:] == '*)') result = ('comment', comment[:-2]) elif t == '{': comment = self._read_until('', lambda temp: temp[-1:] == '}') result = ('comment', comment[:-1]) logger.log(logging.DEBUG, 'Tokenisier: read %s', result[0]) return result # Operator elif (t == ':' and self._peek(1) == '=') or t in '=+-*/><': if t == ':' and self._match_and_read('='): result = ('operator', ':=') elif t == '*' and self._match_and_read('*'): result = ('operator', '**') elif t == '<' and self._match_and_read('>'): result = ('operator', '<>') elif t in '<>' and self._match_and_read('='): result = ('operator', t + '=') else: result = ('operator', t) return result # Symbol elif t in '(),:;[].^': result = ('symbol', t) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result # Catch any single quotes inside a string value. elif t == "'": string = self._read_until('', lambda temp: (temp[-1:] == "'") and (not self._match_and_read("'"))) result = ('string', string[:-1]) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result # Hmm.. unknown token. What did we forget? else: logger.error("Unknown token type: "+t) return ('error', t)
def visit_all_units(file_visitor): logger.info('Processing files') files = [ unit[0] for unit in all_units ] + ['SGSDK'] for each_file in files: logger.debug('Visiting file %s', each_file) find_or_add_file(each_file).visit(file_visitor, None)