def _read_params(self, method): """Read in the parameters declared in a function, procedure or operator""" #look for parameters while not self._match_lookahead('symbol', ')'): #consume ) at end param_tok, other_tok = self._lookahead(2) if param_tok[0] == 'id': if other_tok[0] == 'id': #first one is a modifier modifier = self._next_token()[1] else: modifier = None param_toks = [self._match_token('id')] while self._match_lookahead('symbol', ',', True): #there is a list of parameters param_toks.append(self._match_token('id')) colon_tok = self._match_token('symbol', ':') the_type = self._read_type_usage() for param_tok in param_toks: param = method.create_parameter(param_tok[1]) param.data_type = the_type param.modifier = modifier logger.debug('Parser : Adding parameter %s (%s) to %s', param.name, param.data_type, method.name) if not self._match_lookahead('symbol', ';', True): break else: logger.error('Parser : Error in parameter list %s', self._tokeniser.line_details()) self._match_token('symbol', ')')
def std_type_visitor(the_dict, the_type, modifier = None, dict_name = '_type_switcher'): ''' switch types for the SwinGame library. Params: - the_dict: The dictionary with the type changes with modifier keys - the_type: The type being checked - modifier: The modifier for the type - dict_name: The name of the dictionary for error reporting ''' key = the_type.name.lower() if the_type != None else None if modifier == 'result': modifier = 'return' if key not in the_dict[modifier]: logger.error('WRAPPER : Error changing model type %s - %s', modifier, the_type) logger.error(' : Add \'%s[%s]\': \'%s\': \'????\',', dict_name, modifier, the_type.name.lower()) global _hasError, _dieOnError _hasError = True if _dieOnError: assert False else: the_dict[modifier][key] = "UNKNOWN" return the_dict[modifier][key]
def _check_meta_comments(self, count, desc): if len(self._meta_comments) != count: logger.error( 'Parser Error %s: expected %d but foung %d meta comments for %s', self._tokeniser.line_details(), count, len(self._meta_comments), desc) sys.exit(-1)
def _parse_type_declaration(self, the_type): ''' Parse a type from the next token, add details to the_type. this is called for each type declaration... type_name = BLAH Need to process BLAH and add details to the_type ''' #check what kind of type it is... if self._match_lookahead('symbol', '(', True): values = [] while not self._match_lookahead('symbol',')'): temp = self._match_token('id')[1] if self._match_lookahead('operator', '=', True) or self._match_lookahead('operator', ':=', True): #assigned value values.append('%s = %s' % (temp, self._match_token('number')[1])) else: values.append(temp) self._match_lookahead('symbol', ',', True) #consume commas self._match_token('symbol',')') #read close bracket the_type.values = tuple(values) elif (self._match_lookahead('id', 'packed', True) and self._match_lookahead('id', 'record', True)) or self._match_lookahead('id', 'record', True): #packed record field: Type end; while not self._match_lookahead('id', 'end', True): #read field variables = self._read_variable_list() self._match_token('symbol', ';') for name, data_type in variables: field = SGField(name) field.data_type = data_type the_type.fields.append(field) elif self._match_lookahead('id', 'array') or self._match_lookahead('symbol', '^'): #is pointer or array the_type.clone(self._read_type_usage()) elif self._match_lookahead('id', 'procedure', True): #procedure type: read ( params ); self._match_token('symbol', '(') m = SGMethod(the_type.name) m.in_file = self._current_file self._read_params(m) the_type.is_procedure = True the_type.method = m if self._lookahead(2)[1][1] == 'cdecl': self._match_token('symbol', ';') self._match_token('id', 'cdecl') elif self._match_lookahead('id'): other_id = self._match_token('id')[1] other_type = find_or_add_type(other_id) the_type.related_type = other_type else: tok = self._next_token() logger.error('Parser Error %s: unknown type %s(%s)', self._tokeniser.line_details(), tok[0], tok[1]) assert False if the_type.related_type != None: logger.info(' Parser : Setup type %s = %s', the_type, the_type.related_type) elif the_type.is_enum: logger.info(' Parser : Setup type %s = %s', the_type, the_type.values) else: logger.info(' Parser : Setup type %s = %s', the_type, the_type.fields) self._apply_attributes_to(the_type) self._match_token('symbol', ';')
def create_property_for_field(in_class, field): '''Creates a property to access the ''' if field.data_type.wraps_array: logger.error('WRAPPER : Error structure with array fields must be set to via_pointer') global _hasError, _dieOnError _hasError = True if _dieOnError: assert False prop = SGProperty(field.name) prop.in_class = in_class prop.data_type = field.data_type getter = SGMethod('get' + field.pascalName) getter.return_type = field.data_type getter.in_class = in_class getter.field_name = field.name getter.is_getter = True setter = SGMethod('set' + field.pascalName) setter.params.append(SGParameter('value')) setter.params[0].data_type = field.data_type setter.in_class = in_class setter.field_name = field.name setter.is_setter = True prop.set_getter(getter) prop.set_setter(setter) return prop
def complete_method_processing(self): ''' This is called on methods that are read by the parser from the Pascal file. Set up the call from the library to this method if marked. Check the call's validity Steps: 1: Get other methods related to this one 2: Set parameters on library method (if called) ''' logger.info(' Method : Completing processing of %s', self) #This is 'the' method it has its params self.params = self.params #Find the length method if it exists if self.length_call != None: self.length_call = self.in_class.find_method(self.length_call) #Convert args to appropriate values... self._process_args() self._check_args_match_params() #Get other methods lib_method = self.method_called class_method = self.class_method #check rules if lib_method == None and not self.is_operator: logger.error('Method : Found method %s without lib', self) assert False if lib_method != None: #set up library method self.setup_lib_method(lib_method) logger.info(' Method : %s calls %s', self.name, lib_method.name) lib_method.called_by.append(self) #set up class method if class_method != None: logger.debug(' Method : %s is also %s', self.name, class_method) self._setup_class_method(class_method) logger.info(' Method : %s calls %s', class_method.name, lib_method.name) lib_method.called_by.append(class_method) #library is also called by class elif self.is_operator: assert self.other_class != None self.name = 'operator ' + self.name self.other_method = self.in_class.find_method(self['calls'].other) self.doc = self.other_method.doc self.method_called = self.other_method.method_called self.method_called.called_by.append(self) #library is also called by operator self.is_static = True self.in_class.operators[self.signature] = None self.in_class = self.other_class self.in_class.add_member(self) self.args = list(self.params) # operators must match directly
def _process_args(self): ''' Convert args to parameters, fields, and literals ''' new_args = list() args = self.args if args == None: self.args = list(self.params) return for argv in args: if argv[0] in ['number', 'string', 'boolean']: new_args.append(argv[1]) elif argv[0] in ['id']: param = self.get_parameter(argv[1]) if param != None: new_args.append(param) else: field = self.in_class.get_field(argv[1]) if field != None: new_args.append(field) else: logger.error( 'Method : Error cannot find %s in method %s', argv[1], self.uname) assert False else: logger.error( 'Method : Error unknown type of argument in %s - %s', self.uname, argv[0]) self.args = new_args
def map_data_value(the_dict, key, the_type, result): ''' Returns the code needed to map the swingame (Pascal) data in result to the type used by language X Params: - the_dict: The dictionary with the type changes with modifier keys - key: The key to search for in the_dict - the_type: The type being checked - modifier: The modifier for the type ''' if the_type.name.lower() in the_dict[key]: # print the_type.name.lower(), ' -> ', the_dict[key][the_type.name.lower()] return the_dict[key][the_type.name.lower()] % result #If this is a if the_type.pointer_wrapper and key == 'return_val': logger.error('WRAPPER : Error pointer wrapper without return data mapping - %s', the_type) global _hasError, _dieOnError _hasError = True if _dieOnError: assert False else: the_dict[key][the_type.name.lower()] = "UNKNOWN" return result
def _process_args(self): ''' Convert args to parameters, fields, and literals ''' new_args = list() args = self.args if args == None: self.args = list(self.params) return for argv in args: if argv[0] in ['number', 'string', 'boolean']: new_args.append(argv[1]) elif argv[0] in ['id']: param = self.get_parameter(argv[1]) if param != None: new_args.append(param) else: field = self.in_class.get_field(argv[1]) if field != None: new_args.append(field) else: logger.error('Method : Error cannot find %s in method %s', argv[1], self.uname) assert False else: logger.error('Method : Error unknown type of argument in %s - %s', self.uname, argv[0]) self.args = new_args
def set_getter(self, method): """sets the getter method of the property""" self.set_tag('getter', method) if method == None: return; if self.data_type == None: self.data_type = method.return_type; elif self.data_type != method.return_type: logger.error('Inconsistent types in property %s: %s is not %s', self.name, self.data_type, method.return_type)
def _match_token(self, token_kind, token_value = None): tok = self._next_token() if tok[0] != token_kind or (token_value != None and token_value != tok[1].lower()): logger.error('Parse Error %s: found a %s (%s) expected %s (%s)', self._tokeniser.line_details(), tok[0], tok[1], token_kind, token_value) assert False logger.debug('Parser : Matched token %s (%s)', tok[0], tok[1]) return tok
def _match_token(self, token_kind, token_value=None): tok = self._next_token() if tok[0] != token_kind or (token_value != None and token_value != tok[1].lower()): logger.error('Parse Error %s: found a %s (%s) expected %s (%s)', self._tokeniser.line_details(), tok[0], tok[1], token_kind, token_value) assert False logger.debug('Parser : Matched token %s (%s)', tok[0], tok[1]) return tok
def set_setter(self, method): """sets the setter method of the property""" self.set_tag('setter', method) if method == None: return; if len(method.params) != 1: logger.error('Model error: setter method %s of property %s does '+ ' not have exactly 1 parameter', method.name, self.name) if self.data_type == None: self.data_type = method.params[0].data_type; elif self.data_type != method.params[0].data_type: logger.error('Inconsistent types in property %s: %s is not %s', self.name, self.data_type, method.params[0].data_type)
def _setup_class_method(self, class_method): ''' Setup the class's method. The class method is a clone of the current method, in a class wrapper. This comes from @method or @overload in the pascal source. Steps: 1: clone self to class_method 2: add it to its class or property 3: alter args (add pointer field access) ''' self.clone_to(class_method) #copy self into other class_method.is_class_method = True #if the class method is actually a property... if self.is_getter or self.is_setter: self.create_and_add_property(class_method) else: class_method.in_class.add_member(class_method) #add to its class #static methods and constructors directly call the method # instance methods change the first argument for the pointer field if class_method.is_static or class_method.is_constructor: #use self's args - i.e. it calls the method in the same way args = list(self.args) else: #other is an instance (with ptr) if self.args == None or len(self.args) < self.self_pos: logger.error( 'Class method calling a method without parameter for self pointer... ' + self.name) assert False args = list(self.args) #change the old argument for the self pointer if class_method.in_class.is_pointer_wrapper: args[self.self_pos - 1] = 'self.pointer' elif class_method.in_class.wraps_array: args[self.self_pos - 1] = 'self.data' else: args[self.self_pos - 1] = 'self' #add in self's arguments (-1st which is pointer) #set class method to call the same method this does class_method.calls(self.method_called, args) logger.debug('Method : Setting up call: %s calls %s with args %s', class_method, class_method.method_called, class_method.args) class_method._check_args_match_params()
def calls(self, method, args=None): """indicate which method this method calls, and args if any""" if self.method_called != None: logger.error('Model Error: Changing method called by %s', self.name) assert False self.method_called = method self.args = list() for arg in args: if not isinstance(arg, SGParameter): self.args.append(arg) else: self.args.append(self.get_parameter(arg.name))
def _setup_class_method(self, class_method): ''' Setup the class's method. The class method is a clone of the current method, in a class wrapper. This comes from @method or @overload in the pascal source. Steps: 1: clone self to class_method 2: add it to its class or property 3: alter args (add pointer field access) ''' self.clone_to(class_method) #copy self into other class_method.is_class_method = True #if the class method is actually a property... if self.is_getter or self.is_setter: self.create_and_add_property(class_method) else: class_method.in_class.add_member(class_method) #add to its class #static methods and constructors directly call the method # instance methods change the first argument for the pointer field if class_method.is_static or class_method.is_constructor: #use self's args - i.e. it calls the method in the same way args = list(self.args) else: #other is an instance (with ptr) if self.args == None or len(self.args) < self.self_pos: logger.error('Class method calling a method without parameter for self pointer... ' + self.name) assert False args = list(self.args) #change the old argument for the self pointer if class_method.in_class.is_pointer_wrapper: args[self.self_pos - 1] = 'self.pointer' elif class_method.in_class.wraps_array: args[self.self_pos - 1] = 'self.data' else: args[self.self_pos - 1] = 'self' #add in self's arguments (-1st which is pointer) #set class method to call the same method this does class_method.calls(self.method_called, args) logger.debug('Method : Setting up call: %s calls %s with args %s', class_method, class_method.method_called, class_method.args) class_method._check_args_match_params()
def method_process_visitor(the_method, other): '''Process a method prior to rendering. This does all of the transformations that go into the functions and procedures of the SGSDK unit.''' # Change functions with string or array return type to have a result parameter if the_method.return_type != None and ( the_method.return_type.wraps_array or the_method.return_type.name.lower() in ['string']): #print 'PARS RUN PARAMS', [p.name for p in the_method.params] result_param = SGParameter('result') for param in the_method.params: if 'result' == param.name: logger.error('PARSER RUN: Error adding result parameter to %s', the_method.name) assert False _add_parameter(the_method, result_param) result_param.maps_result = True result_param.modifier = 'result' result_param.data_type = the_method.return_type the_method.return_type = None the_method.was_function = True #print 'PARS RUN PARAMS', [p.name for p in the_method.params] #Replace any variable length arrays orig_params = the_method.params for param in orig_params: #does it wrap a variable length array if param.data_type.array_wrapper: the_method.has_length_params = True logger.debug( 'PARSER RUN: Altering variable length array of %s\'s parameter %s', the_method.name, param.name) # old_type = param.data_type #param.data_type = param.data_type #.fields[0].data_type len_param = SGParameter('%s_len' % param.name) len_param.is_length_param = True len_param.data_type = find_or_add_type('Longint') len_param.modifier = param.modifier if param.modifier is [ 'out', 'var' ] else None len_param.length_of = param param.has_length_param = True param.length_idx = _add_parameter(the_method, len_param) return other
def process_uses_clause(self, token): '''Read the list of units referred to be the uses clause''' self._check_non_commented('uses clause') while True: tok = self._match_token('id') self._current_file.uses.append(find_or_add_file(tok[1])) logger.debug('Parser : Found using unit %s', tok[1]) #found a token/unit next_tok = self._match_token('symbol') if next_tok[1] == ';': break; #found end elif next_tok[1] != ',': logger.error('Parser Error %s: expected , or ; but found %s at %s', self._tokeniser.line_details(), next_tok[1]) sys.exit(-1)
def _match_one_token(self, token_kind_lst): matched = False tok = self._next_token() for token_kind,token_value in token_kind_lst: if tok[0] == token_kind and (token_value == None or token_value == tok[1]): matched = True logger.debug('Parser : Matched %s with %s', tok[0], tok[1]) break if not matched: logger.error('Parser Error %s: unexpected %s(%s) expected %s', self._tokeniser.line_details(), tok[0], tok[1], map(lambda n: '%s(%s)' % (n[0],n[1]),token_kind_lst)) assert False return tok
def _check_args_match_params(self): ''' Ensure that the arguments in the call match the available parameters, if no arguments are provided copy across read in parameters ''' method_called = self.method_called #self.tags['calls'].other[0] #get called method args = self.args #self.tags['calls'].other[1] #the arguments self passes to the method logger.debug('Method : Checking arguments used by %s calling %s (%s)', self, method_called, args) if args == None: logger.error('Method : No arguments that map to parameters for %s', self) assert False else: for arg in args: if isinstance(arg, SGParameter) and not self.has_parameter(arg.name): logger.error("Cannot match parameter %s in call to %s from %s", str(arg), str(method_called), self) assert False
def _match_one_token(self, token_kind_lst): matched = False tok = self._next_token() for token_kind, token_value in token_kind_lst: if tok[0] == token_kind and (token_value == None or token_value == tok[1]): matched = True logger.debug('Parser : Matched %s with %s', tok[0], tok[1]) break if not matched: logger.error( 'Parser Error %s: unexpected %s(%s) expected %s', self._tokeniser.line_details(), tok[0], tok[1], map(lambda n: '%s(%s)' % (n[0], n[1]), token_kind_lst)) assert False return tok
def process_uses_clause(self, token): '''Read the list of units referred to be the uses clause''' self._check_non_commented('uses clause') while True: tok = self._match_token('id') self._current_file.uses.append(find_or_add_file(tok[1])) logger.debug('Parser : Found using unit %s', tok[1]) #found a token/unit next_tok = self._match_token('symbol') if next_tok[1] == ';': break #found end elif next_tok[1] != ',': logger.error( 'Parser Error %s: expected , or ; but found %s at %s', self._tokeniser.line_details(), next_tok[1]) sys.exit(-1)
def process_meta_comments(self): logger.debug('Parser : Starting to process meta comments: clearing old comments and attributes') #self._meta_comments = [] #self._attributes = {} tok = self._lookahead(1)[0] #_next_token() attrs_started = False while tok[0] in ['meta comment','attribute','comment']: if tok[0] == 'attribute': attrs_started = True if attrs_started and tok[0] == 'meta comment': tok = self._next_token() #actually read token if len(tok[1]) > 0: logger.error('Parser Error %s: Found additional meta comment after start of attributes', self._tokeniser.line_details()) assert False else: tok = self._next_token() #actually read token self._processors[tok[0]](tok) tok = self._lookahead(1)[0]
def method_process_visitor(the_method, other): '''Process a method prior to rendering. This does all of the transformations that go into the functions and procedures of the SGSDK unit.''' # Change functions with string or array return type to have a result parameter if the_method.return_type != None and (the_method.return_type.wraps_array or the_method.return_type.name.lower() in ['string']): #print 'PARS RUN PARAMS', [p.name for p in the_method.params] result_param = SGParameter('result') for param in the_method.params: if 'result' == param.name: logger.error('PARSER RUN: Error adding result parameter to %s', the_method.name) assert False _add_parameter(the_method, result_param) result_param.maps_result = True result_param.modifier = 'result' result_param.data_type = the_method.return_type the_method.return_type = None the_method.was_function = True #print 'PARS RUN PARAMS', [p.name for p in the_method.params] #Replace any variable length arrays orig_params = the_method.params for param in orig_params: #does it wrap a variable length array if param.data_type.array_wrapper: the_method.has_length_params = True logger.debug('PARSER RUN: Altering variable length array of %s\'s parameter %s', the_method.name, param.name) # old_type = param.data_type #param.data_type = param.data_type #.fields[0].data_type len_param = SGParameter('%s_len' % param.name) len_param.is_length_param = True len_param.data_type = find_or_add_type('Longint') len_param.modifier = param.modifier if param.modifier is ['out', 'var'] else None len_param.length_of = param param.has_length_param = True param.length_idx = _add_parameter(the_method, len_param) return other
def check_arguments(self): ''' Ensure that the arguments in the call match the available parameters, if no arguments are provided copy across read in parameters ''' if self.method_called == None: logger.error( 'Method : Method %s does not call anything. Check attributes. %s', self.uname, self.file_line_details) assert False method_called = self.method_called #get called method args = self.args #the arguments self passes to the method logger.debug( 'Method : Checking arguments used by %s calling %s (%s)', self, method_called, args) if len(args) != len(method_called.params): logger.error('Method : Error in %s calling %s', self.uname, method_called.uname) assert False for arg in args: if isinstance(arg, SGParameter): if not self.has_parameter(arg.name): logger.error( "Cannot match parameter %s in call to %s from %s", str(arg), str(method_called), self) assert False
def create_and_add_property(self, class_method): ''' The added class method is actually a property, so create and add a property or update the existing property. ''' property_name = self.in_property #get or create the property if property_name in self.other_class.properties: prop = self.other_class.properties[property_name] else: prop = SGProperty(property_name) prop.in_class = self.other_class prop.is_static = self.is_static prop.in_file = self.in_file #add property to class self.other_class.add_member(prop) #setup name of method and its position in the property if self.is_getter: class_method.name = 'get' + property_name class_method.is_getter = True self.is_getter = False #transfer to other methods prop.getter = class_method elif self.is_setter: class_method.name = 'set' + property_name prop.setter = class_method class_method.is_setter = True self.is_setter = False #transfer to other methods #class_method.params[0].name = 'value' else: logger.error('Property is not a getter or a setter: %s - %s', self.name, property_name) assert False #change uname as well... class_method.uname = class_method.name class_method.in_property = prop self.in_property = None
def create_and_add_property(self, class_method): ''' The added class method is actually a property, so create and add a property or update the existing property. ''' property_name = self.in_property #get or create the property if property_name in self.other_class.properties: prop = self.other_class.properties[property_name] else: prop = SGProperty(property_name) prop.in_class = self.other_class; prop.is_static = self.is_static; prop.in_file = self.in_file #add property to class self.other_class.add_member(prop) #setup name of method and its position in the property if self.is_getter: class_method.name = 'get' + property_name class_method.is_getter = True self.is_getter = False #transfer to other methods prop.getter = class_method elif self.is_setter: class_method.name = 'set' + property_name prop.setter = class_method class_method.is_setter = True self.is_setter = False #transfer to other methods #class_method.params[0].name = 'value' else: logger.error('Property is not a getter or a setter: %s - %s', self.name, property_name) assert False #change uname as well... class_method.uname = class_method.name class_method.in_property = prop self.in_property = None
def process_meta_comments(self): logger.debug( 'Parser : Starting to process meta comments: clearing old comments and attributes' ) #self._meta_comments = [] #self._attributes = {} tok = self._lookahead(1)[0] #_next_token() attrs_started = False while tok[0] in ['meta comment', 'attribute', 'comment']: if tok[0] == 'attribute': attrs_started = True if attrs_started and tok[0] == 'meta comment': tok = self._next_token() #actually read token if len(tok[1]) > 0: logger.error( 'Parser Error %s: Found additional meta comment after start of attributes', self._tokeniser.line_details()) assert False else: tok = self._next_token() #actually read token self._processors[tok[0]](tok) tok = self._lookahead(1)[0]
def parse(self, a_file): #clear all existing data self._lookahead_toks = [] self._meta_comments = [] self._attributes = {} self._ordered_attributes = [] self._current_file = a_file self._tokeniser.tokenise(a_file.filename) #read the meta comments before the node self.process_meta_comments() #read the token after meta-comments tok = self._next_token() if tok[0] == 'id' and tok[1] in ['unit', 'library']: try: self._file_processors[tok[1]](tok) except: logger.error("Parse Error on file %s" % a_file.filename) raise else: logger.error('Parse Error %s: found %s expected unit or library', self._tokeniser.line_details(), tok[1]) assert False
def _check_args_match_params(self): ''' Ensure that the arguments in the call match the available parameters, if no arguments are provided copy across read in parameters ''' method_called = self.method_called #self.tags['calls'].other[0] #get called method args = self.args #self.tags['calls'].other[1] #the arguments self passes to the method logger.debug( 'Method : Checking arguments used by %s calling %s (%s)', self, method_called, args) if args == None: logger.error( 'Method : No arguments that map to parameters for %s', self) assert False else: for arg in args: if isinstance( arg, SGParameter) and not self.has_parameter(arg.name): logger.error( "Cannot match parameter %s in call to %s from %s", str(arg), str(method_called), self) assert False
def parse(self, a_file): #clear all existing data self._lookahead_toks = [] self._meta_comments = [] self._attributes = {} self._ordered_attributes = [] self._current_file = a_file self._tokeniser.tokenise(a_file.filename) #read the meta comments before the node self.process_meta_comments() #read the token after meta-comments tok = self._next_token() if tok[0] == 'id' and tok[1] in ['unit','library']: try: self._file_processors[tok[1]](tok) except: logger.error("Parse Error on file %s" % a_file.filename) raise else: logger.error('Parse Error %s: found %s expected unit or library', self._tokeniser.line_details(), tok[1]) assert False
def check_arguments(self): ''' Ensure that the arguments in the call match the available parameters, if no arguments are provided copy across read in parameters ''' if self.method_called == None: logger.error('Method : Method %s does not call anything. Check attributes. %s', self.uname, self.file_line_details) assert False method_called = self.method_called #get called method args = self.args #the arguments self passes to the method logger.debug('Method : Checking arguments used by %s calling %s (%s)', self, method_called, args) if len(args) != len(method_called.params): logger.error('Method : Error in %s calling %s', self.uname, method_called.uname) assert False for arg in args: if isinstance(arg, SGParameter): if not self.has_parameter(arg.name): logger.error("Cannot match parameter %s in call to %s from %s", str(arg), str(method_called), self) assert False
def set_tag(self, title, other=None): if title == "params": #process parameter comments for param_details in other: param_name = param_details[0] param_doc = param_details[1] done = False for param in self.params: if param.name == param_name: param.add_doc(param_doc) done = True break if not done: logger.error( 'Method : Unable to find parameter %s for %s', param_name, self.uname) assert False elif title == 'related_params': # The parameters have related documentation # The passed in details has the param name, and the list of its related # parameters for related_param_details in other: param_name = related_param_details[0] related_params = related_param_details[1] # Find the parameter done = False for param in self.params: # if this is the parameter... if param.name == param_name: # then add related docs param.add_related_params(related_params) done = True break if not done: logger.error( 'Method : Unable to find parameter %s for %s', param_name, self.uname) assert False elif title == 'updatesArrayParams': # check which parameters are being updated and mark so that checking ignores them for idx in other: self.params[idx - 1].being_updated = True elif title == "class": #the class indicates that the @method is for this other class... from sg_code_module import SGCodeModule from sg_library import SGLibrary if other == None: super(SGMethod, self).set_tag(title, None) return elif isinstance(other, SGCodeModule): other_class = other elif isinstance(other, SGLibrary): other_class = other else: other_class = find_or_add_class(other) super(SGMethod, self).set_tag('other_class', other_class) elif title == 'getter' or title == 'setter': # 1: mark as getter/setter super(SGMethod, self).set_tag('is_' + title, True) # 2: set property name self.in_property = other # 3: mark for later processing mthd = SGMethod(other + ' ' + title) super(SGMethod, self).set_tag('class_method', mthd) elif title == 'constructor': const = SGMethod(self.other_class.name) const.is_constructor = True super(SGMethod, self).set_tag('class_method', const) elif title == 'dispose': dest = SGMethod("~" + self.other_class.name) dest.is_destructor = True self.mimic_destructor = True super(SGMethod, self).set_tag('class_method', dest) elif title == 'csn': #assign the 'csn' class special name to the class method if self.class_method == None: logger.error( 'Model Error: Method %s has a csn before the method/property/constructor definition - or should be sn.', self.name) assert False self.class_method.set_tag('sn', other) else: super(SGMethod, self).set_tag(title, other)
def _parse_type_declaration(self, the_type): ''' Parse a type from the next token, add details to the_type. this is called for each type declaration... type_name = BLAH Need to process BLAH and add details to the_type ''' #check what kind of type it is... if self._match_lookahead('symbol', '(', True): values = [] while not self._match_lookahead('symbol', ')'): temp = self._match_token('id')[1] if self._match_lookahead( 'operator', '=', True) or self._match_lookahead( 'operator', ':=', True): #assigned value values.append('%s = %s' % (temp, self._match_token('number')[1])) else: values.append(temp) self._match_lookahead('symbol', ',', True) #consume commas self._match_token('symbol', ')') #read close bracket the_type.values = tuple(values) elif (self._match_lookahead('id', 'packed', True) and self._match_lookahead('id', 'record', True)) or self._match_lookahead( 'id', 'record', True): #packed record field: Type end; while not self._match_lookahead('id', 'end', True): #read field variables = self._read_variable_list() self._match_token('symbol', ';') for name, data_type in variables: field = SGField(name) field.data_type = data_type the_type.fields.append(field) elif self._match_lookahead('id', 'array') or self._match_lookahead( 'symbol', '^'): #is pointer or array the_type.clone(self._read_type_usage()) elif self._match_lookahead('id', 'procedure', True): #procedure type: read ( params ); self._match_token('symbol', '(') m = SGMethod(the_type.name) m.in_file = self._current_file self._read_params(m) the_type.is_procedure = True the_type.method = m if self._lookahead(2)[1][1] == 'cdecl': self._match_token('symbol', ';') self._match_token('id', 'cdecl') elif self._match_lookahead('id'): other_id = self._match_token('id')[1] other_type = find_or_add_type(other_id) the_type.related_type = other_type else: tok = self._next_token() logger.error('Parser Error %s: unknown type %s(%s)', self._tokeniser.line_details(), tok[0], tok[1]) assert False if the_type.related_type != None: logger.info(' Parser : Setup type %s = %s', the_type, the_type.related_type) elif the_type.is_enum: logger.info(' Parser : Setup type %s = %s', the_type, the_type.values) else: logger.info(' Parser : Setup type %s = %s', the_type, the_type.fields) self._apply_attributes_to(the_type) self._match_token('symbol', ';')
def complete_method_processing(self): ''' This is called on methods that are read by the parser from the Pascal file. Set up the call from the library to this method if marked. Check the call's validity Steps: 1: Get other methods related to this one 2: Set parameters on library method (if called) ''' logger.info(' Method : Completing processing of %s', self) #This is 'the' method it has its params self.params = self.params #Find the length method if it exists if self.length_call != None: self.length_call = self.in_class.find_method(self.length_call) #Convert args to appropriate values... self._process_args() self._check_args_match_params() #Get other methods lib_method = self.method_called class_method = self.class_method #check rules if lib_method == None and not self.is_operator: logger.error('Method : Found method %s without lib', self) assert False if lib_method != None: #set up library method self.setup_lib_method(lib_method) logger.info(' Method : %s calls %s', self.name, lib_method.name) lib_method.called_by.append(self) #set up class method if class_method != None: logger.debug(' Method : %s is also %s', self.name, class_method) self._setup_class_method(class_method) logger.info(' Method : %s calls %s', class_method.name, lib_method.name) lib_method.called_by.append( class_method) #library is also called by class elif self.is_operator: assert self.other_class != None self.name = 'operator ' + self.name self.other_method = self.in_class.find_method(self['calls'].other) self.doc = self.other_method.doc self.method_called = self.other_method.method_called self.method_called.called_by.append( self) #library is also called by operator self.is_static = True self.in_class.operators[self.signature] = None self.in_class = self.other_class self.in_class.add_member(self) self.args = list(self.params) # operators must match directly
def next_token(self): '''Find and return a tuple with the next token details. Tuple contains the token (type, value) as strings. The token types and details are: number, # such as 1234, 123.45, -123, +123.4 comment, # single // or multi-line (* ... *), { ... } meta comment, # start with /// ... id, # identifier name starting with alpha, including # alpha-numeric characters and the _ character attribute, # name starting with @... inside meta comment block # follows the id name character rules operator, # one of + - / * ** := < > <= >= <> symbol, # one of ':;,.()' ''' def num_match(cha, tmp): '''Checks for a number in format ##, ##.#. Returns False when at the end of a number.''' if cha in '1234567890': return True elif cha == '.' and '.' not in tmp: return self._peek(1) in '1234567890' else: return False while (True): t = self._next_char() self._token_start = self._char_no # Ignore white space characters if t == ' ' or t == '\t': #ignore white space pass # Move to next line (if at end of line) elif t == '\n': self._advance_line() # Numbers (int or float style format elif t in '1234567890' or (t in '-+' and self._peek(1) in '1234567890'): #is digit or +/- result = ('number', self._read_matching(t, num_match)) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result # Comment, single line // or meta comment line /// elif t == '/' and self._peek(1) == '/': #start of comment if self._match_and_read('/'): if self._match_and_read('/'): kind = 'meta comment' self._meta_comment_start = self._char_no self._meta_comment_line = self._line_no comment = self.read_to_end_of_comment() else: kind = 'comment' comment = self.read_to_eol() result = (kind, comment) else: result = ('error', t) logger.debug('Tokenisier: read %s', result[0]) return result # Attribute identified by an @ symbol then a name elif t == '@': name = self._read_matching( '', lambda cha, tmp: cha.isalnum() or cha == '_') result = ('attribute', name) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result # Identifier (id) of alphanumeric characters including elif t.isalpha(): name = self._read_matching( t, lambda cha, tmp: cha.isalnum() or cha == '_') if name.lower() in ['true', 'false']: result = ('boolean', name) else: result = ('id', name) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result #Bound Comment elif t == '{' or (t == '(' and self._peek(1) == '*'): if t == '(' and self._match_and_read('*'): comment = self._read_until('', lambda temp: temp[-2:] == '*)') result = ('comment', comment[:-2]) elif t == '{': comment = self._read_until('', lambda temp: temp[-1:] == '}') result = ('comment', comment[:-1]) logger.log(logging.DEBUG, 'Tokenisier: read %s', result[0]) return result # Operator elif (t == ':' and self._peek(1) == '=') or t in '=+-*/><': if t == ':' and self._match_and_read('='): result = ('operator', ':=') elif t == '*' and self._match_and_read('*'): result = ('operator', '**') elif t == '<' and self._match_and_read('>'): result = ('operator', '<>') elif t in '<>' and self._match_and_read('='): result = ('operator', t + '=') else: result = ('operator', t) return result # Symbol elif t in '(),:;[].^': result = ('symbol', t) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result # Catch any single quotes inside a string value. elif t == "'": string = self._read_until( '', lambda temp: (temp[-1:] == "'") and (not self._match_and_read("'"))) result = ('string', string[:-1]) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result # Hmm.. unknown token. What did we forget? else: logger.error("Unknown token type: " + t) return ('error', t)
def set_tag(self, title, other = None): if title == "params": #process parameter comments for param_details in other: param_name = param_details[0] param_doc = param_details[1] done = False for param in self.params: if param.name == param_name: param.add_doc(param_doc) done = True break if not done: logger.error('Method : Unable to find parameter %s for %s', param_name, self.uname) assert False elif title == 'related_params': # The parameters have related documentation # The passed in details has the param name, and the list of its related # parameters for related_param_details in other: param_name = related_param_details[0] related_params = related_param_details[1] # Find the parameter done = False for param in self.params: # if this is the parameter... if param.name == param_name: # then add related docs param.add_related_params(related_params) done = True break if not done: logger.error('Method : Unable to find parameter %s for %s', param_name, self.uname) assert False elif title == 'updatesArrayParams': # check which parameters are being updated and mark so that checking ignores them for idx in other: self.params[idx - 1].being_updated = True elif title == "class": #the class indicates that the @method is for this other class... from sg_code_module import SGCodeModule from sg_library import SGLibrary if other == None: super(SGMethod,self).set_tag(title, None) return elif isinstance(other, SGCodeModule): other_class = other elif isinstance(other, SGLibrary): other_class = other else: other_class = find_or_add_class(other) super(SGMethod,self).set_tag('other_class', other_class) elif title == 'getter' or title == 'setter': # 1: mark as getter/setter super(SGMethod,self).set_tag('is_' + title, True) # 2: set property name self.in_property = other # 3: mark for later processing mthd = SGMethod(other + ' ' + title) super(SGMethod,self).set_tag('class_method', mthd) elif title == 'constructor': const = SGMethod(self.other_class.name) const.is_constructor = True super(SGMethod,self).set_tag('class_method', const) elif title == 'dispose': dest = SGMethod("~" + self.other_class.name) dest.is_destructor = True self.mimic_destructor = True super(SGMethod,self).set_tag('class_method', dest) elif title == 'csn': #assign the 'csn' class special name to the class method if self.class_method == None: logger.error('Model Error: Method %s has a csn before the method/property/constructor definition - or should be sn.', self.name) assert False self.class_method.set_tag('sn', other) else: super(SGMethod,self).set_tag(title, other)
def _check_meta_comments(self, count, desc): if len(self._meta_comments) != count: logger.error('Parser Error %s: expected %d but foung %d meta comments for %s', self._tokeniser.line_details(), count, len(self._meta_comments), desc) sys.exit(-1)
def next_token(self): '''Find and return a tuple with the next token details. Tuple contains the token (type, value) as strings. The token types and details are: number, # such as 1234, 123.45, -123, +123.4 comment, # single // or multi-line (* ... *), { ... } meta comment, # start with /// ... id, # identifier name starting with alpha, including # alpha-numeric characters and the _ character attribute, # name starting with @... inside meta comment block # follows the id name character rules operator, # one of + - / * ** := < > <= >= <> symbol, # one of ':;,.()' ''' def num_match(cha, tmp): '''Checks for a number in format ##, ##.#. Returns False when at the end of a number.''' if cha in '1234567890': return True elif cha == '.' and '.' not in tmp: return self._peek(1) in '1234567890' else: return False while (True): t = self._next_char(); self._token_start = self._char_no # Ignore white space characters if t == ' ' or t == '\t': #ignore white space pass # Move to next line (if at end of line) elif t == '\n': self._advance_line() # Numbers (int or float style format elif t in '1234567890' or (t in '-+' and self._peek(1) in '1234567890'): #is digit or +/- result = ('number', self._read_matching(t, num_match)) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result # Comment, single line // or meta comment line /// elif t == '/' and self._peek(1) == '/': #start of comment if self._match_and_read('/'): if self._match_and_read('/'): kind = 'meta comment' self._meta_comment_start = self._char_no self._meta_comment_line = self._line_no comment = self.read_to_end_of_comment() else: kind = 'comment' comment = self.read_to_eol() result = (kind, comment) else: result = ('error', t) logger.debug('Tokenisier: read %s', result[0]) return result # Attribute identified by an @ symbol then a name elif t == '@': name = self._read_matching('', lambda cha, tmp: cha.isalnum() or cha == '_') result = ('attribute', name) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result # Identifier (id) of alphanumeric characters including elif t.isalpha(): name = self._read_matching(t, lambda cha, tmp: cha.isalnum() or cha == '_') if name.lower() in ['true','false']: result = ('boolean', name) else: result = ('id', name) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result #Bound Comment elif t == '{' or (t == '(' and self._peek(1) == '*'): if t == '(' and self._match_and_read('*'): comment = self._read_until('', lambda temp: temp[-2:] == '*)') result = ('comment', comment[:-2]) elif t == '{': comment = self._read_until('', lambda temp: temp[-1:] == '}') result = ('comment', comment[:-1]) logger.log(logging.DEBUG, 'Tokenisier: read %s', result[0]) return result # Operator elif (t == ':' and self._peek(1) == '=') or t in '=+-*/><': if t == ':' and self._match_and_read('='): result = ('operator', ':=') elif t == '*' and self._match_and_read('*'): result = ('operator', '**') elif t == '<' and self._match_and_read('>'): result = ('operator', '<>') elif t in '<>' and self._match_and_read('='): result = ('operator', t + '=') else: result = ('operator', t) return result # Symbol elif t in '(),:;[].^': result = ('symbol', t) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result # Catch any single quotes inside a string value. elif t == "'": string = self._read_until('', lambda temp: (temp[-1:] == "'") and (not self._match_and_read("'"))) result = ('string', string[:-1]) logger.debug('Tokenisier: read %s - %s', result[0], result[1]) return result # Hmm.. unknown token. What did we forget? else: logger.error("Unknown token type: "+t) return ('error', t)