def _read_params(self, method):
"""Read in the parameters declared in a function, procedure or operator"""
#look for parameters
while not self._match_lookahead('symbol', ')'): #consume ) at end
param_tok, other_tok = self._lookahead(2)
if param_tok[0] == 'id':
if other_tok[0] == 'id': #first one is a modifier
modifier = self._next_token()[1]
else:
modifier = None
param_toks = [self._match_token('id')]
while self._match_lookahead('symbol', ',', True):
#there is a list of parameters
param_toks.append(self._match_token('id'))
colon_tok = self._match_token('symbol', ':')
the_type = self._read_type_usage()
for param_tok in param_toks:
param = method.create_parameter(param_tok[1])
param.data_type = the_type
param.modifier = modifier
logger.debug('Parser : Adding parameter %s (%s) to %s', param.name, param.data_type, method.name)
if not self._match_lookahead('symbol', ';', True): break
else:
logger.error('Parser : Error in parameter list %s', self._tokeniser.line_details())
self._match_token('symbol', ')')
def std_type_visitor(the_dict, the_type, modifier = None, dict_name = '_type_switcher'):
'''
switch types for the SwinGame library.
Params:
- the_dict: The dictionary with the type changes with modifier keys
- the_type: The type being checked
- modifier: The modifier for the type
- dict_name: The name of the dictionary for error reporting
'''
key = the_type.name.lower() if the_type != None else None
if modifier == 'result': modifier = 'return'
if key not in the_dict[modifier]:
logger.error('WRAPPER : Error changing model type %s - %s', modifier, the_type)
logger.error(' : Add \'%s[%s]\': \'%s\': \'????\',', dict_name, modifier, the_type.name.lower())
global _hasError, _dieOnError
_hasError = True
if _dieOnError:
assert False
else:
the_dict[modifier][key] = "UNKNOWN"
return the_dict[modifier][key]
def _check_meta_comments(self, count, desc):
if len(self._meta_comments) != count:
logger.error(
'Parser Error %s: expected %d but foung %d meta comments for %s',
self._tokeniser.line_details(), count,
len(self._meta_comments), desc)
sys.exit(-1)
def _parse_type_declaration(self, the_type):
'''
Parse a type from the next token, add details to the_type.
this is called for each type declaration... type_name = BLAH
Need to process BLAH and add details to the_type
'''
#check what kind of type it is...
if self._match_lookahead('symbol', '(', True):
values = []
while not self._match_lookahead('symbol',')'):
temp = self._match_token('id')[1]
if self._match_lookahead('operator', '=', True) or self._match_lookahead('operator', ':=', True): #assigned value
values.append('%s = %s' % (temp, self._match_token('number')[1]))
else:
values.append(temp)
self._match_lookahead('symbol', ',', True) #consume commas
self._match_token('symbol',')') #read close bracket
the_type.values = tuple(values)
elif (self._match_lookahead('id', 'packed', True) and self._match_lookahead('id', 'record', True)) or self._match_lookahead('id', 'record', True):
#packed record field: Type end;
while not self._match_lookahead('id', 'end', True):
#read field
variables = self._read_variable_list()
self._match_token('symbol', ';')
for name, data_type in variables:
field = SGField(name)
field.data_type = data_type
the_type.fields.append(field)
elif self._match_lookahead('id', 'array') or self._match_lookahead('symbol', '^'):
#is pointer or array
the_type.clone(self._read_type_usage())
elif self._match_lookahead('id', 'procedure', True):
#procedure type: read ( params );
self._match_token('symbol', '(')
m = SGMethod(the_type.name)
m.in_file = self._current_file
self._read_params(m)
the_type.is_procedure = True
the_type.method = m
if self._lookahead(2)[1][1] == 'cdecl':
self._match_token('symbol', ';')
self._match_token('id', 'cdecl')
elif self._match_lookahead('id'):
other_id = self._match_token('id')[1]
other_type = find_or_add_type(other_id)
the_type.related_type = other_type
else:
tok = self._next_token()
logger.error('Parser Error %s: unknown type %s(%s)',
self._tokeniser.line_details(), tok[0], tok[1])
assert False
if the_type.related_type != None:
logger.info(' Parser : Setup type %s = %s', the_type, the_type.related_type)
elif the_type.is_enum:
logger.info(' Parser : Setup type %s = %s', the_type, the_type.values)
else:
logger.info(' Parser : Setup type %s = %s', the_type, the_type.fields)
self._apply_attributes_to(the_type)
self._match_token('symbol', ';')
def create_property_for_field(in_class, field):
'''Creates a property to access the '''
if field.data_type.wraps_array:
logger.error('WRAPPER : Error structure with array fields must be set to via_pointer')
global _hasError, _dieOnError
_hasError = True
if _dieOnError:
assert False
prop = SGProperty(field.name)
prop.in_class = in_class
prop.data_type = field.data_type
getter = SGMethod('get' + field.pascalName)
getter.return_type = field.data_type
getter.in_class = in_class
getter.field_name = field.name
getter.is_getter = True
setter = SGMethod('set' + field.pascalName)
setter.params.append(SGParameter('value'))
setter.params[0].data_type = field.data_type
setter.in_class = in_class
setter.field_name = field.name
setter.is_setter = True
prop.set_getter(getter)
prop.set_setter(setter)
return prop
def complete_method_processing(self):
'''
This is called on methods that are read by the parser from the
Pascal file.
Set up the call from the library to this method if marked.
Check the call's validity
Steps:
1: Get other methods related to this one
2: Set parameters on library method (if called)
'''
logger.info(' Method : Completing processing of %s', self)
#This is 'the' method it has its params
self.params = self.params
#Find the length method if it exists
if self.length_call != None:
self.length_call = self.in_class.find_method(self.length_call)
#Convert args to appropriate values...
self._process_args()
self._check_args_match_params()
#Get other methods
lib_method = self.method_called
class_method = self.class_method
#check rules
if lib_method == None and not self.is_operator:
logger.error('Method : Found method %s without lib', self)
assert False
if lib_method != None:
#set up library method
self.setup_lib_method(lib_method)
logger.info(' Method : %s calls %s', self.name, lib_method.name)
lib_method.called_by.append(self)
#set up class method
if class_method != None:
logger.debug(' Method : %s is also %s', self.name, class_method)
self._setup_class_method(class_method)
logger.info(' Method : %s calls %s', class_method.name, lib_method.name)
lib_method.called_by.append(class_method) #library is also called by class
elif self.is_operator:
assert self.other_class != None
self.name = 'operator ' + self.name
self.other_method = self.in_class.find_method(self['calls'].other)
self.doc = self.other_method.doc
self.method_called = self.other_method.method_called
self.method_called.called_by.append(self) #library is also called by operator
self.is_static = True
self.in_class.operators[self.signature] = None
self.in_class = self.other_class
self.in_class.add_member(self)
self.args = list(self.params) # operators must match directly
def _process_args(self):
'''
Convert args to parameters, fields, and literals
'''
new_args = list()
args = self.args
if args == None:
self.args = list(self.params)
return
for argv in args:
if argv[0] in ['number', 'string', 'boolean']:
new_args.append(argv[1])
elif argv[0] in ['id']:
param = self.get_parameter(argv[1])
if param != None:
new_args.append(param)
else:
field = self.in_class.get_field(argv[1])
if field != None:
new_args.append(field)
else:
logger.error(
'Method : Error cannot find %s in method %s',
argv[1], self.uname)
assert False
else:
logger.error(
'Method : Error unknown type of argument in %s - %s',
self.uname, argv[0])
self.args = new_args
def map_data_value(the_dict, key, the_type, result):
'''
Returns the code needed to map the swingame (Pascal) data in result to
the type used by language X
Params:
- the_dict: The dictionary with the type changes with modifier keys
- key: The key to search for in the_dict
- the_type: The type being checked
- modifier: The modifier for the type
'''
if the_type.name.lower() in the_dict[key]:
# print the_type.name.lower(), ' -> ', the_dict[key][the_type.name.lower()]
return the_dict[key][the_type.name.lower()] % result
#If this is a
if the_type.pointer_wrapper and key == 'return_val':
logger.error('WRAPPER : Error pointer wrapper without return data mapping - %s', the_type)
global _hasError, _dieOnError
_hasError = True
if _dieOnError:
assert False
else:
the_dict[key][the_type.name.lower()] = "UNKNOWN"
return result
def _read_params(self, method):
"""Read in the parameters declared in a function, procedure or operator"""
#look for parameters
while not self._match_lookahead('symbol', ')'): #consume ) at end
param_tok, other_tok = self._lookahead(2)
if param_tok[0] == 'id':
if other_tok[0] == 'id': #first one is a modifier
modifier = self._next_token()[1]
else:
modifier = None
param_toks = [self._match_token('id')]
while self._match_lookahead('symbol', ',', True):
#there is a list of parameters
param_toks.append(self._match_token('id'))
colon_tok = self._match_token('symbol', ':')
the_type = self._read_type_usage()
for param_tok in param_toks:
param = method.create_parameter(param_tok[1])
param.data_type = the_type
param.modifier = modifier
logger.debug('Parser : Adding parameter %s (%s) to %s',
param.name, param.data_type, method.name)
if not self._match_lookahead('symbol', ';', True): break
else:
logger.error('Parser : Error in parameter list %s',
self._tokeniser.line_details())
self._match_token('symbol', ')')
def _process_args(self):
'''
Convert args to parameters, fields, and literals
'''
new_args = list()
args = self.args
if args == None:
self.args = list(self.params)
return
for argv in args:
if argv[0] in ['number', 'string', 'boolean']:
new_args.append(argv[1])
elif argv[0] in ['id']:
param = self.get_parameter(argv[1])
if param != None:
new_args.append(param)
else:
field = self.in_class.get_field(argv[1])
if field != None:
new_args.append(field)
else:
logger.error('Method : Error cannot find %s in method %s', argv[1], self.uname)
assert False
else:
logger.error('Method : Error unknown type of argument in %s - %s', self.uname, argv[0])
self.args = new_args
def set_getter(self, method):
"""sets the getter method of the property"""
self.set_tag('getter', method)
if method == None: return;
if self.data_type == None:
self.data_type = method.return_type;
elif self.data_type != method.return_type:
logger.error('Inconsistent types in property %s: %s is not %s',
self.name, self.data_type, method.return_type)
def _match_token(self, token_kind, token_value = None):
tok = self._next_token()
if tok[0] != token_kind or (token_value != None and token_value != tok[1].lower()):
logger.error('Parse Error %s: found a %s (%s) expected %s (%s)',
self._tokeniser.line_details(),
tok[0], tok[1], token_kind, token_value)
assert False
logger.debug('Parser : Matched token %s (%s)', tok[0], tok[1])
return tok
def _match_token(self, token_kind, token_value=None):
tok = self._next_token()
if tok[0] != token_kind or (token_value != None
and token_value != tok[1].lower()):
logger.error('Parse Error %s: found a %s (%s) expected %s (%s)',
self._tokeniser.line_details(), tok[0], tok[1],
token_kind, token_value)
assert False
logger.debug('Parser : Matched token %s (%s)', tok[0], tok[1])
return tok
def set_setter(self, method):
"""sets the setter method of the property"""
self.set_tag('setter', method)
if method == None: return;
if len(method.params) != 1:
logger.error('Model error: setter method %s of property %s does '+
' not have exactly 1 parameter', method.name, self.name)
if self.data_type == None:
self.data_type = method.params[0].data_type;
elif self.data_type != method.params[0].data_type:
logger.error('Inconsistent types in property %s: %s is not %s',
self.name, self.data_type, method.params[0].data_type)
def _setup_class_method(self, class_method):
'''
Setup the class's method.
The class method is a clone of the current method, in
a class wrapper. This comes from @method or @overload
in the pascal source.
Steps:
1: clone self to class_method
2: add it to its class or property
3: alter args (add pointer field access)
'''
self.clone_to(class_method) #copy self into other
class_method.is_class_method = True
#if the class method is actually a property...
if self.is_getter or self.is_setter:
self.create_and_add_property(class_method)
else:
class_method.in_class.add_member(class_method) #add to its class
#static methods and constructors directly call the method
# instance methods change the first argument for the pointer field
if class_method.is_static or class_method.is_constructor:
#use self's args - i.e. it calls the method in the same way
args = list(self.args)
else: #other is an instance (with ptr)
if self.args == None or len(self.args) < self.self_pos:
logger.error(
'Class method calling a method without parameter for self pointer... '
+ self.name)
assert False
args = list(self.args)
#change the old argument for the self pointer
if class_method.in_class.is_pointer_wrapper:
args[self.self_pos - 1] = 'self.pointer'
elif class_method.in_class.wraps_array:
args[self.self_pos - 1] = 'self.data'
else:
args[self.self_pos - 1] = 'self'
#add in self's arguments (-1st which is pointer)
#set class method to call the same method this does
class_method.calls(self.method_called, args)
logger.debug('Method : Setting up call: %s calls %s with args %s',
class_method, class_method.method_called,
class_method.args)
class_method._check_args_match_params()
def calls(self, method, args=None):
"""indicate which method this method calls, and args if any"""
if self.method_called != None:
logger.error('Model Error: Changing method called by %s', self.name)
assert False
self.method_called = method
self.args = list()
for arg in args:
if not isinstance(arg, SGParameter):
self.args.append(arg)
else:
self.args.append(self.get_parameter(arg.name))
def calls(self, method, args=None):
"""indicate which method this method calls, and args if any"""
if self.method_called != None:
logger.error('Model Error: Changing method called by %s',
self.name)
assert False
self.method_called = method
self.args = list()
for arg in args:
if not isinstance(arg, SGParameter):
self.args.append(arg)
else:
self.args.append(self.get_parameter(arg.name))
def _setup_class_method(self, class_method):
'''
Setup the class's method.
The class method is a clone of the current method, in
a class wrapper. This comes from @method or @overload
in the pascal source.
Steps:
1: clone self to class_method
2: add it to its class or property
3: alter args (add pointer field access)
'''
self.clone_to(class_method) #copy self into other
class_method.is_class_method = True
#if the class method is actually a property...
if self.is_getter or self.is_setter:
self.create_and_add_property(class_method)
else:
class_method.in_class.add_member(class_method) #add to its class
#static methods and constructors directly call the method
# instance methods change the first argument for the pointer field
if class_method.is_static or class_method.is_constructor:
#use self's args - i.e. it calls the method in the same way
args = list(self.args)
else: #other is an instance (with ptr)
if self.args == None or len(self.args) < self.self_pos:
logger.error('Class method calling a method without parameter for self pointer... ' + self.name)
assert False
args = list(self.args)
#change the old argument for the self pointer
if class_method.in_class.is_pointer_wrapper:
args[self.self_pos - 1] = 'self.pointer'
elif class_method.in_class.wraps_array:
args[self.self_pos - 1] = 'self.data'
else:
args[self.self_pos - 1] = 'self'
#add in self's arguments (-1st which is pointer)
#set class method to call the same method this does
class_method.calls(self.method_called, args)
logger.debug('Method : Setting up call: %s calls %s with args %s', class_method,
class_method.method_called, class_method.args)
class_method._check_args_match_params()
def method_process_visitor(the_method, other):
'''Process a method prior to rendering. This does all of the transformations
that go into the functions and procedures of the SGSDK unit.'''
# Change functions with string or array return type to have a result parameter
if the_method.return_type != None and (
the_method.return_type.wraps_array
or the_method.return_type.name.lower() in ['string']):
#print 'PARS RUN PARAMS', [p.name for p in the_method.params]
result_param = SGParameter('result')
for param in the_method.params:
if 'result' == param.name:
logger.error('PARSER RUN: Error adding result parameter to %s',
the_method.name)
assert False
_add_parameter(the_method, result_param)
result_param.maps_result = True
result_param.modifier = 'result'
result_param.data_type = the_method.return_type
the_method.return_type = None
the_method.was_function = True
#print 'PARS RUN PARAMS', [p.name for p in the_method.params]
#Replace any variable length arrays
orig_params = the_method.params
for param in orig_params:
#does it wrap a variable length array
if param.data_type.array_wrapper:
the_method.has_length_params = True
logger.debug(
'PARSER RUN: Altering variable length array of %s\'s parameter %s',
the_method.name, param.name)
# old_type = param.data_type
#param.data_type = param.data_type #.fields[0].data_type
len_param = SGParameter('%s_len' % param.name)
len_param.is_length_param = True
len_param.data_type = find_or_add_type('Longint')
len_param.modifier = param.modifier if param.modifier is [
'out', 'var'
] else None
len_param.length_of = param
param.has_length_param = True
param.length_idx = _add_parameter(the_method, len_param)
return other
def process_uses_clause(self, token):
'''Read the list of units referred to be the uses clause'''
self._check_non_commented('uses clause')
while True:
tok = self._match_token('id')
self._current_file.uses.append(find_or_add_file(tok[1]))
logger.debug('Parser : Found using unit %s', tok[1])
#found a token/unit
next_tok = self._match_token('symbol')
if next_tok[1] == ';':
break; #found end
elif next_tok[1] != ',':
logger.error('Parser Error %s: expected , or ; but found %s at %s',
self._tokeniser.line_details(), next_tok[1])
sys.exit(-1)
def _match_one_token(self, token_kind_lst):
matched = False
tok = self._next_token()
for token_kind,token_value in token_kind_lst:
if tok[0] == token_kind and (token_value == None or token_value == tok[1]):
matched = True
logger.debug('Parser : Matched %s with %s', tok[0], tok[1])
break
if not matched:
logger.error('Parser Error %s: unexpected %s(%s) expected %s',
self._tokeniser.line_details(),
tok[0], tok[1],
map(lambda n: '%s(%s)' % (n[0],n[1]),token_kind_lst))
assert False
return tok
def _check_args_match_params(self):
'''
Ensure that the arguments in the call match the available parameters,
if no arguments are provided copy across read in parameters
'''
method_called = self.method_called #self.tags['calls'].other[0] #get called method
args = self.args #self.tags['calls'].other[1] #the arguments self passes to the method
logger.debug('Method : Checking arguments used by %s calling %s (%s)', self, method_called, args)
if args == None:
logger.error('Method : No arguments that map to parameters for %s', self)
assert False
else:
for arg in args:
if isinstance(arg, SGParameter) and not self.has_parameter(arg.name):
logger.error("Cannot match parameter %s in call to %s from %s", str(arg), str(method_called), self)
assert False
def _match_one_token(self, token_kind_lst):
matched = False
tok = self._next_token()
for token_kind, token_value in token_kind_lst:
if tok[0] == token_kind and (token_value == None
or token_value == tok[1]):
matched = True
logger.debug('Parser : Matched %s with %s', tok[0], tok[1])
break
if not matched:
logger.error(
'Parser Error %s: unexpected %s(%s) expected %s',
self._tokeniser.line_details(), tok[0], tok[1],
map(lambda n: '%s(%s)' % (n[0], n[1]), token_kind_lst))
assert False
return tok
def process_uses_clause(self, token):
'''Read the list of units referred to be the uses clause'''
self._check_non_commented('uses clause')
while True:
tok = self._match_token('id')
self._current_file.uses.append(find_or_add_file(tok[1]))
logger.debug('Parser : Found using unit %s', tok[1])
#found a token/unit
next_tok = self._match_token('symbol')
if next_tok[1] == ';':
break
#found end
elif next_tok[1] != ',':
logger.error(
'Parser Error %s: expected , or ; but found %s at %s',
self._tokeniser.line_details(), next_tok[1])
sys.exit(-1)
def process_meta_comments(self):
logger.debug('Parser : Starting to process meta comments: clearing old comments and attributes')
#self._meta_comments = []
#self._attributes = {}
tok = self._lookahead(1)[0] #_next_token()
attrs_started = False
while tok[0] in ['meta comment','attribute','comment']:
if tok[0] == 'attribute':
attrs_started = True
if attrs_started and tok[0] == 'meta comment':
tok = self._next_token() #actually read token
if len(tok[1]) > 0:
logger.error('Parser Error %s: Found additional meta comment after start of attributes',
self._tokeniser.line_details())
assert False
else:
tok = self._next_token() #actually read token
self._processors[tok[0]](tok)
tok = self._lookahead(1)[0]
def method_process_visitor(the_method, other):
'''Process a method prior to rendering. This does all of the transformations
that go into the functions and procedures of the SGSDK unit.'''
# Change functions with string or array return type to have a result parameter
if the_method.return_type != None and (the_method.return_type.wraps_array or the_method.return_type.name.lower() in ['string']):
#print 'PARS RUN PARAMS', [p.name for p in the_method.params]
result_param = SGParameter('result')
for param in the_method.params:
if 'result' == param.name:
logger.error('PARSER RUN: Error adding result parameter to %s', the_method.name)
assert False
_add_parameter(the_method, result_param)
result_param.maps_result = True
result_param.modifier = 'result'
result_param.data_type = the_method.return_type
the_method.return_type = None
the_method.was_function = True
#print 'PARS RUN PARAMS', [p.name for p in the_method.params]
#Replace any variable length arrays
orig_params = the_method.params
for param in orig_params:
#does it wrap a variable length array
if param.data_type.array_wrapper:
the_method.has_length_params = True
logger.debug('PARSER RUN: Altering variable length array of %s\'s parameter %s', the_method.name, param.name)
# old_type = param.data_type
#param.data_type = param.data_type #.fields[0].data_type
len_param = SGParameter('%s_len' % param.name)
len_param.is_length_param = True
len_param.data_type = find_or_add_type('Longint')
len_param.modifier = param.modifier if param.modifier is ['out', 'var'] else None
len_param.length_of = param
param.has_length_param = True
param.length_idx = _add_parameter(the_method, len_param)
return other
def check_arguments(self):
'''
Ensure that the arguments in the call match the available parameters,
if no arguments are provided copy across read in parameters
'''
if self.method_called == None:
logger.error(
'Method : Method %s does not call anything. Check attributes. %s',
self.uname, self.file_line_details)
assert False
method_called = self.method_called #get called method
args = self.args #the arguments self passes to the method
logger.debug(
'Method : Checking arguments used by %s calling %s (%s)', self,
method_called, args)
if len(args) != len(method_called.params):
logger.error('Method : Error in %s calling %s', self.uname,
method_called.uname)
assert False
for arg in args:
if isinstance(arg, SGParameter):
if not self.has_parameter(arg.name):
logger.error(
"Cannot match parameter %s in call to %s from %s",
str(arg), str(method_called), self)
assert False
def create_and_add_property(self, class_method):
'''
The added class method is actually a property, so
create and add a property or update the existing property.
'''
property_name = self.in_property
#get or create the property
if property_name in self.other_class.properties:
prop = self.other_class.properties[property_name]
else:
prop = SGProperty(property_name)
prop.in_class = self.other_class
prop.is_static = self.is_static
prop.in_file = self.in_file
#add property to class
self.other_class.add_member(prop)
#setup name of method and its position in the property
if self.is_getter:
class_method.name = 'get' + property_name
class_method.is_getter = True
self.is_getter = False #transfer to other methods
prop.getter = class_method
elif self.is_setter:
class_method.name = 'set' + property_name
prop.setter = class_method
class_method.is_setter = True
self.is_setter = False #transfer to other methods
#class_method.params[0].name = 'value'
else:
logger.error('Property is not a getter or a setter: %s - %s',
self.name, property_name)
assert False
#change uname as well...
class_method.uname = class_method.name
class_method.in_property = prop
self.in_property = None
def create_and_add_property(self, class_method):
'''
The added class method is actually a property, so
create and add a property or update the existing property.
'''
property_name = self.in_property
#get or create the property
if property_name in self.other_class.properties:
prop = self.other_class.properties[property_name]
else:
prop = SGProperty(property_name)
prop.in_class = self.other_class;
prop.is_static = self.is_static;
prop.in_file = self.in_file
#add property to class
self.other_class.add_member(prop)
#setup name of method and its position in the property
if self.is_getter:
class_method.name = 'get' + property_name
class_method.is_getter = True
self.is_getter = False #transfer to other methods
prop.getter = class_method
elif self.is_setter:
class_method.name = 'set' + property_name
prop.setter = class_method
class_method.is_setter = True
self.is_setter = False #transfer to other methods
#class_method.params[0].name = 'value'
else:
logger.error('Property is not a getter or a setter: %s - %s', self.name, property_name)
assert False
#change uname as well...
class_method.uname = class_method.name
class_method.in_property = prop
self.in_property = None
def process_meta_comments(self):
logger.debug(
'Parser : Starting to process meta comments: clearing old comments and attributes'
)
#self._meta_comments = []
#self._attributes = {}
tok = self._lookahead(1)[0] #_next_token()
attrs_started = False
while tok[0] in ['meta comment', 'attribute', 'comment']:
if tok[0] == 'attribute':
attrs_started = True
if attrs_started and tok[0] == 'meta comment':
tok = self._next_token() #actually read token
if len(tok[1]) > 0:
logger.error(
'Parser Error %s: Found additional meta comment after start of attributes',
self._tokeniser.line_details())
assert False
else:
tok = self._next_token() #actually read token
self._processors[tok[0]](tok)
tok = self._lookahead(1)[0]
def parse(self, a_file):
#clear all existing data
self._lookahead_toks = []
self._meta_comments = []
self._attributes = {}
self._ordered_attributes = []
self._current_file = a_file
self._tokeniser.tokenise(a_file.filename)
#read the meta comments before the node
self.process_meta_comments()
#read the token after meta-comments
tok = self._next_token()
if tok[0] == 'id' and tok[1] in ['unit', 'library']:
try:
self._file_processors[tok[1]](tok)
except:
logger.error("Parse Error on file %s" % a_file.filename)
raise
else:
logger.error('Parse Error %s: found %s expected unit or library',
self._tokeniser.line_details(), tok[1])
assert False
def _check_args_match_params(self):
'''
Ensure that the arguments in the call match the available parameters,
if no arguments are provided copy across read in parameters
'''
method_called = self.method_called #self.tags['calls'].other[0] #get called method
args = self.args #self.tags['calls'].other[1] #the arguments self passes to the method
logger.debug(
'Method : Checking arguments used by %s calling %s (%s)', self,
method_called, args)
if args == None:
logger.error(
'Method : No arguments that map to parameters for %s', self)
assert False
else:
for arg in args:
if isinstance(
arg, SGParameter) and not self.has_parameter(arg.name):
logger.error(
"Cannot match parameter %s in call to %s from %s",
str(arg), str(method_called), self)
assert False
def parse(self, a_file):
#clear all existing data
self._lookahead_toks = []
self._meta_comments = []
self._attributes = {}
self._ordered_attributes = []
self._current_file = a_file
self._tokeniser.tokenise(a_file.filename)
#read the meta comments before the node
self.process_meta_comments()
#read the token after meta-comments
tok = self._next_token()
if tok[0] == 'id' and tok[1] in ['unit','library']:
try:
self._file_processors[tok[1]](tok)
except:
logger.error("Parse Error on file %s" % a_file.filename)
raise
else:
logger.error('Parse Error %s: found %s expected unit or library',
self._tokeniser.line_details(), tok[1])
assert False
def check_arguments(self):
'''
Ensure that the arguments in the call match the available parameters,
if no arguments are provided copy across read in parameters
'''
if self.method_called == None:
logger.error('Method : Method %s does not call anything. Check attributes. %s', self.uname, self.file_line_details)
assert False
method_called = self.method_called #get called method
args = self.args #the arguments self passes to the method
logger.debug('Method : Checking arguments used by %s calling %s (%s)', self, method_called, args)
if len(args) != len(method_called.params):
logger.error('Method : Error in %s calling %s', self.uname, method_called.uname)
assert False
for arg in args:
if isinstance(arg, SGParameter):
if not self.has_parameter(arg.name):
logger.error("Cannot match parameter %s in call to %s from %s", str(arg), str(method_called), self)
assert False
def set_tag(self, title, other=None):
if title == "params":
#process parameter comments
for param_details in other:
param_name = param_details[0]
param_doc = param_details[1]
done = False
for param in self.params:
if param.name == param_name:
param.add_doc(param_doc)
done = True
break
if not done:
logger.error(
'Method : Unable to find parameter %s for %s',
param_name, self.uname)
assert False
elif title == 'related_params':
# The parameters have related documentation
# The passed in details has the param name, and the list of its related
# parameters
for related_param_details in other:
param_name = related_param_details[0]
related_params = related_param_details[1]
# Find the parameter
done = False
for param in self.params:
# if this is the parameter...
if param.name == param_name:
# then add related docs
param.add_related_params(related_params)
done = True
break
if not done:
logger.error(
'Method : Unable to find parameter %s for %s',
param_name, self.uname)
assert False
elif title == 'updatesArrayParams':
# check which parameters are being updated and mark so that checking ignores them
for idx in other:
self.params[idx - 1].being_updated = True
elif title == "class":
#the class indicates that the @method is for this other class...
from sg_code_module import SGCodeModule
from sg_library import SGLibrary
if other == None:
super(SGMethod, self).set_tag(title, None)
return
elif isinstance(other, SGCodeModule):
other_class = other
elif isinstance(other, SGLibrary):
other_class = other
else:
other_class = find_or_add_class(other)
super(SGMethod, self).set_tag('other_class', other_class)
elif title == 'getter' or title == 'setter':
# 1: mark as getter/setter
super(SGMethod, self).set_tag('is_' + title, True)
# 2: set property name
self.in_property = other
# 3: mark for later processing
mthd = SGMethod(other + ' ' + title)
super(SGMethod, self).set_tag('class_method', mthd)
elif title == 'constructor':
const = SGMethod(self.other_class.name)
const.is_constructor = True
super(SGMethod, self).set_tag('class_method', const)
elif title == 'dispose':
dest = SGMethod("~" + self.other_class.name)
dest.is_destructor = True
self.mimic_destructor = True
super(SGMethod, self).set_tag('class_method', dest)
elif title == 'csn':
#assign the 'csn' class special name to the class method
if self.class_method == None:
logger.error(
'Model Error: Method %s has a csn before the method/property/constructor definition - or should be sn.',
self.name)
assert False
self.class_method.set_tag('sn', other)
else:
super(SGMethod, self).set_tag(title, other)
def _parse_type_declaration(self, the_type):
'''
Parse a type from the next token, add details to the_type.
this is called for each type declaration... type_name = BLAH
Need to process BLAH and add details to the_type
'''
#check what kind of type it is...
if self._match_lookahead('symbol', '(', True):
values = []
while not self._match_lookahead('symbol', ')'):
temp = self._match_token('id')[1]
if self._match_lookahead(
'operator', '=', True) or self._match_lookahead(
'operator', ':=', True): #assigned value
values.append('%s = %s' %
(temp, self._match_token('number')[1]))
else:
values.append(temp)
self._match_lookahead('symbol', ',', True) #consume commas
self._match_token('symbol', ')') #read close bracket
the_type.values = tuple(values)
elif (self._match_lookahead('id', 'packed', True)
and self._match_lookahead('id', 'record',
True)) or self._match_lookahead(
'id', 'record', True):
#packed record field: Type end;
while not self._match_lookahead('id', 'end', True):
#read field
variables = self._read_variable_list()
self._match_token('symbol', ';')
for name, data_type in variables:
field = SGField(name)
field.data_type = data_type
the_type.fields.append(field)
elif self._match_lookahead('id', 'array') or self._match_lookahead(
'symbol', '^'):
#is pointer or array
the_type.clone(self._read_type_usage())
elif self._match_lookahead('id', 'procedure', True):
#procedure type: read ( params );
self._match_token('symbol', '(')
m = SGMethod(the_type.name)
m.in_file = self._current_file
self._read_params(m)
the_type.is_procedure = True
the_type.method = m
if self._lookahead(2)[1][1] == 'cdecl':
self._match_token('symbol', ';')
self._match_token('id', 'cdecl')
elif self._match_lookahead('id'):
other_id = self._match_token('id')[1]
other_type = find_or_add_type(other_id)
the_type.related_type = other_type
else:
tok = self._next_token()
logger.error('Parser Error %s: unknown type %s(%s)',
self._tokeniser.line_details(), tok[0], tok[1])
assert False
if the_type.related_type != None:
logger.info(' Parser : Setup type %s = %s', the_type,
the_type.related_type)
elif the_type.is_enum:
logger.info(' Parser : Setup type %s = %s', the_type,
the_type.values)
else:
logger.info(' Parser : Setup type %s = %s', the_type,
the_type.fields)
self._apply_attributes_to(the_type)
self._match_token('symbol', ';')
def complete_method_processing(self):
'''
This is called on methods that are read by the parser from the
Pascal file.
Set up the call from the library to this method if marked.
Check the call's validity
Steps:
1: Get other methods related to this one
2: Set parameters on library method (if called)
'''
logger.info(' Method : Completing processing of %s', self)
#This is 'the' method it has its params
self.params = self.params
#Find the length method if it exists
if self.length_call != None:
self.length_call = self.in_class.find_method(self.length_call)
#Convert args to appropriate values...
self._process_args()
self._check_args_match_params()
#Get other methods
lib_method = self.method_called
class_method = self.class_method
#check rules
if lib_method == None and not self.is_operator:
logger.error('Method : Found method %s without lib', self)
assert False
if lib_method != None:
#set up library method
self.setup_lib_method(lib_method)
logger.info(' Method : %s calls %s', self.name, lib_method.name)
lib_method.called_by.append(self)
#set up class method
if class_method != None:
logger.debug(' Method : %s is also %s', self.name, class_method)
self._setup_class_method(class_method)
logger.info(' Method : %s calls %s', class_method.name,
lib_method.name)
lib_method.called_by.append(
class_method) #library is also called by class
elif self.is_operator:
assert self.other_class != None
self.name = 'operator ' + self.name
self.other_method = self.in_class.find_method(self['calls'].other)
self.doc = self.other_method.doc
self.method_called = self.other_method.method_called
self.method_called.called_by.append(
self) #library is also called by operator
self.is_static = True
self.in_class.operators[self.signature] = None
self.in_class = self.other_class
self.in_class.add_member(self)
self.args = list(self.params) # operators must match directly
def next_token(self):
'''Find and return a tuple with the next token details. Tuple contains
the token (type, value) as strings. The token types and details are:
number, # such as 1234, 123.45, -123, +123.4
comment, # single // or multi-line (* ... *), { ... }
meta comment, # start with /// ...
id, # identifier name starting with alpha, including
# alpha-numeric characters and the _ character
attribute, # name starting with @... inside meta comment block
# follows the id name character rules
operator, # one of + - / * ** := < > <= >= <>
symbol, # one of ':;,.()'
'''
def num_match(cha, tmp):
'''Checks for a number in format ##, ##.#. Returns False when at the
end of a number.'''
if cha in '1234567890':
return True
elif cha == '.' and '.' not in tmp:
return self._peek(1) in '1234567890'
else:
return False
while (True):
t = self._next_char()
self._token_start = self._char_no
# Ignore white space characters
if t == ' ' or t == '\t': #ignore white space
pass
# Move to next line (if at end of line)
elif t == '\n':
self._advance_line()
# Numbers (int or float style format
elif t in '1234567890' or (t in '-+' and self._peek(1)
in '1234567890'): #is digit or +/-
result = ('number', self._read_matching(t, num_match))
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
# Comment, single line // or meta comment line ///
elif t == '/' and self._peek(1) == '/': #start of comment
if self._match_and_read('/'):
if self._match_and_read('/'):
kind = 'meta comment'
self._meta_comment_start = self._char_no
self._meta_comment_line = self._line_no
comment = self.read_to_end_of_comment()
else:
kind = 'comment'
comment = self.read_to_eol()
result = (kind, comment)
else:
result = ('error', t)
logger.debug('Tokenisier: read %s', result[0])
return result
# Attribute identified by an @ symbol then a name
elif t == '@':
name = self._read_matching(
'', lambda cha, tmp: cha.isalnum() or cha == '_')
result = ('attribute', name)
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
# Identifier (id) of alphanumeric characters including
elif t.isalpha():
name = self._read_matching(
t, lambda cha, tmp: cha.isalnum() or cha == '_')
if name.lower() in ['true', 'false']:
result = ('boolean', name)
else:
result = ('id', name)
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
#Bound Comment
elif t == '{' or (t == '(' and self._peek(1) == '*'):
if t == '(' and self._match_and_read('*'):
comment = self._read_until('',
lambda temp: temp[-2:] == '*)')
result = ('comment', comment[:-2])
elif t == '{':
comment = self._read_until('',
lambda temp: temp[-1:] == '}')
result = ('comment', comment[:-1])
logger.log(logging.DEBUG, 'Tokenisier: read %s', result[0])
return result
# Operator
elif (t == ':' and self._peek(1) == '=') or t in '=+-*/><':
if t == ':' and self._match_and_read('='):
result = ('operator', ':=')
elif t == '*' and self._match_and_read('*'):
result = ('operator', '**')
elif t == '<' and self._match_and_read('>'):
result = ('operator', '<>')
elif t in '<>' and self._match_and_read('='):
result = ('operator', t + '=')
else:
result = ('operator', t)
return result
# Symbol
elif t in '(),:;[].^':
result = ('symbol', t)
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
# Catch any single quotes inside a string value.
elif t == "'":
string = self._read_until(
'', lambda temp:
(temp[-1:] == "'") and (not self._match_and_read("'")))
result = ('string', string[:-1])
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
# Hmm.. unknown token. What did we forget?
else:
logger.error("Unknown token type: " + t)
return ('error', t)
def set_tag(self, title, other = None):
if title == "params":
#process parameter comments
for param_details in other:
param_name = param_details[0]
param_doc = param_details[1]
done = False
for param in self.params:
if param.name == param_name:
param.add_doc(param_doc)
done = True
break
if not done:
logger.error('Method : Unable to find parameter %s for %s', param_name, self.uname)
assert False
elif title == 'related_params':
# The parameters have related documentation
# The passed in details has the param name, and the list of its related
# parameters
for related_param_details in other:
param_name = related_param_details[0]
related_params = related_param_details[1]
# Find the parameter
done = False
for param in self.params:
# if this is the parameter...
if param.name == param_name:
# then add related docs
param.add_related_params(related_params)
done = True
break
if not done:
logger.error('Method : Unable to find parameter %s for %s', param_name, self.uname)
assert False
elif title == 'updatesArrayParams':
# check which parameters are being updated and mark so that checking ignores them
for idx in other:
self.params[idx - 1].being_updated = True
elif title == "class":
#the class indicates that the @method is for this other class...
from sg_code_module import SGCodeModule
from sg_library import SGLibrary
if other == None:
super(SGMethod,self).set_tag(title, None)
return
elif isinstance(other, SGCodeModule):
other_class = other
elif isinstance(other, SGLibrary):
other_class = other
else:
other_class = find_or_add_class(other)
super(SGMethod,self).set_tag('other_class', other_class)
elif title == 'getter' or title == 'setter':
# 1: mark as getter/setter
super(SGMethod,self).set_tag('is_' + title, True)
# 2: set property name
self.in_property = other
# 3: mark for later processing
mthd = SGMethod(other + ' ' + title)
super(SGMethod,self).set_tag('class_method', mthd)
elif title == 'constructor':
const = SGMethod(self.other_class.name)
const.is_constructor = True
super(SGMethod,self).set_tag('class_method', const)
elif title == 'dispose':
dest = SGMethod("~" + self.other_class.name)
dest.is_destructor = True
self.mimic_destructor = True
super(SGMethod,self).set_tag('class_method', dest)
elif title == 'csn':
#assign the 'csn' class special name to the class method
if self.class_method == None:
logger.error('Model Error: Method %s has a csn before the method/property/constructor definition - or should be sn.', self.name)
assert False
self.class_method.set_tag('sn', other)
else:
super(SGMethod,self).set_tag(title, other)
def _check_meta_comments(self, count, desc):
if len(self._meta_comments) != count:
logger.error('Parser Error %s: expected %d but foung %d meta comments for %s',
self._tokeniser.line_details(), count, len(self._meta_comments), desc)
sys.exit(-1)
def next_token(self):
'''Find and return a tuple with the next token details. Tuple contains
the token (type, value) as strings. The token types and details are:
number, # such as 1234, 123.45, -123, +123.4
comment, # single // or multi-line (* ... *), { ... }
meta comment, # start with /// ...
id, # identifier name starting with alpha, including
# alpha-numeric characters and the _ character
attribute, # name starting with @... inside meta comment block
# follows the id name character rules
operator, # one of + - / * ** := < > <= >= <>
symbol, # one of ':;,.()'
'''
def num_match(cha, tmp):
'''Checks for a number in format ##, ##.#. Returns False when at the
end of a number.'''
if cha in '1234567890':
return True
elif cha == '.' and '.' not in tmp:
return self._peek(1) in '1234567890'
else:
return False
while (True):
t = self._next_char();
self._token_start = self._char_no
# Ignore white space characters
if t == ' ' or t == '\t': #ignore white space
pass
# Move to next line (if at end of line)
elif t == '\n':
self._advance_line()
# Numbers (int or float style format
elif t in '1234567890' or (t in '-+' and self._peek(1) in '1234567890'): #is digit or +/-
result = ('number', self._read_matching(t, num_match))
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
# Comment, single line // or meta comment line ///
elif t == '/' and self._peek(1) == '/': #start of comment
if self._match_and_read('/'):
if self._match_and_read('/'):
kind = 'meta comment'
self._meta_comment_start = self._char_no
self._meta_comment_line = self._line_no
comment = self.read_to_end_of_comment()
else:
kind = 'comment'
comment = self.read_to_eol()
result = (kind, comment)
else:
result = ('error', t)
logger.debug('Tokenisier: read %s', result[0])
return result
# Attribute identified by an @ symbol then a name
elif t == '@':
name = self._read_matching('', lambda cha, tmp: cha.isalnum() or cha == '_')
result = ('attribute', name)
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
# Identifier (id) of alphanumeric characters including
elif t.isalpha():
name = self._read_matching(t, lambda cha, tmp: cha.isalnum() or cha == '_')
if name.lower() in ['true','false']:
result = ('boolean', name)
else:
result = ('id', name)
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
#Bound Comment
elif t == '{' or (t == '(' and self._peek(1) == '*'):
if t == '(' and self._match_and_read('*'):
comment = self._read_until('', lambda temp: temp[-2:] == '*)')
result = ('comment', comment[:-2])
elif t == '{':
comment = self._read_until('', lambda temp: temp[-1:] == '}')
result = ('comment', comment[:-1])
logger.log(logging.DEBUG, 'Tokenisier: read %s', result[0])
return result
# Operator
elif (t == ':' and self._peek(1) == '=') or t in '=+-*/><':
if t == ':' and self._match_and_read('='):
result = ('operator', ':=')
elif t == '*' and self._match_and_read('*'):
result = ('operator', '**')
elif t == '<' and self._match_and_read('>'):
result = ('operator', '<>')
elif t in '<>' and self._match_and_read('='):
result = ('operator', t + '=')
else:
result = ('operator', t)
return result
# Symbol
elif t in '(),:;[].^':
result = ('symbol', t)
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
# Catch any single quotes inside a string value.
elif t == "'":
string = self._read_until('', lambda temp: (temp[-1:] == "'") and (not self._match_and_read("'")))
result = ('string', string[:-1])
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
# Hmm.. unknown token. What did we forget?
else:
logger.error("Unknown token type: "+t)
return ('error', t)
