def _read_params(self, method):
"""Read in the parameters declared in a function, procedure or operator"""
#look for parameters
while not self._match_lookahead('symbol', ')'): #consume ) at end
param_tok, other_tok = self._lookahead(2)
if param_tok[0] == 'id':
if other_tok[0] == 'id': #first one is a modifier
modifier = self._next_token()[1]
else:
modifier = None
param_toks = [self._match_token('id')]
while self._match_lookahead('symbol', ',', True):
#there is a list of parameters
param_toks.append(self._match_token('id'))
colon_tok = self._match_token('symbol', ':')
the_type = self._read_type_usage()
for param_tok in param_toks:
param = method.create_parameter(param_tok[1])
param.data_type = the_type
param.modifier = modifier
logger.debug('Parser : Adding parameter %s (%s) to %s', param.name, param.data_type, method.name)
if not self._match_lookahead('symbol', ';', True): break
else:
logger.error('Parser : Error in parameter list %s', self._tokeniser.line_details())
self._match_token('symbol', ')')
def _append_attribute(self, attr, val):
logger.debug('Parser : Appending attribute %s with value %s to %s',attr,val, attr + 's')
if (attr + 's') in self._attributes.keys():
self._attributes[attr + 's'].append(val)
else:
self._attributes[attr + 's'] = [val]
self._ordered_attributes.append([attr + 's', self._attributes[attr + 's']])
def process_unit(self, token):
tok = self._match_token('id')
unit_name = tok[1]
name = self._get_attribute('module', unit_name)
self._add_attribute('name',name)
logger.info('-' * 70)
logger.info(' Parser : Processing unit: %s', unit_name)
logger.info('-' * 70)
#self._check_attributes(['class','static'],'unit ' + unit_name)
self._check_meta_comments(1, 'unit ' + unit_name + '\'s declaration')
#Create the unit
unit = self._create_model_element(SGCodeModule)
unit.module_kind = 'module'
unit.is_static = True
logger.info(' Parser : Creating class %s', unit.name)
self._apply_attributes_to(unit)
#unit name;
tok = self._match_token('symbol', ';')
#interface - ignore comments
#self._skip_comments('unit ' + unit_name + '\'s interface')
self._match_token('id', 'interface')
#check 'uses'
tok = self._lookahead()[0]
if tok[0] == 'id' and tok[1] == 'uses':
tok = self._next_token()
self.process_uses_clause(tok)
logger.info('-' * 70)
while True:
logger.debug('Parser : At start of loop to process elements in unit')
#read comments if they follow this position, otherwise leave
# attributes as they are
if self._match_lookahead('meta comment'): self.process_meta_comments()
#Process the body of the unit's interface
tok = self._lookahead()[0] #check the next token
if tok[0] == 'id' and tok[1] == 'implementation':
logger.info(' Parser : Found end of unit\'s interface')
logger.info('-' * 70)
break
#interface contains types, functions, procedures, var, const, operator
tok = self._match_one_token([['id','type'],
['id','function'],
['id','procedure'],
['id','const'],
['id','operator'],
['id','var']])
self._block_header_processors[tok[1]](unit, tok);
logger.info(' Parser : Finished processing unit. Resulting class is:\n%s', str(unit))
def process_block_types(self, block, token):
'''Reads the types within a type declaration'''
logger.info(' Parser : Processing types')
logger.info('-' * 70)
self.process_meta_comments()
#following type... in pascal
while True:
type_name = self._match_token('id')[1] #read the types name
self._match_token('operator','=') #read the =
the_type = find_or_add_type(type_name)
the_type.file_line_details = self._tokeniser.line_details()
the_type.meta_comment_line_details = self._tokeniser.meta_comment_line_details()
self._parse_type_declaration(the_type)
if 'type' or 'class' or 'struct'in the_type.keys():
self._add_class(the_type)
self.process_meta_comments()
tok1, tok2 = self._lookahead(2)
#looking for... type_name =
if tok2[0] != 'operator' or tok2[1] != '=' or tok1[0] != 'id':
logger.info('-' * 70)
logger.debug('Parser : At end of block types')
break
def process_operator_decl(self, block, token):
'''process a operator read from a unit.'''
op_kind = self._match_token('operator')
open_tok = self._match_token('symbol', '(')
#add to the class (e.g. Core module: which creates lib + other)
self._add_attribute('name', op_kind[1]) #name comes from operator
method = self._create_model_element(SGMethod)
method.in_class = block
method.is_operator = True
self._read_params(method)
result_tok = self._match_token('id')
colon_tok = self._match_token('symbol', ':')
the_type = self._read_type_usage()
method.return_type = the_type
logger.debug('Parser : Set return type of operator %s to %s',
method.name, method.return_type)
end_tok = self._match_token('symbol', ';')
self._apply_attributes_to(method)
method.complete_method_processing()
block.add_member(method)
def check_arguments(self):
'''
Ensure that the arguments in the call match the available parameters,
if no arguments are provided copy across read in parameters
'''
if self.method_called == None:
logger.error(
'Method : Method %s does not call anything. Check attributes. %s',
self.uname, self.file_line_details)
assert False
method_called = self.method_called #get called method
args = self.args #the arguments self passes to the method
logger.debug(
'Method : Checking arguments used by %s calling %s (%s)', self,
method_called, args)
if len(args) != len(method_called.params):
logger.error('Method : Error in %s calling %s', self.uname,
method_called.uname)
assert False
for arg in args:
if isinstance(arg, SGParameter):
if not self.has_parameter(arg.name):
logger.error(
"Cannot match parameter %s in call to %s from %s",
str(arg), str(method_called), self)
assert False
def process_unit(self, token):
tok = self._match_token('id')
unit_name = tok[1]
name = self._get_attribute('module', unit_name)
self._add_attribute('name', name)
logger.info('-' * 70)
logger.info(' Parser : Processing unit: %s', unit_name)
logger.info('-' * 70)
#self._check_attributes(['class','static'],'unit ' + unit_name)
self._check_meta_comments(1, 'unit ' + unit_name + '\'s declaration')
#Create the unit
unit = self._create_model_element(SGCodeModule)
unit.module_kind = 'module'
unit.is_static = True
logger.info(' Parser : Creating class %s', unit.name)
self._apply_attributes_to(unit)
#unit name;
tok = self._match_token('symbol', ';')
#interface - ignore comments
#self._skip_comments('unit ' + unit_name + '\'s interface')
self._match_token('id', 'interface')
#check 'uses'
tok = self._lookahead()[0]
if tok[0] == 'id' and tok[1] == 'uses':
tok = self._next_token()
self.process_uses_clause(tok)
logger.info('-' * 70)
while True:
logger.debug(
'Parser : At start of loop to process elements in unit')
#read comments if they follow this position, otherwise leave
# attributes as they are
if self._match_lookahead('meta comment'):
self.process_meta_comments()
#Process the body of the unit's interface
tok = self._lookahead()[0] #check the next token
if tok[0] == 'id' and tok[1] == 'implementation':
logger.info(' Parser : Found end of unit\'s interface')
logger.info('-' * 70)
break
#interface contains types, functions, procedures, var, const, operator
tok = self._match_one_token([['id', 'type'], ['id', 'function'],
['id', 'procedure'], ['id', 'const'],
['id', 'operator'], ['id', 'var']])
self._block_header_processors[tok[1]](unit, tok)
logger.info(
' Parser : Finished processing unit. Resulting class is:\n%s',
str(unit))
def _read_params(self, method):
"""Read in the parameters declared in a function, procedure or operator"""
#look for parameters
while not self._match_lookahead('symbol', ')'): #consume ) at end
param_tok, other_tok = self._lookahead(2)
if param_tok[0] == 'id':
if other_tok[0] == 'id': #first one is a modifier
modifier = self._next_token()[1]
else:
modifier = None
param_toks = [self._match_token('id')]
while self._match_lookahead('symbol', ',', True):
#there is a list of parameters
param_toks.append(self._match_token('id'))
colon_tok = self._match_token('symbol', ':')
the_type = self._read_type_usage()
for param_tok in param_toks:
param = method.create_parameter(param_tok[1])
param.data_type = the_type
param.modifier = modifier
logger.debug('Parser : Adding parameter %s (%s) to %s',
param.name, param.data_type, method.name)
if not self._match_lookahead('symbol', ';', True): break
else:
logger.error('Parser : Error in parameter list %s',
self._tokeniser.line_details())
self._match_token('symbol', ')')
def process_block_types(self, block, token):
'''Reads the types within a type declaration'''
logger.info(' Parser : Processing types')
logger.info('-' * 70)
self.process_meta_comments()
#following type... in pascal
while True:
type_name = self._match_token('id')[1] #read the types name
self._match_token('operator', '=') #read the =
the_type = find_or_add_type(type_name)
the_type.file_line_details = self._tokeniser.line_details()
the_type.meta_comment_line_details = self._tokeniser.meta_comment_line_details(
)
self._parse_type_declaration(the_type)
if 'type' or 'class' or 'struct' in the_type.keys():
self._add_class(the_type)
self.process_meta_comments()
tok1, tok2 = self._lookahead(2)
#looking for... type_name =
if tok2[0] != 'operator' or tok2[1] != '=' or tok1[0] != 'id':
logger.info('-' * 70)
logger.debug('Parser : At end of block types')
break
def complete_method_processing(self):
'''
This is called on methods that are read by the parser from the
Pascal file.
Set up the call from the library to this method if marked.
Check the call's validity
Steps:
1: Get other methods related to this one
2: Set parameters on library method (if called)
'''
logger.info(' Method : Completing processing of %s', self)
#This is 'the' method it has its params
self.params = self.params
#Find the length method if it exists
if self.length_call != None:
self.length_call = self.in_class.find_method(self.length_call)
#Convert args to appropriate values...
self._process_args()
self._check_args_match_params()
#Get other methods
lib_method = self.method_called
class_method = self.class_method
#check rules
if lib_method == None and not self.is_operator:
logger.error('Method : Found method %s without lib', self)
assert False
if lib_method != None:
#set up library method
self.setup_lib_method(lib_method)
logger.info(' Method : %s calls %s', self.name, lib_method.name)
lib_method.called_by.append(self)
#set up class method
if class_method != None:
logger.debug(' Method : %s is also %s', self.name, class_method)
self._setup_class_method(class_method)
logger.info(' Method : %s calls %s', class_method.name, lib_method.name)
lib_method.called_by.append(class_method) #library is also called by class
elif self.is_operator:
assert self.other_class != None
self.name = 'operator ' + self.name
self.other_method = self.in_class.find_method(self['calls'].other)
self.doc = self.other_method.doc
self.method_called = self.other_method.method_called
self.method_called.called_by.append(self) #library is also called by operator
self.is_static = True
self.in_class.operators[self.signature] = None
self.in_class = self.other_class
self.in_class.add_member(self)
self.args = list(self.params) # operators must match directly
def process_operator_decl(self, block, token):
'''process a operator read from a unit.'''
op_kind = self._match_token('operator')
open_tok = self._match_token('symbol', '(')
#add to the class (e.g. Core module: which creates lib + other)
self._add_attribute('name', op_kind[1]) #name comes from operator
method = self._create_model_element(SGMethod)
method.in_class = block
method.is_operator = True
self._read_params(method)
result_tok = self._match_token('id')
colon_tok = self._match_token('symbol', ':')
the_type = self._read_type_usage()
method.return_type = the_type
logger.debug('Parser : Set return type of operator %s to %s', method.name, method.return_type)
end_tok = self._match_token('symbol', ';')
self._apply_attributes_to(method)
method.complete_method_processing()
block.add_member(method)
def _add_attribute(self, attr, val):
logger.debug('Parser : Adding attribute %s with value %s', attr,
val)
if attr in self._attributes and self._attributes[attr] != None:
logger.warning('Parser : Added attribute twice %s = %s', attr,
val)
self._attributes[attr] = val
self._ordered_attributes.append([attr, val])
def _lookahead(self, count=1):
logger.debug('Parser : Looking ahead %d', count)
while len(self._lookahead_toks) < count:
current_token = self._tokeniser.next_token()
while current_token[0] == 'comment':
current_token = self._tokeniser.next_token()
self._lookahead_toks.append(current_token)
return self._lookahead_toks
def _lookahead(self,count=1):
logger.debug('Parser : Looking ahead %d', count)
while len(self._lookahead_toks) < count:
current_token = self._tokeniser.next_token()
while current_token[0] == 'comment':
current_token = self._tokeniser.next_token()
self._lookahead_toks.append(current_token)
return self._lookahead_toks
def _append_attribute(self, attr, val):
logger.debug('Parser : Appending attribute %s with value %s to %s',
attr, val, attr + 's')
if (attr + 's') in self._attributes.keys():
self._attributes[attr + 's'].append(val)
else:
self._attributes[attr + 's'] = [val]
self._ordered_attributes.append(
[attr + 's', self._attributes[attr + 's']])
def _match_lookahead(self, token_kind, token_value = None, consume = False):
logger.debug('Parser : Looking to find %s (%s)%s', token_kind,
token_value if token_value != None else 'any',
' will consume' if consume else '')
token = self._lookahead(1)[0]
result = token[0] == token_kind and (token_value == None or token_value == token[1].lower())
if consume and result:
self._match_token(token_kind, token_value)
return result
def _match_lookahead(self, token_kind, token_value=None, consume=False):
logger.debug('Parser : Looking to find %s (%s)%s', token_kind,
token_value if token_value != None else 'any',
' will consume' if consume else '')
token = self._lookahead(1)[0]
result = token[0] == token_kind and (token_value == None or token_value
== token[1].lower())
if consume and result:
self._match_token(token_kind, token_value)
return result
def _match_token(self, token_kind, token_value = None):
tok = self._next_token()
if tok[0] != token_kind or (token_value != None and token_value != tok[1].lower()):
logger.error('Parse Error %s: found a %s (%s) expected %s (%s)',
self._tokeniser.line_details(),
tok[0], tok[1], token_kind, token_value)
assert False
logger.debug('Parser : Matched token %s (%s)', tok[0], tok[1])
return tok
def _next_token(self):
current_token = None
while current_token == None or current_token[0] == 'comment':
if len(self._lookahead_toks) > 0:
current_token = self._lookahead_toks[0]
self._lookahead_toks = self._lookahead_toks[1:]
else:
current_token = self._tokeniser.next_token()
if current_token[0] == 'comment':
logger.debug('Parser : Skipping comment: %s', current_token[1])
return current_token
def _match_token(self, token_kind, token_value=None):
tok = self._next_token()
if tok[0] != token_kind or (token_value != None
and token_value != tok[1].lower()):
logger.error('Parse Error %s: found a %s (%s) expected %s (%s)',
self._tokeniser.line_details(), tok[0], tok[1],
token_kind, token_value)
assert False
logger.debug('Parser : Matched token %s (%s)', tok[0], tok[1])
return tok
def _next_token(self):
current_token = None
while current_token == None or current_token[0] == 'comment':
if len(self._lookahead_toks) > 0:
current_token = self._lookahead_toks[0]
self._lookahead_toks = self._lookahead_toks[1:]
else:
current_token = self._tokeniser.next_token()
if current_token[0] == 'comment':
logger.debug('Parser : Skipping comment: %s',
current_token[1])
return current_token
def _setup_class_method(self, class_method):
'''
Setup the class's method.
The class method is a clone of the current method, in
a class wrapper. This comes from @method or @overload
in the pascal source.
Steps:
1: clone self to class_method
2: add it to its class or property
3: alter args (add pointer field access)
'''
self.clone_to(class_method) #copy self into other
class_method.is_class_method = True
#if the class method is actually a property...
if self.is_getter or self.is_setter:
self.create_and_add_property(class_method)
else:
class_method.in_class.add_member(class_method) #add to its class
#static methods and constructors directly call the method
# instance methods change the first argument for the pointer field
if class_method.is_static or class_method.is_constructor:
#use self's args - i.e. it calls the method in the same way
args = list(self.args)
else: #other is an instance (with ptr)
if self.args == None or len(self.args) < self.self_pos:
logger.error(
'Class method calling a method without parameter for self pointer... '
+ self.name)
assert False
args = list(self.args)
#change the old argument for the self pointer
if class_method.in_class.is_pointer_wrapper:
args[self.self_pos - 1] = 'self.pointer'
elif class_method.in_class.wraps_array:
args[self.self_pos - 1] = 'self.data'
else:
args[self.self_pos - 1] = 'self'
#add in self's arguments (-1st which is pointer)
#set class method to call the same method this does
class_method.calls(self.method_called, args)
logger.debug('Method : Setting up call: %s calls %s with args %s',
class_method, class_method.method_called,
class_method.args)
class_method._check_args_match_params()
def _setup_class_method(self, class_method):
'''
Setup the class's method.
The class method is a clone of the current method, in
a class wrapper. This comes from @method or @overload
in the pascal source.
Steps:
1: clone self to class_method
2: add it to its class or property
3: alter args (add pointer field access)
'''
self.clone_to(class_method) #copy self into other
class_method.is_class_method = True
#if the class method is actually a property...
if self.is_getter or self.is_setter:
self.create_and_add_property(class_method)
else:
class_method.in_class.add_member(class_method) #add to its class
#static methods and constructors directly call the method
# instance methods change the first argument for the pointer field
if class_method.is_static or class_method.is_constructor:
#use self's args - i.e. it calls the method in the same way
args = list(self.args)
else: #other is an instance (with ptr)
if self.args == None or len(self.args) < self.self_pos:
logger.error('Class method calling a method without parameter for self pointer... ' + self.name)
assert False
args = list(self.args)
#change the old argument for the self pointer
if class_method.in_class.is_pointer_wrapper:
args[self.self_pos - 1] = 'self.pointer'
elif class_method.in_class.wraps_array:
args[self.self_pos - 1] = 'self.data'
else:
args[self.self_pos - 1] = 'self'
#add in self's arguments (-1st which is pointer)
#set class method to call the same method this does
class_method.calls(self.method_called, args)
logger.debug('Method : Setting up call: %s calls %s with args %s', class_method,
class_method.method_called, class_method.args)
class_method._check_args_match_params()
def method_process_visitor(the_method, other):
'''Process a method prior to rendering. This does all of the transformations
that go into the functions and procedures of the SGSDK unit.'''
# Change functions with string or array return type to have a result parameter
if the_method.return_type != None and (
the_method.return_type.wraps_array
or the_method.return_type.name.lower() in ['string']):
#print 'PARS RUN PARAMS', [p.name for p in the_method.params]
result_param = SGParameter('result')
for param in the_method.params:
if 'result' == param.name:
logger.error('PARSER RUN: Error adding result parameter to %s',
the_method.name)
assert False
_add_parameter(the_method, result_param)
result_param.maps_result = True
result_param.modifier = 'result'
result_param.data_type = the_method.return_type
the_method.return_type = None
the_method.was_function = True
#print 'PARS RUN PARAMS', [p.name for p in the_method.params]
#Replace any variable length arrays
orig_params = the_method.params
for param in orig_params:
#does it wrap a variable length array
if param.data_type.array_wrapper:
the_method.has_length_params = True
logger.debug(
'PARSER RUN: Altering variable length array of %s\'s parameter %s',
the_method.name, param.name)
# old_type = param.data_type
#param.data_type = param.data_type #.fields[0].data_type
len_param = SGParameter('%s_len' % param.name)
len_param.is_length_param = True
len_param.data_type = find_or_add_type('Longint')
len_param.modifier = param.modifier if param.modifier is [
'out', 'var'
] else None
len_param.length_of = param
param.has_length_param = True
param.length_idx = _add_parameter(the_method, len_param)
return other
def process_uses_clause(self, token):
'''Read the list of units referred to be the uses clause'''
self._check_non_commented('uses clause')
while True:
tok = self._match_token('id')
self._current_file.uses.append(find_or_add_file(tok[1]))
logger.debug('Parser : Found using unit %s', tok[1])
#found a token/unit
next_tok = self._match_token('symbol')
if next_tok[1] == ';':
break; #found end
elif next_tok[1] != ',':
logger.error('Parser Error %s: expected , or ; but found %s at %s',
self._tokeniser.line_details(), next_tok[1])
sys.exit(-1)
def _match_one_token(self, token_kind_lst):
matched = False
tok = self._next_token()
for token_kind,token_value in token_kind_lst:
if tok[0] == token_kind and (token_value == None or token_value == tok[1]):
matched = True
logger.debug('Parser : Matched %s with %s', tok[0], tok[1])
break
if not matched:
logger.error('Parser Error %s: unexpected %s(%s) expected %s',
self._tokeniser.line_details(),
tok[0], tok[1],
map(lambda n: '%s(%s)' % (n[0],n[1]),token_kind_lst))
assert False
return tok
def _match_one_token(self, token_kind_lst):
matched = False
tok = self._next_token()
for token_kind, token_value in token_kind_lst:
if tok[0] == token_kind and (token_value == None
or token_value == tok[1]):
matched = True
logger.debug('Parser : Matched %s with %s', tok[0], tok[1])
break
if not matched:
logger.error(
'Parser Error %s: unexpected %s(%s) expected %s',
self._tokeniser.line_details(), tok[0], tok[1],
map(lambda n: '%s(%s)' % (n[0], n[1]), token_kind_lst))
assert False
return tok
def _check_args_match_params(self):
'''
Ensure that the arguments in the call match the available parameters,
if no arguments are provided copy across read in parameters
'''
method_called = self.method_called #self.tags['calls'].other[0] #get called method
args = self.args #self.tags['calls'].other[1] #the arguments self passes to the method
logger.debug('Method : Checking arguments used by %s calling %s (%s)', self, method_called, args)
if args == None:
logger.error('Method : No arguments that map to parameters for %s', self)
assert False
else:
for arg in args:
if isinstance(arg, SGParameter) and not self.has_parameter(arg.name):
logger.error("Cannot match parameter %s in call to %s from %s", str(arg), str(method_called), self)
assert False
def tokenise(self, filename):
'''Initialises the tokeniser with characters loaded from the specified
filename. Call `next_token` process each token.
'''
if isinstance(filename, list):
logger.debug('Tokenising list')
self.pas_lines = filename
else:
logger.debug('Tokenising %s', filename)
self._filename = filename
# print filename
f = open(filename)
self.pas_lines = f.readlines()
f.close()
self._char_no = -1
self._line_no = 0 #starts at first line
self._token_val = 'none'
def _create_model_element(self, kind):
name = self._get_attribute('name')
if kind == SGCodeModule:
result = find_or_add_class(name)
if not result in self._current_file.members:
self._current_file.members.append(result)
result.file_line_details = self._tokeniser.line_details()
result.meta_comment_line_details = self._tokeniser.meta_comment_line_details()
elif kind == SGType:
assert False
#result = find_or_add_type(name)
else:
result = kind(name)
result.file_line_details = self._tokeniser.line_details()
result.meta_comment_line_details = self._tokeniser.meta_comment_line_details()
result.in_file = self._current_file
logger.debug('Parser : Creating model element: %s with kind:%s', name, kind)
return result
def process_uses_clause(self, token):
'''Read the list of units referred to be the uses clause'''
self._check_non_commented('uses clause')
while True:
tok = self._match_token('id')
self._current_file.uses.append(find_or_add_file(tok[1]))
logger.debug('Parser : Found using unit %s', tok[1])
#found a token/unit
next_tok = self._match_token('symbol')
if next_tok[1] == ';':
break
#found end
elif next_tok[1] != ',':
logger.error(
'Parser Error %s: expected , or ; but found %s at %s',
self._tokeniser.line_details(), next_tok[1])
sys.exit(-1)
def tokenise(self, filename):
'''Initialises the tokeniser with characters loaded from the specified
filename. Call `next_token` process each token.
'''
if isinstance(filename, list):
logger.debug('Tokenising list')
self.pas_lines = filename
else:
logger.debug('Tokenising %s', filename)
self._filename = filename
# print filename
f = open(filename)
self.pas_lines = f.readlines()
f.close()
self._char_no = -1
self._line_no = 0 #starts at first line
self._token_val = 'none'
def process_meta_comments(self):
logger.debug('Parser : Starting to process meta comments: clearing old comments and attributes')
#self._meta_comments = []
#self._attributes = {}
tok = self._lookahead(1)[0] #_next_token()
attrs_started = False
while tok[0] in ['meta comment','attribute','comment']:
if tok[0] == 'attribute':
attrs_started = True
if attrs_started and tok[0] == 'meta comment':
tok = self._next_token() #actually read token
if len(tok[1]) > 0:
logger.error('Parser Error %s: Found additional meta comment after start of attributes',
self._tokeniser.line_details())
assert False
else:
tok = self._next_token() #actually read token
self._processors[tok[0]](tok)
tok = self._lookahead(1)[0]
def method_process_visitor(the_method, other):
'''Process a method prior to rendering. This does all of the transformations
that go into the functions and procedures of the SGSDK unit.'''
# Change functions with string or array return type to have a result parameter
if the_method.return_type != None and (the_method.return_type.wraps_array or the_method.return_type.name.lower() in ['string']):
#print 'PARS RUN PARAMS', [p.name for p in the_method.params]
result_param = SGParameter('result')
for param in the_method.params:
if 'result' == param.name:
logger.error('PARSER RUN: Error adding result parameter to %s', the_method.name)
assert False
_add_parameter(the_method, result_param)
result_param.maps_result = True
result_param.modifier = 'result'
result_param.data_type = the_method.return_type
the_method.return_type = None
the_method.was_function = True
#print 'PARS RUN PARAMS', [p.name for p in the_method.params]
#Replace any variable length arrays
orig_params = the_method.params
for param in orig_params:
#does it wrap a variable length array
if param.data_type.array_wrapper:
the_method.has_length_params = True
logger.debug('PARSER RUN: Altering variable length array of %s\'s parameter %s', the_method.name, param.name)
# old_type = param.data_type
#param.data_type = param.data_type #.fields[0].data_type
len_param = SGParameter('%s_len' % param.name)
len_param.is_length_param = True
len_param.data_type = find_or_add_type('Longint')
len_param.modifier = param.modifier if param.modifier is ['out', 'var'] else None
len_param.length_of = param
param.has_length_param = True
param.length_idx = _add_parameter(the_method, len_param)
return other
def setup_lib_method(self, lib_method):
'''
Setup the library method
Set:
return type
parameters
calls
'''
if lib_method.file_line_details == None:
lib_method.file_line_details = []
lib_method.file_line_details.append(self.file_line_details)
if self.called_by_lib: #Set up the call from the library to this method
logger.debug('Method : Setting %s in library to call %s', lib_method, self)
lib_method.return_type = self.return_type #set return type
lib_method.params = [] #add parameters
for p in self.params:
lib_method.params.append(p.clone())
lib_method.calls(self, self.args) #..calls this method at other end
def process_method_decl(self, block, token):
'''process a method read from a unit.
block is the block that contains the method, token is the first token.
At the end of this the method has been read in and added to block.
'''
name_tok = self._match_token('id') #name of function/procedure
open_tok = self._match_token('symbol', '(')
#add to the class (e.g. Core module: which creates lib + other)
self._add_attribute('name',
name_tok[1]) #name comes from function/procedure
method = self._create_model_element(SGMethod)
method.in_class = block
self._read_params(method)
if token[1] == 'function': #return return details
colon_tok = self._match_token('symbol', ':')
the_type = self._read_type_usage()
method.return_type = the_type
logger.debug('Parser : Set return type of method %s to %s',
method.name, method.return_type)
end_tok = self._match_token('symbol', ';')
#check overload ;
if self._match_lookahead('id', 'overload', True):
self._match_token('symbol', ';')
self._add_attribute('overload', True)
#logger.info('Adding method %s.%s(%s)', block.name, method.name, method.param_string())
if self._get_attribute(
'called_by_lib',
False) and self._get_attribute('method_called') == None:
self._setup_lib_method(method.uname)
self._apply_attributes_to(method)
method.complete_method_processing()
block.add_member(method)
def _create_model_element(self, kind):
name = self._get_attribute('name')
if kind == SGCodeModule:
result = find_or_add_class(name)
if not result in self._current_file.members:
self._current_file.members.append(result)
result.file_line_details = self._tokeniser.line_details()
result.meta_comment_line_details = self._tokeniser.meta_comment_line_details(
)
elif kind == SGType:
assert False
#result = find_or_add_type(name)
else:
result = kind(name)
result.file_line_details = self._tokeniser.line_details()
result.meta_comment_line_details = self._tokeniser.meta_comment_line_details(
)
result.in_file = self._current_file
logger.debug('Parser : Creating model element: %s with kind:%s',
name, kind)
return result
def setup_lib_method(self, lib_method):
'''
Setup the library method
Set:
return type
parameters
calls
'''
if lib_method.file_line_details == None:
lib_method.file_line_details = []
lib_method.file_line_details.append(self.file_line_details)
if self.called_by_lib: #Set up the call from the library to this method
logger.debug('Method : Setting %s in library to call %s',
lib_method, self)
lib_method.return_type = self.return_type #set return type
lib_method.params = [] #add parameters
for p in self.params:
lib_method.params.append(p.clone())
lib_method.calls(self,
self.args) #..calls this method at other end
def check_arguments(self):
'''
Ensure that the arguments in the call match the available parameters,
if no arguments are provided copy across read in parameters
'''
if self.method_called == None:
logger.error('Method : Method %s does not call anything. Check attributes. %s', self.uname, self.file_line_details)
assert False
method_called = self.method_called #get called method
args = self.args #the arguments self passes to the method
logger.debug('Method : Checking arguments used by %s calling %s (%s)', self, method_called, args)
if len(args) != len(method_called.params):
logger.error('Method : Error in %s calling %s', self.uname, method_called.uname)
assert False
for arg in args:
if isinstance(arg, SGParameter):
if not self.has_parameter(arg.name):
logger.error("Cannot match parameter %s in call to %s from %s", str(arg), str(method_called), self)
assert False
def process_meta_comments(self):
logger.debug(
'Parser : Starting to process meta comments: clearing old comments and attributes'
)
#self._meta_comments = []
#self._attributes = {}
tok = self._lookahead(1)[0] #_next_token()
attrs_started = False
while tok[0] in ['meta comment', 'attribute', 'comment']:
if tok[0] == 'attribute':
attrs_started = True
if attrs_started and tok[0] == 'meta comment':
tok = self._next_token() #actually read token
if len(tok[1]) > 0:
logger.error(
'Parser Error %s: Found additional meta comment after start of attributes',
self._tokeniser.line_details())
assert False
else:
tok = self._next_token() #actually read token
self._processors[tok[0]](tok)
tok = self._lookahead(1)[0]
def process_method_decl(self, block, token):
'''process a method read from a unit.
block is the block that contains the method, token is the first token.
At the end of this the method has been read in and added to block.
'''
name_tok = self._match_token('id') #name of function/procedure
open_tok = self._match_token('symbol', '(')
#add to the class (e.g. Core module: which creates lib + other)
self._add_attribute('name', name_tok[1]) #name comes from function/procedure
method = self._create_model_element(SGMethod)
method.in_class = block
self._read_params(method)
if token[1] == 'function': #return return details
colon_tok = self._match_token('symbol', ':')
the_type = self._read_type_usage()
method.return_type = the_type
logger.debug('Parser : Set return type of method %s to %s', method.name, method.return_type)
end_tok = self._match_token('symbol', ';')
#check overload ;
if self._match_lookahead('id', 'overload', True):
self._match_token('symbol', ';')
self._add_attribute('overload', True)
#logger.info('Adding method %s.%s(%s)', block.name, method.name, method.param_string())
if self._get_attribute('called_by_lib', False) and self._get_attribute('method_called') == None:
self._setup_lib_method(method.uname)
self._apply_attributes_to(method)
method.complete_method_processing()
block.add_member(method)
def _check_args_match_params(self):
'''
Ensure that the arguments in the call match the available parameters,
if no arguments are provided copy across read in parameters
'''
method_called = self.method_called #self.tags['calls'].other[0] #get called method
args = self.args #self.tags['calls'].other[1] #the arguments self passes to the method
logger.debug(
'Method : Checking arguments used by %s calling %s (%s)', self,
method_called, args)
if args == None:
logger.error(
'Method : No arguments that map to parameters for %s', self)
assert False
else:
for arg in args:
if isinstance(
arg, SGParameter) and not self.has_parameter(arg.name):
logger.error(
"Cannot match parameter %s in call to %s from %s",
str(arg), str(method_called), self)
assert False
def process_attribute(self, token):
logger.debug('Parser : Processing attribute: %s', token[1])
self._attribute_processors[token[1]](token)
def complete_method_processing(self):
'''
This is called on methods that are read by the parser from the
Pascal file.
Set up the call from the library to this method if marked.
Check the call's validity
Steps:
1: Get other methods related to this one
2: Set parameters on library method (if called)
'''
logger.info(' Method : Completing processing of %s', self)
#This is 'the' method it has its params
self.params = self.params
#Find the length method if it exists
if self.length_call != None:
self.length_call = self.in_class.find_method(self.length_call)
#Convert args to appropriate values...
self._process_args()
self._check_args_match_params()
#Get other methods
lib_method = self.method_called
class_method = self.class_method
#check rules
if lib_method == None and not self.is_operator:
logger.error('Method : Found method %s without lib', self)
assert False
if lib_method != None:
#set up library method
self.setup_lib_method(lib_method)
logger.info(' Method : %s calls %s', self.name, lib_method.name)
lib_method.called_by.append(self)
#set up class method
if class_method != None:
logger.debug(' Method : %s is also %s', self.name, class_method)
self._setup_class_method(class_method)
logger.info(' Method : %s calls %s', class_method.name,
lib_method.name)
lib_method.called_by.append(
class_method) #library is also called by class
elif self.is_operator:
assert self.other_class != None
self.name = 'operator ' + self.name
self.other_method = self.in_class.find_method(self['calls'].other)
self.doc = self.other_method.doc
self.method_called = self.other_method.method_called
self.method_called.called_by.append(
self) #library is also called by operator
self.is_static = True
self.in_class.operators[self.signature] = None
self.in_class = self.other_class
self.in_class.add_member(self)
self.args = list(self.params) # operators must match directly
def process_attribute(self, token):
logger.debug('Parser : Processing attribute: %s', token[1])
self._attribute_processors[token[1]](token)
def find_method(self, uname, file_name):
for key, method in self.methods.items():
if method.uname == 'sg_%s_%s' % (file_name, uname):
logger.debug('Library : Found match for %s', method.uname)
return method
return None
def visit_all_units(file_visitor):
logger.info('Processing files')
files = [unit[0] for unit in all_units] + ['SGSDK']
for each_file in files:
logger.debug('Visiting file %s', each_file)
find_or_add_file(each_file).visit(file_visitor, None)
def next_token(self):
'''Find and return a tuple with the next token details. Tuple contains
the token (type, value) as strings. The token types and details are:
number, # such as 1234, 123.45, -123, +123.4
comment, # single // or multi-line (* ... *), { ... }
meta comment, # start with /// ...
id, # identifier name starting with alpha, including
# alpha-numeric characters and the _ character
attribute, # name starting with @... inside meta comment block
# follows the id name character rules
operator, # one of + - / * ** := < > <= >= <>
symbol, # one of ':;,.()'
'''
def num_match(cha, tmp):
'''Checks for a number in format ##, ##.#. Returns False when at the
end of a number.'''
if cha in '1234567890':
return True
elif cha == '.' and '.' not in tmp:
return self._peek(1) in '1234567890'
else:
return False
while (True):
t = self._next_char()
self._token_start = self._char_no
# Ignore white space characters
if t == ' ' or t == '\t': #ignore white space
pass
# Move to next line (if at end of line)
elif t == '\n':
self._advance_line()
# Numbers (int or float style format
elif t in '1234567890' or (t in '-+' and self._peek(1)
in '1234567890'): #is digit or +/-
result = ('number', self._read_matching(t, num_match))
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
# Comment, single line // or meta comment line ///
elif t == '/' and self._peek(1) == '/': #start of comment
if self._match_and_read('/'):
if self._match_and_read('/'):
kind = 'meta comment'
self._meta_comment_start = self._char_no
self._meta_comment_line = self._line_no
comment = self.read_to_end_of_comment()
else:
kind = 'comment'
comment = self.read_to_eol()
result = (kind, comment)
else:
result = ('error', t)
logger.debug('Tokenisier: read %s', result[0])
return result
# Attribute identified by an @ symbol then a name
elif t == '@':
name = self._read_matching(
'', lambda cha, tmp: cha.isalnum() or cha == '_')
result = ('attribute', name)
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
# Identifier (id) of alphanumeric characters including
elif t.isalpha():
name = self._read_matching(
t, lambda cha, tmp: cha.isalnum() or cha == '_')
if name.lower() in ['true', 'false']:
result = ('boolean', name)
else:
result = ('id', name)
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
#Bound Comment
elif t == '{' or (t == '(' and self._peek(1) == '*'):
if t == '(' and self._match_and_read('*'):
comment = self._read_until('',
lambda temp: temp[-2:] == '*)')
result = ('comment', comment[:-2])
elif t == '{':
comment = self._read_until('',
lambda temp: temp[-1:] == '}')
result = ('comment', comment[:-1])
logger.log(logging.DEBUG, 'Tokenisier: read %s', result[0])
return result
# Operator
elif (t == ':' and self._peek(1) == '=') or t in '=+-*/><':
if t == ':' and self._match_and_read('='):
result = ('operator', ':=')
elif t == '*' and self._match_and_read('*'):
result = ('operator', '**')
elif t == '<' and self._match_and_read('>'):
result = ('operator', '<>')
elif t in '<>' and self._match_and_read('='):
result = ('operator', t + '=')
else:
result = ('operator', t)
return result
# Symbol
elif t in '(),:;[].^':
result = ('symbol', t)
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
# Catch any single quotes inside a string value.
elif t == "'":
string = self._read_until(
'', lambda temp:
(temp[-1:] == "'") and (not self._match_and_read("'")))
result = ('string', string[:-1])
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
# Hmm.. unknown token. What did we forget?
else:
logger.error("Unknown token type: " + t)
return ('error', t)
def _add_attribute(self, attr, val):
logger.debug('Parser : Adding attribute %s with value %s',attr,val)
if attr in self._attributes and self._attributes[attr] != None:
logger.warning('Parser : Added attribute twice %s = %s', attr, val)
self._attributes[attr] = val
self._ordered_attributes.append([attr, val])
def next_token(self):
'''Find and return a tuple with the next token details. Tuple contains
the token (type, value) as strings. The token types and details are:
number, # such as 1234, 123.45, -123, +123.4
comment, # single // or multi-line (* ... *), { ... }
meta comment, # start with /// ...
id, # identifier name starting with alpha, including
# alpha-numeric characters and the _ character
attribute, # name starting with @... inside meta comment block
# follows the id name character rules
operator, # one of + - / * ** := < > <= >= <>
symbol, # one of ':;,.()'
'''
def num_match(cha, tmp):
'''Checks for a number in format ##, ##.#. Returns False when at the
end of a number.'''
if cha in '1234567890':
return True
elif cha == '.' and '.' not in tmp:
return self._peek(1) in '1234567890'
else:
return False
while (True):
t = self._next_char();
self._token_start = self._char_no
# Ignore white space characters
if t == ' ' or t == '\t': #ignore white space
pass
# Move to next line (if at end of line)
elif t == '\n':
self._advance_line()
# Numbers (int or float style format
elif t in '1234567890' or (t in '-+' and self._peek(1) in '1234567890'): #is digit or +/-
result = ('number', self._read_matching(t, num_match))
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
# Comment, single line // or meta comment line ///
elif t == '/' and self._peek(1) == '/': #start of comment
if self._match_and_read('/'):
if self._match_and_read('/'):
kind = 'meta comment'
self._meta_comment_start = self._char_no
self._meta_comment_line = self._line_no
comment = self.read_to_end_of_comment()
else:
kind = 'comment'
comment = self.read_to_eol()
result = (kind, comment)
else:
result = ('error', t)
logger.debug('Tokenisier: read %s', result[0])
return result
# Attribute identified by an @ symbol then a name
elif t == '@':
name = self._read_matching('', lambda cha, tmp: cha.isalnum() or cha == '_')
result = ('attribute', name)
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
# Identifier (id) of alphanumeric characters including
elif t.isalpha():
name = self._read_matching(t, lambda cha, tmp: cha.isalnum() or cha == '_')
if name.lower() in ['true','false']:
result = ('boolean', name)
else:
result = ('id', name)
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
#Bound Comment
elif t == '{' or (t == '(' and self._peek(1) == '*'):
if t == '(' and self._match_and_read('*'):
comment = self._read_until('', lambda temp: temp[-2:] == '*)')
result = ('comment', comment[:-2])
elif t == '{':
comment = self._read_until('', lambda temp: temp[-1:] == '}')
result = ('comment', comment[:-1])
logger.log(logging.DEBUG, 'Tokenisier: read %s', result[0])
return result
# Operator
elif (t == ':' and self._peek(1) == '=') or t in '=+-*/><':
if t == ':' and self._match_and_read('='):
result = ('operator', ':=')
elif t == '*' and self._match_and_read('*'):
result = ('operator', '**')
elif t == '<' and self._match_and_read('>'):
result = ('operator', '<>')
elif t in '<>' and self._match_and_read('='):
result = ('operator', t + '=')
else:
result = ('operator', t)
return result
# Symbol
elif t in '(),:;[].^':
result = ('symbol', t)
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
# Catch any single quotes inside a string value.
elif t == "'":
string = self._read_until('', lambda temp: (temp[-1:] == "'") and (not self._match_and_read("'")))
result = ('string', string[:-1])
logger.debug('Tokenisier: read %s - %s', result[0], result[1])
return result
# Hmm.. unknown token. What did we forget?
else:
logger.error("Unknown token type: "+t)
return ('error', t)
def find_method(self, uname, file_name):
for key, method in self.methods.items():
if method.uname == 'sg_%s_%s' % (file_name, uname):
logger.debug('Library : Found match for %s', method.uname)
return method
return None
def visit_all_units(file_visitor):
logger.info('Processing files')
files = [ unit[0] for unit in all_units ] + ['SGSDK']
for each_file in files:
logger.debug('Visiting file %s', each_file)
find_or_add_file(each_file).visit(file_visitor, None)
