def parse(self, tokens):
from pas_parser_utils import parse_statement
logger.debug("Parsing case statement")
tokens.match_token(TokenKind.Identifier, 'case')
self._expression = PascalExpression(self._block)
self._expression.parse(tokens)
# expression will consume the 'of' keyword
while not (tokens.match_lookahead(TokenKind.Identifier, 'end') or tokens.match_lookahead(TokenKind.Identifier, 'else') or tokens.match_lookahead(TokenKind.Identifier, 'otherwise')):
constant = None
if (tokens.match_lookahead(TokenKind.Number)):
constant = tokens.match_token(TokenKind.Number).value
elif (tokens.match_lookahead(TokenKind.String)):
constant = tokens.match_token(TokenKind.String.value)
if len(constant) > 1:
raise_error(("Constant expected: %s" %tokens.next_token()), '', is_critical=False)
elif tokens.match_lookahead(TokenKind.Identifier):
constant = self._block.resolve_variable(tokens.match_token(TokenKind.Identifier).value)
else:
raise_error(("CaseStatement: Ordinal type expected: %s", tokens.next_token()), '', is_critical=False)
tokens.match_token(TokenKind.Symbol, ':')
statement = parse_statement(tokens, self._block)
tokens.match_lookahead(TokenKind.Symbol, ';', consume=True)
self._case.append(tuple((constant, statement)))
if tokens.match_lookahead(TokenKind.Identifier, 'else', consume=True) or tokens.match_lookahead(TokenKind.Identifier, 'otherwise', consume=True):
while not (tokens.match_lookahead(TokenKind.Identifier, 'end', consume=True)):
self._else_statements.append(parse_statement(tokens, self._block))
else:
tokens.match_token(TokenKind.Identifier, 'end')
tokens.match_token(TokenKind.Symbol, ';')
logger.debug("Finished parsing if statement")
def parse(self, tokens, do_resolve=True):
"""
parses a function call
"""
from pas_arguments import PascalArguments
logger.debug('Processing function call %s', self._identifier)
self._identifier = tokens.match_token(TokenKind.Identifier).value
self._arguments = PascalArguments(self._block)
self._arguments.parse(tokens)
if not self._inExpr:
tokens.match_lookahead(TokenKind.Symbol, ';', consume=True)
if do_resolve:
self._points_to = self._block.resolve_function_call(self)
if self._points_to is self:
# recursive call...
# need to get the function I'm being called from...
self._points_to = self._block._functions[self.name]
elif self._points_to is None:
raise_error(
("Unable to resolve function call: " + self.identifier),
'',
is_critical=False)
logger.debug('Ended function call %s', self._identifier)
def parse(self, tokens):
from pas_parser_utils import parse_statement
logger.debug("Parsing while statement")
tokens.match_token(TokenKind.Identifier, 'while')
self._expression = PascalExpression(self._block)
self._expression.parse(tokens)
self._statement = parse_statement(tokens, self._block)
# expression will consume the 'then' delimiter
logger.debug("Finished parsing while statement")
def parse(self, tokens):
from pas_parser_utils import parse_statement
logger.debug("Parsing repeat statement")
tokens.match_token(TokenKind.Identifier, 'repeat')
while not tokens.match_lookahead(TokenKind.Identifier, 'until'):
self._statements.append(parse_statement(tokens, self._block))
tokens.match_token(TokenKind.Identifier, 'until')
self._expression = PascalExpression(self._block)
self._expression.parse(tokens)
# expression will consume the 'then' delimiter
logger.debug("Finished parsing repeat statement")
def next_token(self):
current_token = None
while current_token == None or current_token._kind == TokenKind.Comment:
if len(self._lookahead_toks) > 0:
current_token = self._lookahead_toks[0]
self._lookahead_toks = self._lookahead_toks[1:]
else:
current_token = self._tokeniser.next_token()
if current_token._kind == TokenKind.Comment:
logger.debug('TokenStream : Skipping comment: %s', current_token._value)
return current_token
def lookahead(self, count=1):
"""
lookahead generates a list of Tokens that has count number of members
It ignores comments
"""
logger.debug('TokenStream : Looking ahead %d', count)
while len(self._lookahead_toks) < count:
current_token = self._tokeniser.next_token()
while current_token._kind == TokenKind.Comment:
current_token = self._tokeniser.next_token()
self._lookahead_toks.append(current_token)
return self._lookahead_toks
def lookahead(self,count=1):
"""
lookahead generates a list of Tokens that has count number of members
It ignores comments
"""
logger.debug('TokenStream : Looking ahead %d', count)
while len(self._lookahead_toks) < count:
current_token = self._tokeniser.next_token()
while current_token._kind == TokenKind.Comment:
current_token = self._tokeniser.next_token()
self._lookahead_toks.append(current_token)
return self._lookahead_toks
def next_token(self):
current_token = None
while current_token == None or current_token._kind == TokenKind.Comment:
if len(self._lookahead_toks) > 0:
current_token = self._lookahead_toks[0]
self._lookahead_toks = self._lookahead_toks[1:]
else:
current_token = self._tokeniser.next_token()
if current_token._kind == TokenKind.Comment:
logger.debug('TokenStream : Skipping comment: %s',
current_token._value)
return current_token
def find_or_add_type(name):
if name == None: return None
from types.pas_type import PascalType
name_key = name.lower()
if name_key in _loaded_types:
return _loaded_types[name_key]
else:
logger.debug('Cache : Created Type %s', name)
result = PascalType(name)
_loaded_types[name_key] = result
return result
def parse(self, tokens):
from pas_parser_utils import parse_statement
logger.debug("Parsing if statement")
tokens.match_token(TokenKind.Identifier, 'if')
self._expression = PascalExpression(self._block)
self._expression.parse(tokens)
self._statement = parse_statement(tokens, self._block)
# expression will consume the 'then' delimiter
if tokens.match_lookahead(TokenKind.Identifier, 'else'):
tokens.match_token(TokenKind.Identifier, 'else')
self._else_statement = parse_statement(tokens, self._block)
tokens.match_lookahead(TokenKind.Symbol, ';', consume=True)
logger.debug("Finished parsing if statement")
def match_lookahead(self, token_kind, token_value = None, consume = False):
"""
looks forward one position for a token with a specific value and kind
if the token is found it returns true and optionally consume the token.
Otherwise it returns false without consuming a token.
"""
logger.debug('TokenStream : Looking to find %s (%s)%s', token_kind,
token_value if token_value != None else 'any',
' will consume' if consume else '')
token = self.lookahead(1)[0]
result = token._kind == token_kind and (token_value == None or token_value == token._value.lower())
if consume and result:
self.match_token(token_kind, token_value)
return result
def match_lookahead(self, token_kind, token_value=None, consume=False):
"""
looks forward one position for a token with a specific value and kind
if the token is found it returns true and optionally consume the token.
Otherwise it returns false without consuming a token.
"""
logger.debug('TokenStream : Looking to find %s (%s)%s', token_kind,
token_value if token_value != None else 'any',
' will consume' if consume else '')
token = self.lookahead(1)[0]
result = token._kind == token_kind and (
token_value == None or token_value == token._value.lower())
if consume and result:
self.match_token(token_kind, token_value)
return result
def parse(self, tokens):
from pas_parser_utils import parse_statement
logger.debug("Parsing case statement")
tokens.match_token(TokenKind.Identifier, 'case')
self._expression = PascalExpression(self._block)
self._expression.parse(tokens)
# expression will consume the 'of' keyword
while not (tokens.match_lookahead(TokenKind.Identifier, 'end')
or tokens.match_lookahead(TokenKind.Identifier, 'else') or
tokens.match_lookahead(TokenKind.Identifier, 'otherwise')):
constant = None
if (tokens.match_lookahead(TokenKind.Number)):
constant = tokens.match_token(TokenKind.Number).value
elif (tokens.match_lookahead(TokenKind.String)):
constant = tokens.match_token(TokenKind.String.value)
if len(constant) > 1:
raise_error(
("Constant expected: %s" % tokens.next_token()),
'',
is_critical=False)
elif tokens.match_lookahead(TokenKind.Identifier):
constant = self._block.resolve_variable(
tokens.match_token(TokenKind.Identifier).value)
else:
raise_error(("CaseStatement: Ordinal type expected: %s",
tokens.next_token()),
'',
is_critical=False)
tokens.match_token(TokenKind.Symbol, ':')
statement = parse_statement(tokens, self._block)
tokens.match_lookahead(TokenKind.Symbol, ';', consume=True)
self._case.append(tuple((constant, statement)))
if tokens.match_lookahead(
TokenKind.Identifier, 'else',
consume=True) or tokens.match_lookahead(
TokenKind.Identifier, 'otherwise', consume=True):
while not (tokens.match_lookahead(
TokenKind.Identifier, 'end', consume=True)):
self._else_statements.append(
parse_statement(tokens, self._block))
else:
tokens.match_token(TokenKind.Identifier, 'end')
tokens.match_token(TokenKind.Symbol, ';')
logger.debug("Finished parsing if statement")
def parse(self, tokens):
from pascal_parser.pas_parser_utils import parse_type
from pascal_parser.pas_var import PascalVariable
from pascal_parser.pas_param_declaration import PascalParameterDeclaration
logger.debug("Parsing overload declaration")
tokens.match_token(TokenKind.Identifier, 'operator')
self._operator = tokens.match_token(TokenKind.Operator).value
self._parameters = PascalParameterDeclaration(self._block)
self._parameters.parse(tokens, self)
name = tokens.match_token(TokenKind.Identifier)
tokens.match_token(TokenKind.Symbol, ':')
type = parse_type(tokens, self._block)
self._result = PascalVariable(name, type)
tokens.match_token(TokenKind.Symbol, ';')
logger.debug("Finished parsing operator declaration")
def parse(self, tokens):
from pascal_parser.pas_parser_utils import parse_type
from pascal_parser.pas_var import PascalVariable
from pascal_parser.pas_param_declaration import PascalParameterDeclaration
logger.debug("Parsing overload declaration")
tokens.match_token(TokenKind.Identifier, 'operator')
self._operator = tokens.match_token(TokenKind.Operator).value
self._parameters = PascalParameterDeclaration(self._block)
self._parameters.parse(tokens, self)
name = tokens.match_token(TokenKind.Identifier)
tokens.match_token(TokenKind.Symbol, ':')
type = parse_type(tokens, self._block)
self._result = PascalVariable(name, type)
tokens.match_token(TokenKind.Symbol, ';')
logger.debug("Finished parsing operator declaration")
def tokenise(self, filename):
'''Initialises the tokeniser with characters loaded from the specified
filename. Call `next_token` process each token.
'''
if isinstance(filename, list):
logger.debug('Tokenising list')
self.pas_lines = filename
else:
logger.debug('Tokenising %s', filename)
self._filename = filename
f = open(filename)
self.pas_lines = f.readlines()
f.close()
self._char_no = -1
self._line_no = 0 #starts at first line
self._token_val = 'none'
def tokenise(self, filename):
'''Initialises the tokeniser with characters loaded from the specified
filename. Call `next_token` process each token.
'''
if isinstance(filename, list):
logger.debug('Tokenising list')
self.pas_lines = filename
else:
logger.debug('Tokenising %s', filename)
self._filename = filename
f = open(filename)
self.pas_lines = f.readlines()
f.close()
self._char_no = -1
self._line_no = 0 #starts at first line
self._token_val = 'none'
def match_token(self, token_kind, token_value = None):
"""
Looks at the next token, and if it is the same kind as 'token_kind' and
has the same value as 'token_value' then it is returned. Otherwise an
error occurs and the program stops.
If 'token_value' is None then only the 'token_kind' is checked.
"""
tok = self.next_token()
if tok._kind != token_kind or (token_value != None and token_value != tok._value.lower()):
logger.error('TokenStream %s: found a %s (%s) expected %s (%s)',
self._tokeniser.line_details(),
tok._kind, tok._value, token_kind, token_value)
assert False
logger.debug('TokenStream : Matched token %s (%s)', tok._kind, tok._value)
return tok
def match_one_lookahead(self, token_lst, consume = False):
"""
Tries to match the next token's value with a list of values
Returns True if a match is found, and false if there was no match
"""
tok = self.lookahead()[0]
logger.debug('TokenStream : Looking to find %s %s', token_lst,
' will consume' if consume else '')
for token_kind,token_value in token_lst:
if tok._kind == token_kind and (token_value == None or token_value == tok._value):
matched = True
logger.debug('TokenStream : Found %s (%s)', tok._kind, tok._value)
if consume:
self.match_token(tok._kind, tok._value)
return True
return False
def match_one_lookahead(self, token_lst, consume=False):
"""
Tries to match the next token's value with a list of values
Returns True if a match is found, and false if there was no match
"""
tok = self.lookahead()[0]
logger.debug('TokenStream : Looking to find %s %s', token_lst,
' will consume' if consume else '')
for token_kind, token_value in token_lst:
if tok._kind == token_kind and (token_value == None
or token_value == tok._value):
matched = True
logger.debug('TokenStream : Found %s (%s)', tok._kind,
tok._value)
if consume:
self.match_token(tok._kind, tok._value)
return True
return False
def parse(self, tokens):
logger.debug("Parsing compound statement")
if tokens.match_lookahead(TokenKind.Identifier, 'begin'):
tokens.match_token(TokenKind.Identifier, 'begin')
while (True):
# compound statement currently consumes the end keyword, but not the symbol ';' or '.'
if tokens.match_lookahead(TokenKind.Identifier, 'end'):
tokens.match_token(TokenKind.Identifier, 'end') # consume end token
break
elif tokens.match_lookahead(TokenKind.Symbol, ';') or tokens.match_lookahead(TokenKind.Symbol, '.'):
# consumed end already -> somehow?
tokens.match_token(TokenKind.Symbol)
break
elif tokens.match_lookahead(TokenKind.Identifier, 'until'):
# repeat..until needs to have the until token so that it knows it's at the end
break
else:
self._statements.append(parse_statement(tokens, self._block))
logger.debug("Finished parsing compound statement")
def match_token(self, token_kind, token_value=None):
"""
Looks at the next token, and if it is the same kind as 'token_kind' and
has the same value as 'token_value' then it is returned. Otherwise an
error occurs and the program stops.
If 'token_value' is None then only the 'token_kind' is checked.
"""
tok = self.next_token()
if tok._kind != token_kind or (token_value != None
and token_value != tok._value.lower()):
logger.error(
'TokenStream %s: found a %s (%s) expected %s (%s)',
self._tokeniser.line_details(), tok._kind, tok._value,
token_kind, token_value)
assert False
logger.debug('TokenStream : Matched token %s (%s)', tok._kind,
tok._value)
return tok
def parse(self, tokens, do_resolve=True):
"""
parses a function call
"""
from pas_arguments import PascalArguments
logger.debug('Processing function call %s', self._identifier)
self._identifier = tokens.match_token(TokenKind.Identifier).value
self._arguments = PascalArguments(self._block)
self._arguments.parse(tokens)
if not self._inExpr:
tokens.match_lookahead(TokenKind.Symbol, ';', consume=True)
if do_resolve:
self._points_to = self._block.resolve_function_call(self)
if self._points_to is self:
# recursive call...
# need to get the function I'm being called from...
self._points_to = self._block._functions[self.name]
elif self._points_to is None:
raise_error(("Unable to resolve function call: " + self.identifier), '', is_critical=False)
logger.debug('Ended function call %s', self._identifier)
def parse(self, tokens):
logger.debug("Parsing compound statement")
if tokens.match_lookahead(TokenKind.Identifier, 'begin'):
tokens.match_token(TokenKind.Identifier, 'begin')
while (True):
# compound statement currently consumes the end keyword, but not the symbol ';' or '.'
if tokens.match_lookahead(TokenKind.Identifier, 'end'):
tokens.match_token(TokenKind.Identifier,
'end') # consume end token
break
elif tokens.match_lookahead(TokenKind.Symbol,
';') or tokens.match_lookahead(
TokenKind.Symbol, '.'):
# consumed end already -> somehow?
tokens.match_token(TokenKind.Symbol)
break
elif tokens.match_lookahead(TokenKind.Identifier, 'until'):
# repeat..until needs to have the until token so that it knows it's at the end
break
else:
self._statements.append(parse_statement(tokens, self._block))
logger.debug("Finished parsing compound statement")
def parse(self, tokens):
"""
this method parses the expression
"""
from tokeniser.pas_token_kind import TokenKind
logger.debug('Parsing %s expression' % ('inner' if self._innerExpr else ''))
newContent = None
while (True):
# expression over?
if tokens.match_lookahead(TokenKind.Symbol, ';') or tokens.match_lookahead(TokenKind.Symbol, ',') or tokens.match_lookahead(TokenKind.Identifier, 'then') or tokens.match_lookahead(TokenKind.Identifier, 'do') or tokens.match_lookahead(TokenKind.Identifier, 'of') or tokens.match_lookahead(TokenKind.Symbol, ']'):
if self._innerExpr:
raise_error(('Inner expression terminator expected, %s: ', tokens.next_token()), '', is_critical=False)
tokens.next_token() # consume the delimiter
logger.debug('Expression ended')
break
# Inner expression ended
elif tokens.match_lookahead(TokenKind.Symbol, ')'):
if self._innerExpr:
tokens.match_token(TokenKind.Symbol, ')') # consume the delimiter
logger.debug('Inner expression ended')
break
# starts with an Identifier
elif tokens.match_lookahead(TokenKind.Identifier):
# Function Call
if tokens.lookahead(2)[1].value is '(':
newContent = PascalFunctionCall(self._block, inExpr=True)
newContent.parse(tokens)
# Variable
else:
newContent = PascalVariableReference(self._block)
newContent.parse(tokens)
# Number
elif tokens.match_lookahead(TokenKind.Number):
newContent = PascalNumber(tokens.next_token().value)
# string
elif tokens.match_lookahead(TokenKind.String):
newContent = PascalString(tokens.next_token().value)
# Operator
elif tokens.match_lookahead(TokenKind.Operator):
newContent = PascalOperator(tokens.match_token(TokenKind.Operator).value)
# inner expression
elif tokens.match_lookahead(TokenKind.Symbol, '('):
tokens.next_token() # consume the delimiter
newContent = PascalExpression(self._block, innerExpr = True)
newContent.parse(tokens)
# Boolean
elif tokens.match_lookahead(TokenKind.Boolean):
newContent = PascalBool(tokens.match_token(TokenKind.Boolean).value)
else:
raise_error(('Unknown expression token... %s' % str(tokens.next_token().value)), '', is_critical=False)
self._contents.append(newContent)
def next_token(self):
'''
Find and return the next token
'''
def num_match(cha, tmp):
'''Checks for a number in format ##, ##.#. Returns False when at the
end of a number.'''
if cha in '1234567890':
return True
elif cha == '.' and '.' not in tmp:
return self._peek(1) in '1234567890'
else:
return False
while (True):
t = self._next_char()
self._token_start = self._char_no
kind = None
value = ''
# Ignore white space characters
if t == ' ' or t == '\t': #ignore white space
pass
# Move to next line (if at end of line)
elif t == '\n':
self._advance_line()
# Numbers (int or float style format
elif t in '1234567890' or (t in '-+' and self._peek(1)
in '1234567890'): #is digit or +/-
kind = TokenKind.Number
char_number = self._char_no
value = self._read_matching(t, num_match)
logger.debug('Tokeniser : read %s (%s)', kind, value)
return Token(kind, value, self._line_no + 1, char_number + 1)
# Comment, single line // or meta comment line ///
elif t == '/' and self._peek(1) == '/': #start of comment
if self._match_and_read('/'):
if self._match_and_read('/'):
self._meta_comment_start = self._char_no
self._meta_comment_line = self._line_no
kind = TokenKind.MetaComment
char_number = self._char_no
value = self.read_to_end_of_comment()
else:
kind = TokenKind.Comment
char_number = self._char_no
value = self.read_to_eol()
logger.debug('Tokeniser : read %s (%s)', kind,
value)
return Token(kind, value, self._line_no + 1,
char_number + 1)
else:
logger.error("Unexpected error: " + self.line_details)
# Attribute identified by an @ symbol then a name
elif t == '@':
kind = TokenKind.Attribute
char_number = self._char_no
value = self._read_matching(
'', lambda cha, tmp: cha.isalnum() or cha == '_')
logger.debug('Tokeniser : read %s (%s)', kind, value)
return Token(kind, value, self._line_no + 1, char_number + 1)
# Identifier (id) of alphanumeric characters including
elif t.isalpha():
char_number = self._char_no
value = self._read_matching(
t, lambda cha, tmp: cha.isalnum() or cha == '_')
if value.lower() in ['true', 'false']:
kind = TokenKind.Boolean
elif value.lower() in [
'or', 'and', 'not', 'xor', 'mod', 'div', 'in'
]:
kind = TokenKind.Operator
else:
kind = TokenKind.Identifier
logger.debug('Tokeniser : read %s (%s)', kind, value)
return Token(kind, value, self._line_no + 1, char_number + 1)
#Bound Comment
elif t == '{' or (t == '(' and self._peek(1) == '*'):
if t == '(' and self._match_and_read('*'):
char_number = self._char_no
comment = self._read_until('',
lambda temp: temp[-2:] == '*)')
kind = TokenKind.Comment
value = comment[:-2]
logger.debug('Tokeniser : read %s (%s)', kind, value)
return Token(kind, value, self._line_no + 1,
char_number + 1)
elif t == '{':
comment = self._read_until('',
lambda temp: temp[-1:] == '}')
kind = TokenKind.Comment
value = comment[:-1]
logger.debug('Tokeniser : read %s (%s)', kind, value)
return Token(kind, value, self._line_no + 1,
char_number + 1)
# Operator
elif (t == ':' and self._peek(1) == '=') or t in '=+-*/><':
kind = TokenKind.Operator
char_number = self._char_no
if t == ':' and self._match_and_read('='):
value = ':='
elif t in '+' and self._match_and_read('='):
value = t + '='
elif t in '-' and self._match_and_read('='):
value = t + '='
elif t in '/' and self._match_and_read('='):
value = t + '='
elif t in '*' and self._match_and_read('='):
value = t + '='
elif t == '*' and self._match_and_read('*'):
value = '**'
elif t == '<' and self._match_and_read('>'):
value = '<>'
elif t in '<>' and self._match_and_read('='):
value = t + '='
else:
value = t
logger.debug('Tokeniser : read %s (%s)', kind, value)
return Token(kind, value, self._line_no + 1, char_number + 1)
# Symbol
elif t in '(),:;[].^':
kind = TokenKind.Symbol
char_number = self._char_no
value = t
logger.debug('Tokeniser : read %s (%s)', kind, value)
return Token(kind, value, self._line_no + 1, char_number + 1)
# Catch any single quotes inside a string value.
elif t == "'":
char_number = self._char_no
string = self._read_until(
'', lambda temp:
(temp[-1:] == "'") and (not self._match_and_read("'")))
kind = TokenKind.String
value = string[:-1]
logger.debug('Tokeniser : read %s (%s)', kind, value)
return Token(kind, value, self._line_no + 1, char_number + 1)
# Hmm.. unknown token. What did we forget?
else:
logger.error("Unknown token type: " + t if t else 'NONE!')
def next_token(self):
'''
Find and return the next token
'''
def num_match(cha, tmp):
'''Checks for a number in format ##, ##.#. Returns False when at the
end of a number.'''
if cha in '1234567890':
return True
elif cha == '.' and '.' not in tmp:
return self._peek(1) in '1234567890'
else:
return False
while (True):
t = self._next_char();
self._token_start = self._char_no
kind = None
value = ''
# Ignore white space characters
if t == ' ' or t == '\t': #ignore white space
pass
# Move to next line (if at end of line)
elif t == '\n':
self._advance_line()
# Numbers (int or float style format
elif t in '1234567890' or (t in '-+' and self._peek(1) in '1234567890'): #is digit or +/-
kind = TokenKind.Number
char_number = self._char_no
value = self._read_matching(t, num_match)
logger.debug('Tokeniser : read %s (%s)', kind, value)
return Token(kind, value, self._line_no+1, char_number+1)
# Comment, single line // or meta comment line ///
elif t == '/' and self._peek(1) == '/': #start of comment
if self._match_and_read('/'):
if self._match_and_read('/'):
self._meta_comment_start = self._char_no
self._meta_comment_line = self._line_no
kind = TokenKind.MetaComment
char_number = self._char_no
value = self.read_to_end_of_comment()
else:
kind = TokenKind.Comment
char_number = self._char_no
value = self.read_to_eol()
logger.debug('Tokeniser : read %s (%s)', kind, value)
return Token(kind, value, self._line_no+1, char_number+1)
else:
logger.error("Unexpected error: " + self.line_details)
# Attribute identified by an @ symbol then a name
elif t == '@':
kind = TokenKind.Attribute
char_number = self._char_no
value = self._read_matching('', lambda cha, tmp: cha.isalnum() or cha == '_')
logger.debug('Tokeniser : read %s (%s)', kind, value)
return Token(kind, value, self._line_no+1, char_number+1)
# Identifier (id) of alphanumeric characters including
elif t.isalpha():
char_number = self._char_no
value = self._read_matching(t, lambda cha, tmp: cha.isalnum() or cha == '_')
if value.lower() in ['true','false']:
kind = TokenKind.Boolean
elif value.lower() in ['or', 'and', 'not', 'xor', 'mod', 'div', 'in']:
kind = TokenKind.Operator
else:
kind = TokenKind.Identifier
logger.debug('Tokeniser : read %s (%s)', kind, value)
return Token(kind, value, self._line_no+1, char_number+1)
#Bound Comment
elif t == '{' or (t == '(' and self._peek(1) == '*'):
if t == '(' and self._match_and_read('*'):
char_number = self._char_no
comment = self._read_until('', lambda temp: temp[-2:] == '*)')
kind = TokenKind.Comment
value = comment[:-2]
logger.debug('Tokeniser : read %s (%s)', kind, value)
return Token(kind, value, self._line_no+1, char_number+1)
elif t == '{':
comment = self._read_until('', lambda temp: temp[-1:] == '}')
kind = TokenKind.Comment
value = comment[:-1]
logger.debug('Tokeniser : read %s (%s)', kind, value)
return Token(kind, value, self._line_no+1, char_number+1)
# Operator
elif (t == ':' and self._peek(1) == '=') or t in '=+-*/><':
kind = TokenKind.Operator
char_number = self._char_no
if t == ':' and self._match_and_read('='):
value = ':='
elif t in '+' and self._match_and_read('='):
value = t + '='
elif t in '-' and self._match_and_read('='):
value = t + '='
elif t in '/' and self._match_and_read('='):
value = t + '='
elif t in '*' and self._match_and_read('='):
value = t + '='
elif t == '*' and self._match_and_read('*'):
value = '**'
elif t == '<' and self._match_and_read('>'):
value = '<>'
elif t in '<>' and self._match_and_read('='):
value = t + '='
else:
value = t
logger.debug('Tokeniser : read %s (%s)', kind, value)
return Token(kind, value, self._line_no+1, char_number+1)
# Symbol
elif t in '(),:;[].^':
kind = TokenKind.Symbol
char_number = self._char_no
value = t
logger.debug('Tokeniser : read %s (%s)', kind, value)
return Token(kind, value, self._line_no+1, char_number+1)
# Catch any single quotes inside a string value.
elif t == "'":
char_number = self._char_no
string = self._read_until('', lambda temp: (temp[-1:] == "'") and (not self._match_and_read("'")))
kind = TokenKind.String
value = string[:-1]
logger.debug('Tokeniser : read %s (%s)', kind, value)
return Token(kind, value, self._line_no+1, char_number+1)
# Hmm.. unknown token. What did we forget?
else:
logger.error("Unknown token type: " + t if t else 'NONE!')
def parse(self, tokens):
"""
this method parses the expression
"""
from tokeniser.pas_token_kind import TokenKind
logger.debug('Parsing %s expression' %
('inner' if self._innerExpr else ''))
newContent = None
while (True):
# expression over?
if tokens.match_lookahead(
TokenKind.Symbol, ';') or tokens.match_lookahead(
TokenKind.Symbol, ',') or tokens.match_lookahead(
TokenKind.Identifier,
'then') or tokens.match_lookahead(
TokenKind.Identifier,
'do') or tokens.match_lookahead(
TokenKind.Identifier,
'of') or tokens.match_lookahead(
TokenKind.Symbol, ']'):
if self._innerExpr:
raise_error(('Inner expression terminator expected, %s: ',
tokens.next_token()),
'',
is_critical=False)
tokens.next_token() # consume the delimiter
logger.debug('Expression ended')
break
# Inner expression ended
elif tokens.match_lookahead(TokenKind.Symbol, ')'):
if self._innerExpr:
tokens.match_token(TokenKind.Symbol,
')') # consume the delimiter
logger.debug('Inner expression ended')
break
# starts with an Identifier
elif tokens.match_lookahead(TokenKind.Identifier):
# Function Call
if tokens.lookahead(2)[1].value is '(':
newContent = PascalFunctionCall(self._block, inExpr=True)
newContent.parse(tokens)
# Variable
else:
newContent = PascalVariableReference(self._block)
newContent.parse(tokens)
# Number
elif tokens.match_lookahead(TokenKind.Number):
newContent = PascalNumber(tokens.next_token().value)
# string
elif tokens.match_lookahead(TokenKind.String):
newContent = PascalString(tokens.next_token().value)
# Operator
elif tokens.match_lookahead(TokenKind.Operator):
newContent = PascalOperator(
tokens.match_token(TokenKind.Operator).value)
# inner expression
elif tokens.match_lookahead(TokenKind.Symbol, '('):
tokens.next_token() # consume the delimiter
newContent = PascalExpression(self._block, innerExpr=True)
newContent.parse(tokens)
# Boolean
elif tokens.match_lookahead(TokenKind.Boolean):
newContent = PascalBool(
tokens.match_token(TokenKind.Boolean).value)
else:
raise_error(('Unknown expression token... %s' %
str(tokens.next_token().value)),
'',
is_critical=False)
self._contents.append(newContent)