def __parse_body(self): """ parse macro content, store subtokens """ if self.args is None: self.tokens.append( DT_Code(self.body) ) else: rd = CodeReader(self.body) buff = '' while not rd.has_end(): if rd.has_identifier(): ident = rd.consume_identifier() # check if macro argument if ident in self.args: # append collected code fragment if len(buff) > 0: t = DT_Code(buff) buff = '' self.tokens.append(t) # append the var t = DT_Var(ident) self.tokens.append(t) else: buff += ident elif rd.has_string(): buff += rd.consume_string() elif rd.has_char(): buff += rd.consume_char() else: # just add the character to the currently built DT_Code buff += rd.consume() # add trailing code fragment if len(buff) > 0: t = DT_Code(buff) buff = '' self.tokens.append(t)
def _tokenize(self):
""" Parse expression sub-tokens """
rd = CodeReader(self.value)
while not rd.has_end():
rd.sweep()
if rd.has_identifier():
# an identifier
# can be variable or a function call
s = rd.consume_identifier()
t = T_Name(s)
self.tokens.append(t)
rd.sweep()
if rd.has_bracket():
# array index
s = rd.consume_block()
t = T_Bracket(s)
self.tokens.append(t)
elif rd.has_paren():
# paren with arguments for the function
s = rd.consume_block()
t = T_Paren(s)
t.set_type(ParenType.ARGVALS)
self.tokens.append(t)
elif rd.has_paren():
# Parenthesised sub-expression
s = rd.consume_block()
t = T_Paren(s)
t.set_type(ParenType.EXPR)
self.tokens.append(t)
elif rd.has_number():
# Number literal
s = rd.consume_number()
t = T_Number(s)
self.tokens.append(t)
elif (len(self.tokens) > 0 and
type(self.tokens[-1:][0]) is T_Operator
and rd.matches(r'[-+]\s*[0-9a-z_]+')):
# Number literal
sign = rd.consume()
if sign == '+':
sign = ''
rd.sweep()
if sign == '-':
self.tokens.append(T_Number('-1'))
self.tokens.append(T_Operator('*'))
elif rd.has_operator():
# Operator
s = rd.consume_operator()
t = T_Operator(s)
self.tokens.append(t)
elif rd.has_char():
# Char literal
s = rd.consume_char()
t = T_Char(s)
self.tokens.append(t)
elif rd.has_string():
# String literal
s = rd.consume_string()
t = T_String(s)
self.tokens.append(t)
else:
raise Exception('Unexpected expression token near' + rd.peek(10))
for t in self.tokens:
if t.is_composite():
t.tokenize()
def _tokenize(self):
""" Parse expression sub-tokens """
rd = CodeReader(self.value)
while not rd.has_end():
rd.sweep()
if rd.has_identifier():
# an identifier
# can be variable or a function call
s = rd.consume_identifier()
t = T_Name(s)
self.tokens.append(t)
rd.sweep()
if rd.has_bracket():
# array index
s = rd.consume_block()
t = T_Bracket(s)
self.tokens.append(t)
elif rd.has_paren():
# paren with arguments for the function
s = rd.consume_block()
t = T_Paren(s)
t.set_type(ParenType.ARGVALS)
self.tokens.append(t)
elif rd.has_paren():
# Parenthesised sub-expression
s = rd.consume_block()
t = T_Paren(s)
t.set_type(ParenType.EXPR)
self.tokens.append(t)
elif rd.has_number():
# Number literal
s = rd.consume_number()
t = T_Number(s)
self.tokens.append(t)
elif (((len(self.tokens) > 0 and
type(self.tokens[-1:][0]) is T_Operator) or len(self.tokens) == 0)
and rd.matches(r'[-+]\s*[0-9a-z_]+')):
# Unary operator
sign = rd.consume()
if sign == '+':
sign = ''
rd.sweep()
if sign == '-':
self.tokens.append(T_Operator('@-'))
elif rd.has_operator():
# Operator
s = rd.consume_operator()
t = T_Operator(s)
self.tokens.append(t)
elif rd.has_char():
# Char literal
s = rd.consume_char()
t = T_Char(s)
self.tokens.append(t)
elif rd.has_string():
# String literal
s = rd.consume_string()
t = T_String(s)
self.tokens.append(t)
else:
raise Exception('Unexpected expression token near' + rd.peek(10))
for t in self.tokens:
if t.is_composite():
t.tokenize()