def tokenize(self):
""" Convert to tokens, get token list
Returns:
A list of obtained tokens, recursively tokenized.
"""
if self.tokens is not None:
return self.tokens
self.tokens = []
rd = CodeReader(self.source, self.filename)
while not rd.has_end():
# discard garbage
rd.sweep()
# End of string.
if rd.has_end():
break
# <identifier>
elif rd.has_identifier():
self._tokenize_identifier(rd)
# {...stuff...}
elif rd.has_code_block():
s = rd.consume_block()
self._add( T_CodeBlock(s) )
# ;
elif rd.starts(';'):
self._collect_semicolon(rd)
else:
rd.error('Unexpected syntax here.')
# tokenize all composite tokens
for t in self.tokens:
if t.is_composite():
t.tokenize()
return self.tokens
def tokenize(self):
""" Convert to tokens, get token list
Returns:
A list of obtained tokens, recursively tokenized.
"""
if self.tokens is not None:
return self.tokens
self.tokens = []
rd = CodeReader(self.source, self.filename)
while not rd.has_end():
# discard garbage
rd.sweep()
# End of string.
if rd.has_end():
break
# <identifier>
elif rd.has_identifier():
self._tokenize_identifier(rd)
# {...stuff...}
elif rd.has_code_block():
s = rd.consume_block()
self._add( T_CodeBlock(s) )
# ;
elif rd.starts(';'):
self._collect_semicolon(rd)
else:
rd.error('Unexpected syntax here.')
# tokenize all composite tokens
for t in self.tokens:
if t.is_composite():
t.tokenize()
return self.tokens
def __init__(self, value):
super().__init__(value)
rd = CodeReader(value)
rd.consume_exact('#pragma')
rd.consume_inline_whitespace()
self.name = rd.consume_identifier()
rd.consume_inline_whitespace()
if rd.has_identifier():
self.value = rd.consume_identifier() # identifier without quotes
elif rd.has_number():
n = rd.consume_number()
try:
self.value = int(n, 10)
except ValueError:
try:
self.value = int(n, 16)
except ValueError:
try:
self.value = int(n, 2)
except ValueError:
rd.error('Could not parse number: %s' % n)
elif rd.has_string():
self.value = rd.consume_string()[1:-1] # crop quotes
else:
self.value = True # boolean directive (flag)
v = self.value
if type(v) is str:
self.value = {'true': True, 'false': False}.get(v.lower(), v)
def __init__(self, value):
super().__init__(value)
rd = CodeReader(value)
rd.consume_exact('#define')
rd.consume_inline_whitespace()
# get macro name
self.name = rd.consume_identifier()
# arraylike flag
self.arraylike = False
self.functionlike = False
# macro arguments
self.args = None
# which argument is variadic
self.vararg_pos = None
#print(str(rd.has_bracket()))
if rd.has_paren():
tmp = rd.consume_block()[1:-1] # inside the paren
self.args = []
for a in tmp.split(','):
a = a.strip()
if len(a) > 0:
if a[-3:] == '...':
# a is a variadic argument
if self.vararg_pos is not None:
rd.error('Macro can have only one variadic argument!')
self.vararg_pos = len(self.args)
a = a[:-3].strip()
self.args.append(a)
self.functionlike = True
elif rd.has_bracket():
tmp = rd.consume_block()[1:-1].strip() # inside the bracket
if not re.match(r'\A[a-zA-Z_][a-zA-Z0-9_]*\Z', tmp):
rd.error('Invalid argument format for macro "%s": %s' % (self.name, tmp))
self.args = [tmp]
self.arraylike = True
rd.consume_inline_whitespace()
# macro body
self.body = rd.consume_all()
# macro body tokens
self.tokens = []
self.__parse_body()
