def tokenize(self): """ Convert to tokens, get token list Returns: A list of obtained tokens, recursively tokenized. """ if self.tokens is not None: return self.tokens self.tokens = [] rd = CodeReader(self.source, self.filename) while not rd.has_end(): # discard garbage rd.sweep() # End of string. if rd.has_end(): break # <identifier> elif rd.has_identifier(): self._tokenize_identifier(rd) # {...stuff...} elif rd.has_code_block(): s = rd.consume_block() self._add( T_CodeBlock(s) ) # ; elif rd.starts(';'): self._collect_semicolon(rd) else: rd.error('Unexpected syntax here.') # tokenize all composite tokens for t in self.tokens: if t.is_composite(): t.tokenize() return self.tokens
def __init__(self, value): super().__init__(value) rd = CodeReader(value) rd.consume_exact('#pragma') rd.consume_inline_whitespace() self.name = rd.consume_identifier() rd.consume_inline_whitespace() if rd.has_identifier(): self.value = rd.consume_identifier() # identifier without quotes elif rd.has_number(): n = rd.consume_number() try: self.value = int(n, 10) except ValueError: try: self.value = int(n, 16) except ValueError: try: self.value = int(n, 2) except ValueError: rd.error('Could not parse number: %s' % n) elif rd.has_string(): self.value = rd.consume_string()[1:-1] # crop quotes else: self.value = True # boolean directive (flag) v = self.value if type(v) is str: self.value = {'true': True, 'false': False}.get(v.lower(), v)
def __init__(self, value): super().__init__(value) rd = CodeReader(value) rd.consume_exact('#define') rd.consume_inline_whitespace() # get macro name self.name = rd.consume_identifier() # arraylike flag self.arraylike = False self.functionlike = False # macro arguments self.args = None # which argument is variadic self.vararg_pos = None #print(str(rd.has_bracket())) if rd.has_paren(): tmp = rd.consume_block()[1:-1] # inside the paren self.args = [] for a in tmp.split(','): a = a.strip() if len(a) > 0: if a[-3:] == '...': # a is a variadic argument if self.vararg_pos is not None: rd.error('Macro can have only one variadic argument!') self.vararg_pos = len(self.args) a = a[:-3].strip() self.args.append(a) self.functionlike = True elif rd.has_bracket(): tmp = rd.consume_block()[1:-1].strip() # inside the bracket if not re.match(r'\A[a-zA-Z_][a-zA-Z0-9_]*\Z', tmp): rd.error('Invalid argument format for macro "%s": %s' % (self.name, tmp)) self.args = [tmp] self.arraylike = True rd.consume_inline_whitespace() # macro body self.body = rd.consume_all() # macro body tokens self.tokens = [] self.__parse_body()