Example #1
0
def parseDefine( match, lineno, colno, terminalId, lexer ):
  identifier_regex = r'([a-zA-Z_]|\\[uU]([0-9a-fA-F]{4})([0-9a-fA-F]{4})?)([a-zA-Z_0-9]|\\[uU]([0-9a-fA-F]{4})([0-9a-fA-F]{4})?)*'
  if re.match(r'[ \t]+%s\(' % (identifier_regex), lexer.string):
    terminalId = pp_Parser.TERMINAL_DEFINE_FUNCTION
  else:
    terminalId = pp_Parser.TERMINAL_DEFINE
  lexer.addToken(ppToken(terminalId, lexer.resource, pp_Parser.terminals[terminalId], match, lineno, colno))
Example #2
0
def parseInclude( match, lineno, colno, terminalId, lexer ):
  headerGlobal = re.compile(r'[<][^\n>]+[>]')
  headerLocal = re.compile(r'["][^\n"]+["]')
  leadingWhitespace = re.compile(r'[\t ]*')

  lexer.addToken(ppToken(terminalId, lexer.resource, pp_Parser.terminals[terminalId], match, lineno, colno))
  lexer.advance( leadingWhitespace.match(lexer.string).group(0) )

  regexes = {
    pp_Parser.TERMINAL_HEADER_GLOBAL: headerGlobal,
    pp_Parser.TERMINAL_HEADER_LOCAL: headerLocal
  }

  for terminalId, regex in regexes.items():
    rmatch = regex.match(lexer.string)
    if rmatch:
      rstring = rmatch.group(0)
      token = ppToken(terminalId, lexer.resource, pp_Parser.terminals[terminalId], rstring, lexer.lineno, lexer.colno)
      lexer.addToken(token)
      lexer.advance(rstring)
      break
Example #3
0
  def eval_identifier(self, cPPAST):
    replacementList = TokenList()
    if cPPAST.getString() in self.symbols:
      replacementList = self.symbols[cPPAST.getString()]
   
    def tokenize(token):
      tId = self.cTtocPPT[token.id]
      return ppToken(tId, token.resource, pp_Parser.terminals[tId], token.source_string, token.lineno, token.colno)

    self.cPPP.tokens = TokenStream(list(map(tokenize, replacementList)))
    parsetree = self.cPPP.parse__expr()
    ast = parsetree.toAst()
    value = self._eval(ast)
    ppZero = ppToken(self.cPPP.terminals['pp_number'], None, 'pp_number', value, 0, 0)

    if isinstance(value, Token):
      return value
    return ppZero
Example #4
0
 def tokenize(token):
   tId = self.cTtocPPT[token.id]
   return ppToken(tId, token.resource, pp_Parser.terminals[tId], token.source_string, token.lineno, token.colno)
Example #5
0
def token(string, lineno, colno, terminalId, lexer):
  lexer.addToken(ppToken(terminalId, lexer.resource, pp_Parser.terminals[terminalId], string, lineno, colno))
Example #6
0
def parseDefined( match, lineno, colno, terminalId, lexer ):
  separatorId = pp_Parser.TERMINAL_DEFINED_SEPARATOR
  lexer.addToken(ppToken(terminalId, lexer.resource, pp_Parser.terminals[terminalId], match, lineno, colno))
  lexer.addToken(ppToken(separatorId, lexer.resource, pp_Parser.terminals[separatorId], match, lineno, colno))
Example #7
0
  def __next__(self):
    if self._hasToken():
      token = self._popToken()
      return token

    if not len(self.cST_lines):
      raise StopIteration()

    buf = []
    buf_line = 0
    lines = 0
    token = None
    emit_separator = False
    emit_csource = False
    continuation = False
    advance = 0
    cComment = False
    for index, line in enumerate(self.cST_lines):
      self.lineno += 1

      if not cComment and (self._isPreprocessingLine( line ) or continuation):
        continuation = False
        if len(buf):
          self.lineno -= 1
          emit_csource = True
          break
        if '/*' in line and '*/' not in line:
          line = re.sub('/\*.*$', '', line)
          try:
            i = index
            while True:
              i += 1
              lines += 1
              self.lineno += 1
              if '*/' in self.cST_lines[i]:
                line += re.sub('^.*\*/', '', self.cST_lines[i])
                break
          except IndexError:
            pass
        if line.strip() == '#':
          lines += 1
          continue
        if len(line) and line[-1] == '\\':
          line = line[:-1]
          continuation = True
        cPPL_PatternMatcher = PatternMatchingLexer( SourceCodeString(self.resource, line, self.lineno, 1), self.regex )
        for cPPT in cPPL_PatternMatcher:
          self._addToken(ppToken(cPPT.id, self.resource, cPPT.terminal_str, cPPT.source_string, cPPT.lineno, cPPT.colno))
          if cPPT.terminal_str.upper() in ['INCLUDE', 'DEFINE', 'DEFINE_FUNCTION', 'PRAGMA', 'ERROR', 'WARNING', 'LINE', 'ENDIF', 'UNDEF']:
            emit_separator = True
        if continuation:
          lines += 1
          continue
        if emit_separator:
          terminalId = pp_Parser.TERMINAL_SEPARATOR
          self._addToken( ppToken(terminalId, self.resource, pp_Parser.terminals[terminalId], '', self.lineno, 1) )
        self._advance( lines + 1 )
        if self._hasToken():
          return self._popToken()
        raise Exception('Unexpected')
      else:
        emit_csource = True
        if not len(buf):
          buf_line = self.lineno
        buf.append(line)
        lines += 1
        if not cComment and '/*' in line and '*/' not in line:
          cComment = True
        if cComment and '*/' in line:
          cComment = False

    self._advance(lines)
    if emit_csource:
      csourceId = pp_Parser.TERMINAL_CSOURCE
      separatorId = pp_Parser.TERMINAL_SEPARATOR
      token = ppToken(csourceId, self.resource, pp_Parser.terminals[csourceId], '\n'.join(buf), buf_line, 1)
      self._addToken( ppToken(separatorId, self.resource, pp_Parser.terminals[separatorId], '', self.lineno, 1) )
      return token
    raise StopIteration()
Example #8
0
 def matchString(self, string):
   for (regex, terminalId, function) in self.regex:
     match = regex.match(string)
     if match:
       return ppToken(terminalId, self.resource, pp_Parser.terminals[terminalId], match.group(0), 0, 0)
   return None