def parseDefine( match, lineno, colno, terminalId, lexer ): identifier_regex = r'([a-zA-Z_]|\\[uU]([0-9a-fA-F]{4})([0-9a-fA-F]{4})?)([a-zA-Z_0-9]|\\[uU]([0-9a-fA-F]{4})([0-9a-fA-F]{4})?)*' if re.match(r'[ \t]+%s\(' % (identifier_regex), lexer.string): terminalId = pp_Parser.TERMINAL_DEFINE_FUNCTION else: terminalId = pp_Parser.TERMINAL_DEFINE lexer.addToken(ppToken(terminalId, lexer.resource, pp_Parser.terminals[terminalId], match, lineno, colno))
def parseInclude( match, lineno, colno, terminalId, lexer ): headerGlobal = re.compile(r'[<][^\n>]+[>]') headerLocal = re.compile(r'["][^\n"]+["]') leadingWhitespace = re.compile(r'[\t ]*') lexer.addToken(ppToken(terminalId, lexer.resource, pp_Parser.terminals[terminalId], match, lineno, colno)) lexer.advance( leadingWhitespace.match(lexer.string).group(0) ) regexes = { pp_Parser.TERMINAL_HEADER_GLOBAL: headerGlobal, pp_Parser.TERMINAL_HEADER_LOCAL: headerLocal } for terminalId, regex in regexes.items(): rmatch = regex.match(lexer.string) if rmatch: rstring = rmatch.group(0) token = ppToken(terminalId, lexer.resource, pp_Parser.terminals[terminalId], rstring, lexer.lineno, lexer.colno) lexer.addToken(token) lexer.advance(rstring) break
def eval_identifier(self, cPPAST): replacementList = TokenList() if cPPAST.getString() in self.symbols: replacementList = self.symbols[cPPAST.getString()] def tokenize(token): tId = self.cTtocPPT[token.id] return ppToken(tId, token.resource, pp_Parser.terminals[tId], token.source_string, token.lineno, token.colno) self.cPPP.tokens = TokenStream(list(map(tokenize, replacementList))) parsetree = self.cPPP.parse__expr() ast = parsetree.toAst() value = self._eval(ast) ppZero = ppToken(self.cPPP.terminals['pp_number'], None, 'pp_number', value, 0, 0) if isinstance(value, Token): return value return ppZero
def tokenize(token): tId = self.cTtocPPT[token.id] return ppToken(tId, token.resource, pp_Parser.terminals[tId], token.source_string, token.lineno, token.colno)
def token(string, lineno, colno, terminalId, lexer): lexer.addToken(ppToken(terminalId, lexer.resource, pp_Parser.terminals[terminalId], string, lineno, colno))
def parseDefined( match, lineno, colno, terminalId, lexer ): separatorId = pp_Parser.TERMINAL_DEFINED_SEPARATOR lexer.addToken(ppToken(terminalId, lexer.resource, pp_Parser.terminals[terminalId], match, lineno, colno)) lexer.addToken(ppToken(separatorId, lexer.resource, pp_Parser.terminals[separatorId], match, lineno, colno))
def __next__(self): if self._hasToken(): token = self._popToken() return token if not len(self.cST_lines): raise StopIteration() buf = [] buf_line = 0 lines = 0 token = None emit_separator = False emit_csource = False continuation = False advance = 0 cComment = False for index, line in enumerate(self.cST_lines): self.lineno += 1 if not cComment and (self._isPreprocessingLine( line ) or continuation): continuation = False if len(buf): self.lineno -= 1 emit_csource = True break if '/*' in line and '*/' not in line: line = re.sub('/\*.*$', '', line) try: i = index while True: i += 1 lines += 1 self.lineno += 1 if '*/' in self.cST_lines[i]: line += re.sub('^.*\*/', '', self.cST_lines[i]) break except IndexError: pass if line.strip() == '#': lines += 1 continue if len(line) and line[-1] == '\\': line = line[:-1] continuation = True cPPL_PatternMatcher = PatternMatchingLexer( SourceCodeString(self.resource, line, self.lineno, 1), self.regex ) for cPPT in cPPL_PatternMatcher: self._addToken(ppToken(cPPT.id, self.resource, cPPT.terminal_str, cPPT.source_string, cPPT.lineno, cPPT.colno)) if cPPT.terminal_str.upper() in ['INCLUDE', 'DEFINE', 'DEFINE_FUNCTION', 'PRAGMA', 'ERROR', 'WARNING', 'LINE', 'ENDIF', 'UNDEF']: emit_separator = True if continuation: lines += 1 continue if emit_separator: terminalId = pp_Parser.TERMINAL_SEPARATOR self._addToken( ppToken(terminalId, self.resource, pp_Parser.terminals[terminalId], '', self.lineno, 1) ) self._advance( lines + 1 ) if self._hasToken(): return self._popToken() raise Exception('Unexpected') else: emit_csource = True if not len(buf): buf_line = self.lineno buf.append(line) lines += 1 if not cComment and '/*' in line and '*/' not in line: cComment = True if cComment and '*/' in line: cComment = False self._advance(lines) if emit_csource: csourceId = pp_Parser.TERMINAL_CSOURCE separatorId = pp_Parser.TERMINAL_SEPARATOR token = ppToken(csourceId, self.resource, pp_Parser.terminals[csourceId], '\n'.join(buf), buf_line, 1) self._addToken( ppToken(separatorId, self.resource, pp_Parser.terminals[separatorId], '', self.lineno, 1) ) return token raise StopIteration()
def matchString(self, string): for (regex, terminalId, function) in self.regex: match = regex.match(string) if match: return ppToken(terminalId, self.resource, pp_Parser.terminals[terminalId], match.group(0), 0, 0) return None