def main(): import sys env = {} if len(sys.argv) < 2: filename = "/usr/include/stdio.h" else: filename = sys.argv[1] with open(filename, 'r') as fd: contents = fd.read() stream = CharacterStream(trigraphs.translate(contents), 1, filename) state = PreprocessorState(stream, env, include_chain=(filename, ), include=include_stub) try: position = (1, filename) for token in chomp(state): if position_of(token) != position: position = advance_position(position, position_of(token)) # Token generation. This is still wrong if name_of(token) == 'string': sys.stdout.write('"' + value_of(token) + '" ') elif name_of(token) == 'char': sys.stdout.write("'" + value_of(token) + "' ") else: sys.stdout.write(value_of(token) + ' ') sys.stdout.write('\n') except AssertionError as ass: traceback.print_exc() for x in range(5): print tokenize.chop(stream)
def main(): import sys env = {} if len(sys.argv) < 2: filename = "/usr/include/stdio.h" else: filename = sys.argv[1] with open(filename, 'r') as fd: contents = fd.read() stream = CharacterStream(trigraphs.translate(contents), 1, filename) state = PreprocessorState(stream, env, include_chain=(filename,), include=include_stub) try: position = (1, filename) for token in chomp(state): if position_of(token) != position: position = advance_position(position, position_of(token)) # Token generation. This is still wrong if name_of(token) == 'string': sys.stdout.write('"' + value_of(token) + '" ') elif name_of(token) == 'char': sys.stdout.write("'" + value_of(token) + "' ") else: sys.stdout.write(value_of(token) + ' ') sys.stdout.write('\n') except AssertionError as ass: traceback.print_exc() for x in range(5): print tokenize.chop(stream)
def include_stub(state, position, name, local): if local: includes = [os.path.dirname(state.include_chain[-1])] + local_includes else: includes = global_includes filename = name for dirname in includes: path = os.path.join(dirname, name) if os.path.exists(path): filename = path break else: return [] # incorrect, but lets lol later. if filename in state.include_chain: print "{1}: {0}: cyclic include: ".format(*position) + filename return [] # incorrect again with open(filename, 'r') as fd: contents = fd.read() stream = CharacterStream(trigraphs.translate(contents), 1, filename) return chomp(state.fork(stream, filename))
def main(): env = {} filename = "/usr/include/stdio.h" with open(filename, 'r') as fd: contents = fd.read() stream = CharacterStream(trigraphs.translate(contents), 1, filename) state = PreprocessorState(stream, env, include_chain=(filename,), include=include_stub) try: position = (1, filename) for token in chomp(state): if position_of(token) != position: position = advance_position(position, position_of(token)) sys.stdout.write(value_of(token) + ' ') sys.stdout.write('\n') except AssertionError as ass: traceback.print_exc() for x in range(5): print tokenize.chop(stream)
def include_stub(state, position, name, local): if local: includes = [os.path.dirname(state.include_chain[-1]) ] + local_includes else: includes = global_includes filename = name for dirname in includes: path = os.path.join(dirname, name) if os.path.exists(path): filename = path break else: return [] # incorrect, but lets lol later. if filename in state.include_chain: print "{1}: {0}: cyclic include: ".format(*position) + filename return [] # incorrect again with open(filename, 'r') as fd: contents = fd.read() stream = CharacterStream(trigraphs.translate(contents), 1, filename) return chomp(state.fork(stream, filename))
exponents = set(["e+", "e-", "E+", "E-", "p+", "p-", "P+", "P-"]) punctuators = set([ "!", "#", "$", "%", "&", "(", ")", "*", "+", ",", "-", ".", "/", ":", ";", "<", "=", ">", "?", "[", "\\", "]", "^", "_", "{", "|", "}", "~", ]) long_punctuators = set([ "<=", ">=", "!=", "&&", "||", "++", "--", "==", "<<", ">>", "+=" "-=", "*=", "/=", "%=", "&=", "^=", "|=", "->", "..", "##", "...", "<<=", ">>=" ]) digraphs = {"<%":"{", "&>":"}", "<:":"[", ":>":"]", "%:":"#", "%:%:":"##"} # This is actually only used here for testing. def chop_chop(stream): stream.skip_spaces() while stream.character is not "": yield chop(stream) if __name__=='__main__': from character_stream import CharacterStream import trigraphs test = "#What ??=will c0me ??/\n0ut of/***sho*sho**/this line?//lollipops?" stream = CharacterStream(trigraphs.translate(test)) for token_ in chop_chop(stream): print token_
long_punctuators = set([ "<=", ">=", "!=", "&&", "||", "++", "--", "==", "<<", ">>", "+=" "-=", "*=", "/=", "%=", "&=", "^=", "|=", "->", "..", "##", "...", "<<=", ">>=" ]) digraphs = { "<%": "{", "&>": "}", "<:": "[", ":>": "]", "%:": "#", "%:%:": "##" } # This is actually only used here for testing. def chop_chop(stream): stream.skip_spaces() while stream.character is not "": yield chop(stream) if __name__ == '__main__': from character_stream import CharacterStream import trigraphs test = "#What ??=will c0me ??/\n0ut of/***sho*sho**/this line?//lollipops?" stream = CharacterStream(trigraphs.translate(test)) for token_ in chop_chop(stream): print token_