def setUp(self): self.parser = HermesParserFactory().getExprRuleParser()
def Cli(): ver = sys.version_info # Version 3.2 required for argparse if ver.major < 3 or (ver.major == 3 and ver.minor < 2): print("Python 3.2+ required. %d.%d.%d installed" %(ver.major, ver.minor, ver.micro)) sys.exit(-1) parser = argparse.ArgumentParser(description = 'cAST: C Preprocessor and Parser') commands = dict() subparsers = parser.add_subparsers(help='Available actions', dest='command') commands['pp'] = subparsers.add_parser('pp', help='Preprocess.') commands['pptok'] = subparsers.add_parser('pptok', help='Tokenize C preprocessor.') commands['ppast'] = subparsers.add_parser('ppast', help='Parse C preprocessor.') commands['ctok'] = subparsers.add_parser('ctok', help='Preprocess and tokenize C code.') commands['cparse'] = subparsers.add_parser('cparse', help='Parse C code') commands['ast'] = subparsers.add_parser('ast', help='Parse C code and transform parse tree into an AST') parser.add_argument('source_file', metavar = 'SOURCE_FILE', nargs = 1, help = 'C Source File') parser.add_argument('-d', '--debug', action='store_true', help = 'Writes debug information') parser.add_argument('--skip-includes', action='store_true', help = 'Don\'t process #include directives') parser.add_argument('-e', '--encoding', help = 'File encoding') parser.add_argument('-I', '--include-path', default = '', help = "A path containing the list of directories separated by colons.") parser.add_argument('-c', '--color', action='store_true', help = "Colorize output to stdout.") parser.add_argument('--highlight', action='store_true', help = "Colorize tokens belonging to this AST node.") cli = parser.parse_args() logger = LoggerFactory().initialize(cli.debug) #logger.debug('CLI Parameters: {}'.format(cli)) if not os.path.isfile( cli.source_file[0] ) and \ not os.path.islink( cli.source_file[0] ): sys.stderr.write("Error: Source file does not exist\n") sys.exit(-1) if not len(cli.source_file) or not cli.source_file[0]: cSourcePath = open('/dev/stdin') else: cSourcePath = cli.source_file[0] try: cSourceFp = open(cSourcePath, encoding='utf-8') except UnicodeDecodeError: cSourceFp = open(cSourcePath, encoding='iso-8859-1') cSourceCode = SourceCode(cSourcePath, cSourceFp) target = subprocess.check_output(["gcc", "-dumpmachine"]).decode('ascii').strip() include_path_global = ['/usr/include', '/usr/local/include', 'usr/' + target + '/include'] include_path_global.extend( list(filter(lambda x: x, cli.include_path.split(':'))) ) include_path_local = [os.path.dirname(os.path.abspath(cSourcePath))] cPPFactory = PreProcessorFactory() #cPP = cPPFactory.create( include_path_global, include_path_local, skipIncludes=cli.skip_includes ) cPP = cPPFactory.createLikeGcc() if cli.command == 'pp': try: (cT, symbols) = cPP.process(cSourceCode) parser = c_Parser() parsetree = parser.parse(TokenStream(cT)) ast = parsetree.toAst() factory = HermesParserFactory() fp = GrammarFileParser(factory.create()) from pkg_resources import Requirement, resource_filename filename = resource_filename(__name__, '../grammars/c.zgr') grammar = fp.parse( 'c', open(filename) ) print(cT.toString(parsetree=parsetree, grammar=grammar, ast=ast, highlight=cli.highlight)) except Exception as e: print(e, '\n', e.tracer) sys.exit(-1) if cli.command == 'pptok': for token in ppLexer(cSourceCode): print(token.toString()) if cli.command == 'ppast': try: cPPL = TokenStream(ppLexer(cSourceCode)) parser = pp_Parser() parsetree = parser.parse(cPPL) ast = parsetree.toAst() print(AstPrettyPrintable(ast, color=cli.color)) except Exception as e: print(e, '\n', e.tracer) sys.exit(-1) if cli.command == 'ctok': try: cT, symbols = cPP.process( cSourceCode ) for token in cT: print(token.toString()) except Exception as e: print(e, '\n', e.tracer) sys.exit(-1) if cli.command == 'cparse': try: cT, symbols = cPP.process( cSourceCode ) parsetree = c_Parser().parse(TokenStream(cT)) print(ParseTreePrettyPrintable(parsetree, color=cli.color)) except Exception as e: print(e, '\n', e.tracer) sys.exit(-1) if cli.command == 'ast': try: cT, symbols = cPP.process( cSourceCode ) parser = c_Parser() parsetree = parser.parse(TokenStream(cT)) ast = parsetree.toAst() print(AstPrettyPrintable(ast, color=cli.color)) except Exception as e: print(e, '\n', e.tracer) sys.exit(-1)
class ExprRuleParserTest(unittest.TestCase): def setUp(self): self.parser = HermesParserFactory().getExprRuleParser() def assertProduction(self, expected, production): self.assertEqual(expected, list(map(lambda x: (x.string, type(x)), production.morphemes))) def assertAstSpecification(self, name, expected, ast): self.assertEqual(AstSpecification, type(ast)) self.assertEqual(name, ast.name) self.assertEqual(expected, list(map(lambda x: (str(x[0]), str(x[1])), ast.parameters.items()))) def assertAstTranslation(self, index, ast): self.assertEqual(AstTranslation, type(ast)) self.assertEqual(index, ast.idx) def test_one(self): rules = self.parser.parse("expr := {'a' + b -> NudAst( x=$1 )} + {'c' + d} -> Ast( y=$0 )") self.assertProduction([("a", Terminal), ("b", NonTerminal)], rules[0].nudProduction) self.assertProduction([("c", Terminal), ("d", NonTerminal)], rules[0].ledProduction) self.assertAstSpecification("NudAst", [("x", "1")], rules[0].nudAst) self.assertAstSpecification("Ast", [("y", "0")], rules[0].ast) def test_two(self): rules = self.parser.parse("expr := {'a' + 'b'} + {'c' + d}") self.assertProduction([("a", Terminal), ("b", Terminal)], rules[0].nudProduction) self.assertProduction([("c", Terminal), ("d", NonTerminal)], rules[0].ledProduction) self.assertAstTranslation(0, rules[0].nudAst) self.assertAstTranslation(0, rules[0].ast) def test_three(self): rules = self.parser.parse("expr := {'a' + 'b' -> $1} + {'c' + d} -> $$") self.assertProduction([("a", Terminal), ("b", Terminal)], rules[0].nudProduction) self.assertProduction([("c", Terminal), ("d", NonTerminal)], rules[0].ledProduction) self.assertAstTranslation(1, rules[0].nudAst) self.assertAstTranslation("$", rules[0].ast) def test_four(self): rules = self.parser.parse("expr := {'a' + 'b' -> NudAst( x=$0 )} + {'c' + d} -> $$") self.assertProduction([("a", Terminal), ("b", Terminal)], rules[0].nudProduction) self.assertProduction([("c", Terminal), ("d", NonTerminal)], rules[0].ledProduction) self.assertAstSpecification("NudAst", [("x", "0")], rules[0].nudAst) self.assertAstTranslation("$", rules[0].ast) def test_four(self): rules = self.parser.parse("expr := {'a' + 'b' -> NudAst()} + {'c' + d} -> Ast()") self.assertProduction([("a", Terminal), ("b", Terminal)], rules[0].nudProduction) self.assertProduction([("c", Terminal), ("d", NonTerminal)], rules[0].ledProduction) self.assertAstSpecification("NudAst", [], rules[0].nudAst) self.assertAstSpecification("Ast", [], rules[0].ast) def test_five(self): rules = self.parser.parse("expr := {'a' + 'b' -> NudAst()}") self.assertProduction([("a", Terminal), ("b", Terminal)], rules[0].nudProduction) self.assertProduction([], rules[0].ledProduction) self.assertAstSpecification("NudAst", [], rules[0].nudAst) self.assertAstTranslation(0, rules[0].ast) def test_six(self): rules = self.parser.parse("expr := {'a'}") self.assertProduction([("a", Terminal)], rules[0].nudProduction) self.assertProduction([], rules[0].ledProduction) self.assertAstTranslation(0, rules[0].nudAst) self.assertAstTranslation(0, rules[0].ast) def test_seven(self): rules = self.parser.parse("expr := {u}") self.assertProduction([("u", NonTerminal)], rules[0].nudProduction) self.assertProduction([], rules[0].ledProduction) self.assertAstTranslation(0, rules[0].nudAst) self.assertAstTranslation(0, rules[0].ast) def test_eight(self): self.assertRaises(Exception, self.parser.parse, "expr := ") def test_nine(self): rules = self.parser.parse("expr := expr + 'plus' + expr") self.assertProduction([], rules[0].nudProduction) self.assertProduction([("plus", Terminal), ("expr", NonTerminal)], rules[0].ledProduction) self.assertAstTranslation(0, rules[0].nudAst) self.assertAstTranslation(0, rules[0].ast) def test_ten(self): rules = self.parser.parse("expr := 'minus' + expr") self.assertProduction([("minus", Terminal), ("expr", NonTerminal)], rules[0].nudProduction) self.assertProduction([], rules[0].ledProduction) self.assertAstTranslation(0, rules[0].nudAst) self.assertAstTranslation(0, rules[0].ast)
class ExprRuleParserTest(unittest.TestCase): def setUp(self): self.parser = HermesParserFactory().getExprRuleParser() def assertProduction(self, expected, production): self.assertEqual( expected, list(map(lambda x: (x.string, type(x)), production.morphemes))) def assertAstSpecification(self, name, expected, ast): self.assertEqual(AstSpecification, type(ast)) self.assertEqual(name, ast.name) self.assertEqual( expected, list(map(lambda x: (str(x[0]), str(x[1])), ast.parameters.items()))) def assertAstTranslation(self, index, ast): self.assertEqual(AstTranslation, type(ast)) self.assertEqual(index, ast.idx) def test_one(self): rules = self.parser.parse( "expr := {'a' + b -> NudAst( x=$1 )} + {'c' + d} -> Ast( y=$0 )") self.assertProduction([('a', Terminal), ('b', NonTerminal)], rules[0].nudProduction) self.assertProduction([('c', Terminal), ('d', NonTerminal)], rules[0].ledProduction) self.assertAstSpecification('NudAst', [('x', '1')], rules[0].nudAst) self.assertAstSpecification('Ast', [('y', '0')], rules[0].ast) def test_two(self): rules = self.parser.parse("expr := {'a' + 'b'} + {'c' + d}") self.assertProduction([('a', Terminal), ('b', Terminal)], rules[0].nudProduction) self.assertProduction([('c', Terminal), ('d', NonTerminal)], rules[0].ledProduction) self.assertAstTranslation(0, rules[0].nudAst) self.assertAstTranslation(0, rules[0].ast) def test_three(self): rules = self.parser.parse( "expr := {'a' + 'b' -> $1} + {'c' + d} -> $$") self.assertProduction([('a', Terminal), ('b', Terminal)], rules[0].nudProduction) self.assertProduction([('c', Terminal), ('d', NonTerminal)], rules[0].ledProduction) self.assertAstTranslation(1, rules[0].nudAst) self.assertAstTranslation('$', rules[0].ast) def test_four(self): rules = self.parser.parse( "expr := {'a' + 'b' -> NudAst( x=$0 )} + {'c' + d} -> $$") self.assertProduction([('a', Terminal), ('b', Terminal)], rules[0].nudProduction) self.assertProduction([('c', Terminal), ('d', NonTerminal)], rules[0].ledProduction) self.assertAstSpecification('NudAst', [('x', '0')], rules[0].nudAst) self.assertAstTranslation('$', rules[0].ast) def test_four(self): rules = self.parser.parse( "expr := {'a' + 'b' -> NudAst()} + {'c' + d} -> Ast()") self.assertProduction([('a', Terminal), ('b', Terminal)], rules[0].nudProduction) self.assertProduction([('c', Terminal), ('d', NonTerminal)], rules[0].ledProduction) self.assertAstSpecification('NudAst', [], rules[0].nudAst) self.assertAstSpecification('Ast', [], rules[0].ast) def test_five(self): rules = self.parser.parse("expr := {'a' + 'b' -> NudAst()}") self.assertProduction([('a', Terminal), ('b', Terminal)], rules[0].nudProduction) self.assertProduction([], rules[0].ledProduction) self.assertAstSpecification('NudAst', [], rules[0].nudAst) self.assertAstTranslation(0, rules[0].ast) def test_six(self): rules = self.parser.parse("expr := {'a'}") self.assertProduction([('a', Terminal)], rules[0].nudProduction) self.assertProduction([], rules[0].ledProduction) self.assertAstTranslation(0, rules[0].nudAst) self.assertAstTranslation(0, rules[0].ast) def test_seven(self): rules = self.parser.parse("expr := {u}") self.assertProduction([('u', NonTerminal)], rules[0].nudProduction) self.assertProduction([], rules[0].ledProduction) self.assertAstTranslation(0, rules[0].nudAst) self.assertAstTranslation(0, rules[0].ast) def test_eight(self): self.assertRaises(Exception, self.parser.parse, "expr := ") def test_nine(self): rules = self.parser.parse("expr := expr + 'plus' + expr") self.assertProduction([], rules[0].nudProduction) self.assertProduction([('plus', Terminal), ('expr', NonTerminal)], rules[0].ledProduction) self.assertAstTranslation(0, rules[0].nudAst) self.assertAstTranslation(0, rules[0].ast) def test_ten(self): rules = self.parser.parse("expr := 'minus' + expr") self.assertProduction([('minus', Terminal), ('expr', NonTerminal)], rules[0].nudProduction) self.assertProduction([], rules[0].ledProduction) self.assertAstTranslation(0, rules[0].nudAst) self.assertAstTranslation(0, rules[0].ast)
def load_tests(loader, tests, pattern): grammarTestsDirectory = os.path.join(directory, 'grammar') parsingTestsDirectory = os.path.join(directory, 'parsing') suite = unittest.TestSuite() jsonifySets = lambda arg:'{\n%s\n}' % (',\n'.join([' "%s": [%s]' % (nt, ', '.join(['"'+z+'"' for z in theSet])) for nt, theSet in arg.items()])) for parsingTest in os.listdir(parsingTestsDirectory): try: int(parsingTest) except ValueError: continue testDirectory = os.path.join(parsingTestsDirectory, parsingTest) grammarFile = os.path.join(testDirectory, 'grammar.zgr') tokensFile = os.path.join(testDirectory, 'tokens') grammarParser = GrammarFileParser(HermesParserFactory().create()) grammar = grammarParser.parse( 'grammar', open(grammarFile) ) path = os.path.join(testDirectory, 'parsetree') if os.path.exists(path): expectedParseTree = open(path).read().strip() if len(expectedParseTree): suite.addTest(HermesPythonParseTreeTest(testDirectory, grammar, expectedParseTree)) suite.addTest(HermesCParseTreeTest(testDirectory, grammar, expectedParseTree)) suite.addTest(HermesJavaParseTreeTest(testDirectory, grammar, expectedParseTree)) else: fp = open(path, 'w') fp.write(getParseTree(grammar, testDirectory)) fp.close() print('generated %s' % (path)) path = os.path.join(testDirectory, 'ast') if os.path.exists(path): expectedAst = open(path).read().strip() suite.addTest(HermesPythonAbstractSyntaxTreeTest(testDirectory, grammar, expectedAst)) suite.addTest(HermesCAbstractSyntaxTreeTest(testDirectory, grammar, expectedAst)) suite.addTest(HermesJavaAbstractSyntaxTreeTest(testDirectory, grammar, expectedAst)) else: fp = open(path, 'w') fp.write(getAst(grammar, testDirectory)) fp.close() print('generated %s' % (path)) for grammarTest in os.listdir(grammarTestsDirectory): try: int(grammarTest) except ValueError: continue testDirectory = os.path.join(grammarTestsDirectory, grammarTest) grammarParser = GrammarFileParser(HermesParserFactory().create()) grammar = grammarParser.parse( 'grammar', open(os.path.join(testDirectory, 'grammar.zgr')) ) grammarFirst = dict() for k,v in grammar.first.items(): if isinstance(k, NonTerminal): grammarFirst[k.string] = set(map(lambda x: x.string, v)) grammarFollow = dict() for k,v in grammar.follow.items(): if isinstance(k, NonTerminal): grammarFollow[k.string] = set(map(lambda x: x.string, v)) path = os.path.join(testDirectory, 'conflicts.json') if os.path.exists(path): contents = open(path).read() if len(contents): expected = json.loads(contents) for k,v in expected.items(): suite.addTest(HermesConflictTest(testDirectory, k, contents, '\n'.join([x.toJson() for x in grammar.conflicts]))) else: if len(grammar.conflicts): fp = open(path, 'w') fp.write('\n'.join([x.toJson() for x in grammar.conflicts])) fp.close() else: fp = open(path, 'w') fp.close() print('generated %s/conflicts.json (%d conflicts)' % (path, len(grammar.conflicts))) path = os.path.join(testDirectory, 'first.json') if os.path.exists(path): contents = open(path).read() if len(contents): expected = json.loads(contents) for k,v in expected.items(): suite.addTest(HermesFirstSetTest(testDirectory, k, set(expected[k]), grammarFirst[k])) else: if len(grammar.conflicts): fp = open(path, 'w') fp.close() print('generated %s/first.json (empty file because of conflicts)' % (path)) else: for k,v in grammarFirst.items(): grammarFirst[k] = list(v) fp = open(path, 'w') fp.write(jsonifySets(grammarFirst)) fp.close() print('generated %s/first.json' % (path)) path = os.path.join(testDirectory, 'follow.json') if os.path.exists(path): contents = open(path).read() if len(contents): expected = json.loads(contents) for k,v in expected.items(): suite.addTest(HermesFollowSetTest(testDirectory, k, set(expected[k]), grammarFollow[k])) else: if len(grammar.conflicts): fp = open(path, 'w') fp.close() print('generated %s/follow.json (empty file because of conflicts)' % (path)) else: for k,v in grammarFollow.items(): grammarFollow[k] = list(v) fp = open(path, 'w') fp.write(jsonifySets(grammarFollow)) fp.close() print('generated %s/follow.json' % (path)) return suite
def Cli(): ver = sys.version_info # Version 3.2 required for argparse if ver.major < 3 or (ver.major == 3 and ver.minor < 2): print("Python 3.2+ required. %d.%d.%d installed" %(ver.major, ver.minor, ver.micro)) sys.exit(-1) parser = argparse.ArgumentParser( description = 'Hermes Parser Generator', epilog = '(c) 2011-2012 Scott Frazer') parser.add_argument('action', choices = ['analyze', 'generate'], help = 'Parser Generator Actions') parser.add_argument('grammar', metavar = 'GRAMMAR', nargs = '+', help = 'Grammar file') parser.add_argument('--version', action='version', version=str(pkg_resources.get_distribution('hermes-parser'))) parser.add_argument('-D', '--debug', required = False, action='store_true', help = 'Open the floodgates') parser.add_argument('-d', '--directory', required=False, default='.', help='Directory to write generated code to') parser.add_argument('-l', '--language', required = False, default='python', choices=['c', 'java', 'python'], help = 'Language for generated parser') parser.add_argument('--java-package', required = False, help = 'If generating Java code, this is the package.') parser.add_argument('-c', '--color', required = False, action = 'store_true', help = 'Prints things in color! For the colorblind, this is a no-op.') parser.add_argument('-m', '--add-main', required = False, action = 'store_true', help = 'If this is specified, a main() function will be generated in the source code.') cli = parser.parse_args() logger = LoggerFactory().initialize(cli.debug) logger.debug('CLI Parameters: %s' % (cli)) factory = HermesParserFactory() fp = GrammarFileParser(factory.create()) grammars = [] for grammar in cli.grammar: if not os.path.isfile( grammar ): sys.stderr.write("Error: Grammar file doesn't exist\n") sys.exit(-1) name = os.path.basename(grammar) if not name.endswith('.zgr'): sys.stderr.write("Error: Grammar file must have .zgr extension\n") sys.exit(-1) name = name[:-4] grammars.append( fp.parse(name, open(grammar)) ) if cli.action == 'analyze': if cli.color: theme = TerminalColorTheme() else: theme = TerminalDefaultTheme() for grammar in grammars: analyzer = GrammarAnalyzer(grammar) analyzer.analyze( theme=theme ) if cli.action == 'generate': cli.directory = os.path.abspath(os.path.expanduser(cli.directory)) if not os.path.isdir( cli.directory ): sys.stderr.write("Error: Directory doesn't exist\n") sys.exit(-1) elif not os.access(cli.directory, os.W_OK): sys.stderr.write("Error: Directory not writable\n") sys.exit(-1) templateFactory = TemplateFactoryFactory().create(outputLanguage=cli.language.lower()) templateWriter = TemplateWriter(templateFactory) templateWriter.write(grammars, cli.directory, addMain=cli.add_main, javaPackage=cli.java_package)
def Cli(): ver = sys.version_info # Version 3.2 required for argparse if ver.major < 3 or (ver.major == 3 and ver.minor < 2): print("Python 3.2+ required. %d.%d.%d installed" % (ver.major, ver.minor, ver.micro)) sys.exit(-1) parser = argparse.ArgumentParser(description='Hermes Parser Generator', epilog='(c) 2011-2012 Scott Frazer') parser.add_argument('action', choices=['analyze', 'generate'], help='Parser Generator Actions') parser.add_argument('grammar', metavar='GRAMMAR', nargs='+', help='Grammar file') parser.add_argument('--version', action='version', version=str( pkg_resources.get_distribution('hermes-parser'))) parser.add_argument('-D', '--debug', required=False, action='store_true', help='Open the floodgates') parser.add_argument('-d', '--directory', required=False, default='.', help='Directory to write generated code to') parser.add_argument('-l', '--language', required=False, default='python', choices=['c', 'java', 'python'], help='Language for generated parser') parser.add_argument('--java-package', required=False, help='If generating Java code, this is the package.') parser.add_argument( '-c', '--color', required=False, action='store_true', help='Prints things in color! For the colorblind, this is a no-op.') parser.add_argument( '-m', '--add-main', required=False, action='store_true', help= 'If this is specified, a main() function will be generated in the source code.' ) cli = parser.parse_args() logger = LoggerFactory().initialize(cli.debug) logger.debug('CLI Parameters: %s' % (cli)) factory = HermesParserFactory() fp = GrammarFileParser(factory.create()) grammars = [] for grammar in cli.grammar: if not os.path.isfile(grammar): sys.stderr.write("Error: Grammar file doesn't exist\n") sys.exit(-1) name = os.path.basename(grammar) if not name.endswith('.zgr'): sys.stderr.write("Error: Grammar file must have .zgr extension\n") sys.exit(-1) name = name[:-4] grammars.append(fp.parse(name, open(grammar))) if cli.action == 'analyze': if cli.color: theme = TerminalColorTheme() else: theme = TerminalDefaultTheme() for grammar in grammars: analyzer = GrammarAnalyzer(grammar) analyzer.analyze(theme=theme) if cli.action == 'generate': cli.directory = os.path.abspath(os.path.expanduser(cli.directory)) if not os.path.isdir(cli.directory): sys.stderr.write("Error: Directory doesn't exist\n") sys.exit(-1) elif not os.access(cli.directory, os.W_OK): sys.stderr.write("Error: Directory not writable\n") sys.exit(-1) templateFactory = TemplateFactoryFactory().create( outputLanguage=cli.language.lower()) templateWriter = TemplateWriter(templateFactory) templateWriter.write(grammars, cli.directory, addMain=cli.add_main, javaPackage=cli.java_package)