def Cli():
ver = sys.version_info
# Version 3.2 required for argparse
if ver.major < 3 or (ver.major == 3 and ver.minor < 2):
print("Python 3.2+ required. %d.%d.%d installed" %(ver.major, ver.minor, ver.micro))
sys.exit(-1)
parser = argparse.ArgumentParser(description = 'cAST: C Preprocessor and Parser')
commands = dict()
subparsers = parser.add_subparsers(help='Available actions', dest='command')
commands['pp'] = subparsers.add_parser('pp', help='Preprocess.')
commands['pptok'] = subparsers.add_parser('pptok', help='Tokenize C preprocessor.')
commands['ppast'] = subparsers.add_parser('ppast', help='Parse C preprocessor.')
commands['ctok'] = subparsers.add_parser('ctok', help='Preprocess and tokenize C code.')
commands['cparse'] = subparsers.add_parser('cparse', help='Parse C code')
commands['ast'] = subparsers.add_parser('ast', help='Parse C code and transform parse tree into an AST')
parser.add_argument('source_file',
metavar = 'SOURCE_FILE',
nargs = 1,
help = 'C Source File')
parser.add_argument('-d', '--debug',
action='store_true',
help = 'Writes debug information')
parser.add_argument('--skip-includes',
action='store_true',
help = 'Don\'t process #include directives')
parser.add_argument('-e', '--encoding',
help = 'File encoding')
parser.add_argument('-I', '--include-path',
default = '',
help = "A path containing the list of directories separated by colons.")
parser.add_argument('-c', '--color',
action='store_true',
help = "Colorize output to stdout.")
parser.add_argument('--highlight',
action='store_true',
help = "Colorize tokens belonging to this AST node.")
cli = parser.parse_args()
logger = LoggerFactory().initialize(cli.debug)
#logger.debug('CLI Parameters: {}'.format(cli))
if not os.path.isfile( cli.source_file[0] ) and \
not os.path.islink( cli.source_file[0] ):
sys.stderr.write("Error: Source file does not exist\n")
sys.exit(-1)
if not len(cli.source_file) or not cli.source_file[0]:
cSourcePath = open('/dev/stdin')
else:
cSourcePath = cli.source_file[0]
try:
cSourceFp = open(cSourcePath, encoding='utf-8')
except UnicodeDecodeError:
cSourceFp = open(cSourcePath, encoding='iso-8859-1')
cSourceCode = SourceCode(cSourcePath, cSourceFp)
target = subprocess.check_output(["gcc", "-dumpmachine"]).decode('ascii').strip()
include_path_global = ['/usr/include', '/usr/local/include', 'usr/' + target + '/include']
include_path_global.extend( list(filter(lambda x: x, cli.include_path.split(':'))) )
include_path_local = [os.path.dirname(os.path.abspath(cSourcePath))]
cPPFactory = PreProcessorFactory()
#cPP = cPPFactory.create( include_path_global, include_path_local, skipIncludes=cli.skip_includes )
cPP = cPPFactory.createLikeGcc()
if cli.command == 'pp':
try:
(cT, symbols) = cPP.process(cSourceCode)
parser = c_Parser()
parsetree = parser.parse(TokenStream(cT))
ast = parsetree.toAst()
factory = HermesParserFactory()
fp = GrammarFileParser(factory.create())
from pkg_resources import Requirement, resource_filename
filename = resource_filename(__name__, '../grammars/c.zgr')
grammar = fp.parse( 'c', open(filename) )
print(cT.toString(parsetree=parsetree, grammar=grammar, ast=ast, highlight=cli.highlight))
except Exception as e:
print(e, '\n', e.tracer)
sys.exit(-1)
if cli.command == 'pptok':
for token in ppLexer(cSourceCode):
print(token.toString())
if cli.command == 'ppast':
try:
cPPL = TokenStream(ppLexer(cSourceCode))
parser = pp_Parser()
parsetree = parser.parse(cPPL)
ast = parsetree.toAst()
print(AstPrettyPrintable(ast, color=cli.color))
except Exception as e:
print(e, '\n', e.tracer)
sys.exit(-1)
if cli.command == 'ctok':
try:
cT, symbols = cPP.process( cSourceCode )
for token in cT:
print(token.toString())
except Exception as e:
print(e, '\n', e.tracer)
sys.exit(-1)
if cli.command == 'cparse':
try:
cT, symbols = cPP.process( cSourceCode )
parsetree = c_Parser().parse(TokenStream(cT))
print(ParseTreePrettyPrintable(parsetree, color=cli.color))
except Exception as e:
print(e, '\n', e.tracer)
sys.exit(-1)
if cli.command == 'ast':
try:
cT, symbols = cPP.process( cSourceCode )
parser = c_Parser()
parsetree = parser.parse(TokenStream(cT))
ast = parsetree.toAst()
print(AstPrettyPrintable(ast, color=cli.color))
except Exception as e:
print(e, '\n', e.tracer)
sys.exit(-1)
import sys
try:
from hermes.GrammarCodeGenerator import TemplateWriter
from hermes.GrammarCodeGenerator import FactoryFactory as TemplateFactoryFactory
from hermes.GrammarFileParser import GrammarFileParser, HermesParserFactory
except:
sys.stderr.write('Hermes is not installed. Cannot bootstrap.\n')
sys.exit(-1)
grammars = [
('pp', 'grammars/pp.zgr'),
('c', 'grammars/c.zgr')
]
cGrammars = []
for name, grammarFile in grammars:
print('generating parser for grammar %s' % (grammarFile))
grammarFileParser = GrammarFileParser(HermesParserFactory().create())
cGrammars.append( grammarFileParser.parse( name, open(grammarFile) ) )
templateFactory = TemplateFactoryFactory().create('python')
templateWriter = TemplateWriter(templateFactory)
templateWriter.write(cGrammars, 'cast', addMain=False)
import setup
def Cli():
ver = sys.version_info
# Version 3.2 required for argparse
if ver.major < 3 or (ver.major == 3 and ver.minor < 2):
print("Python 3.2+ required. %d.%d.%d installed" %(ver.major, ver.minor, ver.micro))
sys.exit(-1)
parser = argparse.ArgumentParser(
description = 'Hermes Parser Generator',
epilog = '(c) 2011-2012 Scott Frazer')
parser.add_argument('action',
choices = ['analyze', 'generate'],
help = 'Parser Generator Actions')
parser.add_argument('grammar',
metavar = 'GRAMMAR',
nargs = '+',
help = 'Grammar file')
parser.add_argument('--version',
action='version',
version=str(pkg_resources.get_distribution('hermes-parser')))
parser.add_argument('-D', '--debug',
required = False,
action='store_true',
help = 'Open the floodgates')
parser.add_argument('-d', '--directory',
required=False,
default='.',
help='Directory to write generated code to')
parser.add_argument('-l', '--language',
required = False,
default='python',
choices=['c', 'java', 'python'],
help = 'Language for generated parser')
parser.add_argument('--java-package',
required = False,
help = 'If generating Java code, this is the package.')
parser.add_argument('-c', '--color',
required = False,
action = 'store_true',
help = 'Prints things in color! For the colorblind, this is a no-op.')
parser.add_argument('-m', '--add-main',
required = False,
action = 'store_true',
help = 'If this is specified, a main() function will be generated in the source code.')
cli = parser.parse_args()
logger = LoggerFactory().initialize(cli.debug)
logger.debug('CLI Parameters: %s' % (cli))
factory = HermesParserFactory()
fp = GrammarFileParser(factory.create())
grammars = []
for grammar in cli.grammar:
if not os.path.isfile( grammar ):
sys.stderr.write("Error: Grammar file doesn't exist\n")
sys.exit(-1)
name = os.path.basename(grammar)
if not name.endswith('.zgr'):
sys.stderr.write("Error: Grammar file must have .zgr extension\n")
sys.exit(-1)
name = name[:-4]
grammars.append( fp.parse(name, open(grammar)) )
if cli.action == 'analyze':
if cli.color:
theme = TerminalColorTheme()
else:
theme = TerminalDefaultTheme()
for grammar in grammars:
analyzer = GrammarAnalyzer(grammar)
analyzer.analyze( theme=theme )
if cli.action == 'generate':
cli.directory = os.path.abspath(os.path.expanduser(cli.directory))
if not os.path.isdir( cli.directory ):
sys.stderr.write("Error: Directory doesn't exist\n")
sys.exit(-1)
elif not os.access(cli.directory, os.W_OK):
sys.stderr.write("Error: Directory not writable\n")
sys.exit(-1)
templateFactory = TemplateFactoryFactory().create(outputLanguage=cli.language.lower())
templateWriter = TemplateWriter(templateFactory)
templateWriter.write(grammars, cli.directory, addMain=cli.add_main, javaPackage=cli.java_package)
def load_tests(loader, tests, pattern):
grammarTestsDirectory = os.path.join(directory, 'grammar')
parsingTestsDirectory = os.path.join(directory, 'parsing')
suite = unittest.TestSuite()
jsonifySets = lambda arg:'{\n%s\n}' % (',\n'.join([' "%s": [%s]' % (nt, ', '.join(['"'+z+'"' for z in theSet])) for nt, theSet in arg.items()]))
for parsingTest in os.listdir(parsingTestsDirectory):
try:
int(parsingTest)
except ValueError:
continue
testDirectory = os.path.join(parsingTestsDirectory, parsingTest)
grammarFile = os.path.join(testDirectory, 'grammar.zgr')
tokensFile = os.path.join(testDirectory, 'tokens')
grammarParser = GrammarFileParser(HermesParserFactory().create())
grammar = grammarParser.parse( 'grammar', open(grammarFile) )
path = os.path.join(testDirectory, 'parsetree')
if os.path.exists(path):
expectedParseTree = open(path).read().strip()
if len(expectedParseTree):
suite.addTest(HermesPythonParseTreeTest(testDirectory, grammar, expectedParseTree))
suite.addTest(HermesCParseTreeTest(testDirectory, grammar, expectedParseTree))
suite.addTest(HermesJavaParseTreeTest(testDirectory, grammar, expectedParseTree))
else:
fp = open(path, 'w')
fp.write(getParseTree(grammar, testDirectory))
fp.close()
print('generated %s' % (path))
path = os.path.join(testDirectory, 'ast')
if os.path.exists(path):
expectedAst = open(path).read().strip()
suite.addTest(HermesPythonAbstractSyntaxTreeTest(testDirectory, grammar, expectedAst))
suite.addTest(HermesCAbstractSyntaxTreeTest(testDirectory, grammar, expectedAst))
suite.addTest(HermesJavaAbstractSyntaxTreeTest(testDirectory, grammar, expectedAst))
else:
fp = open(path, 'w')
fp.write(getAst(grammar, testDirectory))
fp.close()
print('generated %s' % (path))
for grammarTest in os.listdir(grammarTestsDirectory):
try:
int(grammarTest)
except ValueError:
continue
testDirectory = os.path.join(grammarTestsDirectory, grammarTest)
grammarParser = GrammarFileParser(HermesParserFactory().create())
grammar = grammarParser.parse( 'grammar', open(os.path.join(testDirectory, 'grammar.zgr')) )
grammarFirst = dict()
for k,v in grammar.first.items():
if isinstance(k, NonTerminal):
grammarFirst[k.string] = set(map(lambda x: x.string, v))
grammarFollow = dict()
for k,v in grammar.follow.items():
if isinstance(k, NonTerminal):
grammarFollow[k.string] = set(map(lambda x: x.string, v))
path = os.path.join(testDirectory, 'conflicts.json')
if os.path.exists(path):
contents = open(path).read()
if len(contents):
expected = json.loads(contents)
for k,v in expected.items():
suite.addTest(HermesConflictTest(testDirectory, k, contents, '\n'.join([x.toJson() for x in grammar.conflicts])))
else:
if len(grammar.conflicts):
fp = open(path, 'w')
fp.write('\n'.join([x.toJson() for x in grammar.conflicts]))
fp.close()
else:
fp = open(path, 'w')
fp.close()
print('generated %s/conflicts.json (%d conflicts)' % (path, len(grammar.conflicts)))
path = os.path.join(testDirectory, 'first.json')
if os.path.exists(path):
contents = open(path).read()
if len(contents):
expected = json.loads(contents)
for k,v in expected.items():
suite.addTest(HermesFirstSetTest(testDirectory, k, set(expected[k]), grammarFirst[k]))
else:
if len(grammar.conflicts):
fp = open(path, 'w')
fp.close()
print('generated %s/first.json (empty file because of conflicts)' % (path))
else:
for k,v in grammarFirst.items():
grammarFirst[k] = list(v)
fp = open(path, 'w')
fp.write(jsonifySets(grammarFirst))
fp.close()
print('generated %s/first.json' % (path))
path = os.path.join(testDirectory, 'follow.json')
if os.path.exists(path):
contents = open(path).read()
if len(contents):
expected = json.loads(contents)
for k,v in expected.items():
suite.addTest(HermesFollowSetTest(testDirectory, k, set(expected[k]), grammarFollow[k]))
else:
if len(grammar.conflicts):
fp = open(path, 'w')
fp.close()
print('generated %s/follow.json (empty file because of conflicts)' % (path))
else:
for k,v in grammarFollow.items():
grammarFollow[k] = list(v)
fp = open(path, 'w')
fp.write(jsonifySets(grammarFollow))
fp.close()
print('generated %s/follow.json' % (path))
return suite
def Cli():
ver = sys.version_info
# Version 3.2 required for argparse
if ver.major < 3 or (ver.major == 3 and ver.minor < 2):
print("Python 3.2+ required. %d.%d.%d installed" %
(ver.major, ver.minor, ver.micro))
sys.exit(-1)
parser = argparse.ArgumentParser(description='Hermes Parser Generator',
epilog='(c) 2011-2012 Scott Frazer')
parser.add_argument('action',
choices=['analyze', 'generate'],
help='Parser Generator Actions')
parser.add_argument('grammar',
metavar='GRAMMAR',
nargs='+',
help='Grammar file')
parser.add_argument('--version',
action='version',
version=str(
pkg_resources.get_distribution('hermes-parser')))
parser.add_argument('-D',
'--debug',
required=False,
action='store_true',
help='Open the floodgates')
parser.add_argument('-d',
'--directory',
required=False,
default='.',
help='Directory to write generated code to')
parser.add_argument('-l',
'--language',
required=False,
default='python',
choices=['c', 'java', 'python'],
help='Language for generated parser')
parser.add_argument('--java-package',
required=False,
help='If generating Java code, this is the package.')
parser.add_argument(
'-c',
'--color',
required=False,
action='store_true',
help='Prints things in color! For the colorblind, this is a no-op.')
parser.add_argument(
'-m',
'--add-main',
required=False,
action='store_true',
help=
'If this is specified, a main() function will be generated in the source code.'
)
cli = parser.parse_args()
logger = LoggerFactory().initialize(cli.debug)
logger.debug('CLI Parameters: %s' % (cli))
factory = HermesParserFactory()
fp = GrammarFileParser(factory.create())
grammars = []
for grammar in cli.grammar:
if not os.path.isfile(grammar):
sys.stderr.write("Error: Grammar file doesn't exist\n")
sys.exit(-1)
name = os.path.basename(grammar)
if not name.endswith('.zgr'):
sys.stderr.write("Error: Grammar file must have .zgr extension\n")
sys.exit(-1)
name = name[:-4]
grammars.append(fp.parse(name, open(grammar)))
if cli.action == 'analyze':
if cli.color:
theme = TerminalColorTheme()
else:
theme = TerminalDefaultTheme()
for grammar in grammars:
analyzer = GrammarAnalyzer(grammar)
analyzer.analyze(theme=theme)
if cli.action == 'generate':
cli.directory = os.path.abspath(os.path.expanduser(cli.directory))
if not os.path.isdir(cli.directory):
sys.stderr.write("Error: Directory doesn't exist\n")
sys.exit(-1)
elif not os.access(cli.directory, os.W_OK):
sys.stderr.write("Error: Directory not writable\n")
sys.exit(-1)
templateFactory = TemplateFactoryFactory().create(
outputLanguage=cli.language.lower())
templateWriter = TemplateWriter(templateFactory)
templateWriter.write(grammars,
cli.directory,
addMain=cli.add_main,
javaPackage=cli.java_package)
