def test_basic_h(self): import parsing.grammar import parsing.automaton parsing.grammar.in_h = True class TestGlrParser(parsing.Glr): def __init__(self, spec): parsing.Glr.__init__(self, spec) try: from parsing.tests.specs import h spec = parsing.Spec(h, logFile='h.log', skinny=False) parser = TestGlrParser(spec) parser.token_from_class(h.TokenI) parser.token_from_class(h.TokenPlus) parser.token_from_class(h.TokenI) parser.token_from_class(h.TokenStar) parser.token_from_class(h.TokenI) parser.eoi() self.assertEqual(len(parser.start), 1) self.assertEqual(repr(parser.start[0]), '(i + (i * i))') finally: parsing.grammar.in_h = False
def _compile_parsers(build_lib, inplace=False): import parsing import edb.edgeql.parser.grammar.single as edgeql_spec import edb.edgeql.parser.grammar.block as edgeql_spec2 import edb.edgeql.parser.grammar.sdldocument as schema_spec for spec in (edgeql_spec, edgeql_spec2, schema_spec): spec_path = pathlib.Path(spec.__file__).parent subpath = pathlib.Path(str(spec_path)[len(str(ROOT_PATH)) + 1:]) pickle_name = spec.__name__.rpartition('.')[2] + '.pickle' pickle_path = subpath / pickle_name cache = build_lib / pickle_path cache.parent.mkdir(parents=True, exist_ok=True) parsing.Spec(spec, pickleFile=str(cache), verbose=True) if inplace: shutil.copy2(cache, ROOT_PATH / pickle_path)
def test_basic_h(self): class TestGlrParser(parsing.Glr): def __init__(self, spec): parsing.Glr.__init__(self, spec) from parsing.tests.specs import h spec = parsing.Spec(h, skinny=False) parser = TestGlrParser(spec) parser.token(h.TokenI(parser)) parser.token(h.TokenPlus(parser)) parser.token(h.TokenI(parser)) parser.token(h.TokenStar(parser)) parser.token(h.TokenI(parser)) parser.eoi() self.assertEqual(len(parser.start), 1) self.assertEqual(repr(parser.start[0]), '(i + (i * i))')
def _compile_parsers(self): import parsing import edb.lang.edgeql.parser.grammar.single as edgeql_spec import edb.lang.edgeql.parser.grammar.block as edgeql_spec2 import edb.server.pgsql.parser.pgsql as pgsql_spec import edb.lang.schema.parser.grammar.declarations as schema_spec import edb.lang.graphql.parser.grammar.document as graphql_spec base_path = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) for spec in (edgeql_spec, edgeql_spec2, pgsql_spec, schema_spec, graphql_spec): subpath = os.path.dirname(spec.__file__)[len(base_path) + 1:] cache_dir = os.path.join(self.build_lib, subpath) os.makedirs(cache_dir, exist_ok=True) cache = os.path.join(cache_dir, spec.__name__.rpartition('.')[2] + '.pickle') parsing.Spec(spec, pickleFile=cache, verbose=True)
def get_parser_spec(self): cls = self.__class__ try: spec = cls.__dict__['parser_spec'] except KeyError: pass else: if spec is not None: return spec mod = self.get_parser_spec_module() spec = parsing.Spec(mod, pickleFile=self.localpath(mod, "pickle"), skinny=not self.get_debug(), logFile=self.localpath(mod, "log"), verbose=self.get_debug()) self.__class__.parser_spec = spec return spec
def test_basic_d(self): class TestParser(parsing.Glr): def __init__(self, spec): parsing.Glr.__init__(self, spec) from parsing.tests.specs import d spec = parsing.Spec(d, skinny=False) parser = TestParser(spec) parser.token_from_class(d.id) parser.token_from_class(d.star) parser.token_from_class(d.id) parser.token_from_class(d.plus) parser.token_from_class(d.id) parser.token_from_class(d.star) parser.token_from_class(d.id) parser.eoi() self.assertEqual(len(parser.start), 1) self.assertEqual(parser.start[0].val, '[[ID * ID] + [ID * ID]]')
def test_basic_pickle(self): class TestGlrParser(parsing.Glr): def __init__(self, spec): parsing.Glr.__init__(self, spec) from parsing.tests.specs import b spec = parsing.Spec(b, skinny=False) import six.moves.cPickle specPickle = six.moves.cPickle.dumps(spec) spec2 = six.moves.cPickle.loads(specPickle) parser = TestGlrParser(spec2) parser.token_from_class(b.id) parser.token_from_class(b.star) parser.token_from_class(b.id) parser.token_from_class(b.plus) parser.token_from_class(b.id) parser.eoi() self.assertEqual(len(parser.start), 1) self.assertEqual(parser.start[0].val, '[[ID * ID] + ID]')
def _compile_parsers(build_lib, inplace=False): import parsing import edb.lang.edgeql.parser.grammar.single as edgeql_spec import edb.lang.edgeql.parser.grammar.block as edgeql_spec2 import edb.server.pgsql.parser.pgsql as pgsql_spec import edb.lang.schema.parser.grammar.declarations as schema_spec import edb.lang.graphql.parser.grammar.document as graphql_spec base_path = pathlib.Path(__file__).parent.resolve() for spec in (edgeql_spec, edgeql_spec2, pgsql_spec, schema_spec, graphql_spec): spec_path = pathlib.Path(spec.__file__).parent subpath = pathlib.Path(str(spec_path)[len(str(base_path)) + 1:]) pickle_name = spec.__name__.rpartition('.')[2] + '.pickle' pickle_path = subpath / pickle_name cache = build_lib / pickle_path cache.parent.mkdir(parents=True, exist_ok=True) parsing.Spec(spec, pickleFile=str(cache), verbose=True) if inplace: shutil.copy2(cache, base_path / pickle_path)
def test_basic_a(self): class TestParser(parsing.Lr): def __init__(self, spec): parsing.Lr.__init__(self, spec) from parsing.tests.specs import a spec = parsing.Spec(a) parser = TestParser(spec) parser.token_from_class(a.TokenId) parser.token_from_class(a.TokenStar) parser.token_from_class(a.TokenId) parser.token_from_class(a.TokenPlus) parser.token_from_class(a.TokenId) parser.eoi() self.assertEqual(len(parser.start), 1) self.assertEqual(parser.start[0].val, '[[ID * ID] + ID]') parser = TestParser(spec) parser.token_from_class(a.TokenId) parser.token_from_class(a.TokenPlus) parser.token_from_class(a.TokenId) parser.token_from_class(a.TokenStar) parser.token_from_class(a.TokenId) parser.eoi() self.assertEqual(len(parser.start), 1) self.assertEqual(parser.start[0].val, '[ID + [ID * ID]]') parser = TestParser(spec) parser.token_from_class(a.TokenId) parser.token_from_class(a.TokenStar) parser.token_from_class(a.TokenLparen) parser.token_from_class(a.TokenId) parser.token_from_class(a.TokenPlus) parser.token_from_class(a.TokenId) parser.token_from_class(a.TokenRparen) parser.eoi() self.assertEqual(len(parser.start), 1) self.assertEqual(parser.start[0].val, '[ID * ([ID + ID])]')
def test_basic_b(self): class TestParser(parsing.Glr): def __init__(self, spec): parsing.Glr.__init__(self, spec) from parsing.tests.specs import b spec = parsing.Spec(b) parser = TestParser(spec) parser.token_from_class(b.id) parser.token_from_class(b.star) parser.token_from_class(b.id) parser.token_from_class(b.plus) parser.token_from_class(b.id) parser.eoi() self.assertEqual(len(parser.start), 1) self.assertEqual(parser.start[0].val, '[[ID * ID] + ID]') parser = TestParser(spec) parser.token_from_class(b.id) parser.token_from_class(b.plus) parser.token_from_class(b.id) parser.token_from_class(b.star) parser.token_from_class(b.id) parser.eoi() self.assertEqual(len(parser.start), 1) self.assertEqual(parser.start[0].val, '[ID + [ID * ID]]') parser = TestParser(spec) parser.token_from_class(b.id) parser.token_from_class(b.star) parser.token_from_class(b.lparen) parser.token_from_class(b.id) parser.token_from_class(b.plus) parser.token_from_class(b.id) parser.token_from_class(b.rparen) parser.eoi() self.assertEqual(len(parser.start), 1) self.assertEqual(parser.start[0].val, '[ID * ([ID + ID])]')
except: raise SyntaxError("Unrecognized token: %s" % word) # Tell the parser that the end of input has been reached. self.eoi() # =============================================================================== # Main code. # Introspect this module to generate a parser. Enable all the bells and # whistles. adapter = ModuleSpecSource(sys.modules[__name__]) spec = Parsing.Spec(adapter, pickleFile="example1.pickle", skinny=False, logFile="example1.log", graphFile="example1.dot", verbose=True) # example1.log is a human-readable representation of the parser tables. # Suppose that you are trying to figure out what the parser is doing for a # particular input, in this case, "2 * * 3". If you have parsing verbosity # enabled, you will see something like this: # # STACK: <e> # 0 # INPUT: int # --> [shift 1] # STACK: <e> int # 0 1 # INPUT: star # --> [reduce Expr ::= int. [none]]
g = tokenize.generate_tokens(f.readline) l = [] while g: code, val, spos, epos, line = g.next() name = tokenize.tok_name[code] # see tokenize docs to grok the 'NL' token if name not in ('COMMENT', 'NL', 'INDENT'): #print '*****', code, name, repr(val) #print yield (tokenize.tok_name[code], val) spec = parsing.Spec( sys.modules[__name__], pickleFile="meta.pickle", #skinny=False, #logFile="meta.log", #graphFile="meta.dot", #verbose=True ) def parse_grammar(filename): p = parser(spec) #p.verbose = True g = tokenize_grammar(filename) return p.scan(g) if __name__ == '__main__': import sys from pprint import pprint as pp