def test_subclassing(self): """ A subclass of an OMeta subclass should be able to call rules on its parent, and access variables in its scope. """ grammar1 = """ dig = :x ?(a <= x <= b) -> int(x) """ TestGrammar1 = OMeta.makeGrammar(grammar1, "G").createParserClass(OMetaBase, {'a':'0', 'b':'9'}) grammar2 = """ num = (num:n dig:d -> n * base + d | dig) """ TestGrammar2 = OMeta.makeGrammar(grammar2, "G2").createParserClass(TestGrammar1, {'base':10}) g = TestGrammar2("314159") self.assertEqual(g.apply("num")[0], 314159) grammar3 = """ dig = :x ?(a <= x <= b or c <= x <= d) -> int(x, base) """ TestGrammar3 = OMeta.makeGrammar(grammar3, "G3").createParserClass( TestGrammar2, {'c':'a', 'd':'f', 'base':16}) g = TestGrammar3("abc123") self.assertEqual(g.apply("num")[0], 11256099)
def compile(self, grammar, globals=None): """ Produce an object capable of parsing via this grammar. @param grammar: A string containing an OMeta grammar. """ g = OMeta(grammar) tree = g.parseGrammar('TestGrammar') g = GrammarInterpreter(tree, OMetaBase, globals) return HandyInterpWrapper(g)
def test_super(self): """ Rules can call the implementation in a superclass. """ grammar1 = "expr = letter" TestGrammar1 = OMeta.makeGrammar(grammar1, "G").createParserClass(OMetaBase, {}) grammar2 = "expr = super | digit" TestGrammar2 = OMeta.makeGrammar(grammar2, "G2").createParserClass(TestGrammar1, {}) self.assertEqual(TestGrammar2("x").apply("expr")[0], "x") self.assertEqual(TestGrammar2("3").apply("expr")[0], "3")
def test_failure(self): g = OMeta(""" foo = 'a':one baz:two 'd'+ 'e' -> (one, two) baz = 'b' | 'c' """, {}) tree = g.parseGrammar('TestGrammar') i = TrampolinedGrammarInterpreter( tree, 'foo', callback=lambda x: setattr(self, 'result', x)) e = self.assertRaises(ParseError, i.receive, 'foobar') self.assertEqual(str(e), "\nfoobar\n^\nParse error at line 2, column 0:" " expected the character 'a'. trail: []\n")
def test_failure(self): g = OMeta(""" foo = 'a':one baz:two 'd'+ 'e' -> (one, two) baz = 'b' | 'c' """, {}) tree = g.parseGrammar('TestGrammar') i = TrampolinedGrammarInterpreter( tree, 'foo', callback=lambda x: setattr(self, 'result', x)) e = self.assertRaises(ParseError, i.receive, 'foobar') self.assertEqual(str(e), "\nfoobar\n^\nParse error at line 1, column 0:" " expected the character 'a'. trail: []\n")
def makeGrammar(source, bindings, name='Grammar', unwrap=False, extends=wrapGrammar(OMetaBase), tracefunc=None): """ Create a class from a Parsley grammar. :param source: A grammar, as a string. :param bindings: A mapping of variable names to objects. :param name: Name used for the generated class. :param unwrap: If True, return a parser class suitable for subclassing. If False, return a wrapper with the friendly API. :param extends: The superclass for the generated parser class. :param tracefunc: A 3-arg function which takes a fragment of grammar source, the start/end indexes in the grammar of this fragment, and a position in the input. Invoked for terminals and rule applications. """ g = OMeta.makeGrammar(source, name).createParserClass( unwrapGrammar(extends), bindings) if unwrap: return g else: return wrapGrammar(g, tracefunc=tracefunc)
def makeProtocol(source, senderFactory, receiverFactory, bindings=None, name='Grammar'): """ Create a Twisted ``Protocol`` factory from a Parsley grammar. :param source: A grammar, as a string. :param senderFactory: A one-argument callable that takes a twisted ``Transport`` and returns a :ref:`sender <senders>`. :param receiverFactory: A one-argument callable that takes the sender returned by the ``senderFactory`` and returns a :ref:`receiver <receivers>`. :param bindings: A mapping of variable names to objects which will be accessible from python code in the grammar. :param name: The name used for the generated grammar class. :returns: A nullary callable which will return an instance of :class:`~.ParserProtocol`. """ from ometa.protocol import ParserProtocol if bindings is None: bindings = {} grammar = OMeta(source).parseGrammar(name) return functools.partial(ParserProtocol, grammar, senderFactory, receiverFactory, bindings)
def compile(self, grammar): """ Produce an object capable of parsing via this grammar. @param grammar: A string containing an OMeta grammar. """ g = OMeta.makeGrammar(grammar, 'TestGrammar').createParserClass(OMetaBase, {}) return HandyWrapper(g)
def __init__(self): source = OMeta(grammar.grammarSource).parseGrammar('Grammar') bindings = grammar.bindings if bindings is None: bindings = {} ParserProtocol.__init__(self, grammar=source, senderFactory=SOCKS4Sender, receiverFactory=SOCKS4Receiver, bindings=bindings)
def test_stringConsumedBy(self): called = [] grammarSource = "rule = <'x'+>:y -> y" grammar = OMeta(grammarSource).parseGrammar("Parser") def interp(result, error): called.append(result) trampoline = TrampolinedGrammarInterpreter(grammar, "rule", interp) trampoline.receive("xxxxx") trampoline.end() self.assertEqual(called, ["xxxxx"])
def test_makeGrammar(self): results = [] grammar = """ digit = :x ?('0' <= x <= '9') -> int(x) num = (num:n digit:d !(results.append(True)) -> n * 10 + d | digit) """ TestGrammar = OMeta.makeGrammar(grammar, "G").createParserClass(OMetaBase, {'results':results}) g = TestGrammar("314159") self.assertEqual(g.apply("num")[0], 314159) self.assertNotEqual(len(results), 0)
def __init__(self): source = OMeta(grammar.grammarSource).parseGrammar('Grammar') bindings = grammar.bindings if bindings is None: bindings = {} ParserProtocol.__init__(self, grammar=source, senderFactory=SOCKS5Sender, receiverFactory=stack(SOCKS5AuthDispatcher, SOCKS5Receiver), bindings=bindings)
def test_foreign(self): """ Rules can call the implementation in a superclass. """ grammar_letter = "expr = letter" GrammarLetter = OMeta.makeGrammar(grammar_letter, "G").createParserClass(OMetaBase, {}) grammar_digit = "expr '5' = digit" GrammarDigit = OMeta.makeGrammar(grammar_digit, "H").createParserClass(OMetaBase, {}) grammar = ("expr = !(grammar_digit_global):grammar_digit " "grammar_letter.expr | grammar_digit.expr('5')") TestGrammar = OMeta.makeGrammar(grammar, "I").createParserClass( OMetaBase, {"grammar_letter": GrammarLetter, "grammar_digit_global": GrammarDigit }) self.assertEqual(TestGrammar("x").apply("expr")[0], "x") self.assertEqual(TestGrammar("3").apply("expr")[0], "3")
def _register_plugin(self, mod): rules = mod.GRAMMAR verbs = getattr(mod, "VERBS", []) self.grammar_generation = self.grammar_generation + 1 classes = inspect.getmembers(mod, inspect.isclass) self.grammar.valid_verbs.extend(verbs) newgrammar = OMeta.makeGrammar( rules, "Grammar%i" % self.grammar_generation).createParserClass( self.grammar, {}) newgrammar.globals = self.grammar.globals for v in verbs: self.grammar.globals[v] = newgrammar for c in classes: self.plugin_classes[c[0]] = c[1] self._rebuild_parser()
class SOCKS4ClientProtocol(ParserProtocol): source = OMeta(grammar.grammarSource).parseGrammar('Grammar') def __init__(self): bindings = grammar.bindings if bindings is None: bindings = {} ParserProtocol.__init__(self, grammar=self.source, senderFactory=SOCKS4Sender, receiverFactory=SOCKS4Receiver, bindings=bindings) def dataReceived(self, data): data = to_string(data) return ParserProtocol.dataReceived(self, data)
def makeGrammar(source, bindings, name='Grammar', unwrap=False, extends=wrapGrammar(OMetaBase)): """ Create a class from a Parsley grammar. :param source: A grammar, as a string. :param bindings: A mapping of variable names to objects. :param name: Name used for the generated class. :param unwrap: If True, return a parser class suitable for subclassing. If False, return a wrapper with the friendly API. :param extends: The superclass for the generated parser class. """ g = OMeta.makeGrammar(source, name).createParserClass( unwrapGrammar(extends), bindings) if unwrap: return g else: return wrapGrammar(g)
import sys from ometa.grammar import OMeta from ometa.builder import writePython if len(sys.argv) != 3: print "Usage: %s grammar-filename python-filename" % (sys.argv[0],) sys.exit(1) with open(sys.argv[1]) as infile: grammar = infile.read() g = OMeta(grammar) tree = g.parseGrammar("Parser") source = writePython(tree, grammar) + '\n' with open(sys.argv[2], 'w') as outfile: outfile.write(source)
from ometa.runtime import ParseError protocol = pytest.importorskip('ometa.protocol') ParserProtocol = protocol.ParserProtocol testingGrammarSource = """ someA = ('a' 'a') -> receiver('a') someB = ('b' 'b') -> receiver('b') someC = ('c' 'c') -> receiver('c') someExc = 'e' -> receiver.raiseSomething() initial = someA | someExc """ testGrammar = OMeta(testingGrammarSource).parseGrammar('testGrammar') class SenderFactory(object): def __init__(self, transport): self.transport = transport class SomeException(Exception): pass class ReceiverFactory(object): currentRule = 'initial' def __init__(self, sender):
def getRule(source, name): o = OMeta(source).parseGrammar('grammar') return decomposeGrammar(o)[name]
import sys from ometa.grammar import OMeta from ometa.builder import writePython if len(sys.argv) != 3: print "Usage: %s grammar-filename python-filename" % (sys.argv[0], ) sys.exit(1) with open(sys.argv[1]) as infile: grammar = infile.read() g = OMeta(grammar) tree = g.parseGrammar("Parser") source = writePython(tree, grammar) + '\n' with open(sys.argv[2], 'w') as outfile: outfile.write(source)
from os import path from ometa.grammar import OMeta from ometa.runtime import OMetaBase from parsley import wrapGrammar from lib.path_utils import ROOT with open(path.join(ROOT, 'cyder/search/compiler/search.parsley')) as g: B = OMeta.makeGrammar(g.read()).createParserClass(OMetaBase, globals()) class ICompiler(B): def directive(self, d, v): raise NotImplemented() def mac_addr(self, addr): raise NotImplemented() def regexpr(self, r): raise NotImplemented() def text(self, t): raise NotImplemented() def compile(self, initial, values): raise NotImplemented() def OR_op(self, a, b): raise NotImplemented()
class DhcpConfigContext( OMeta.makeGrammar( grammar, name='DhcpConfigContext').createParserClass(OMetaBase, globals())): stdout = stdout def __init__(self, *args, **kwargs): self.hosts = set() self.subnets = set() self.groups = set() self.classes = set() self.options = set() self.parameters = set() super(DhcpConfigContext, self).__init__(*args, **kwargs) def apply_attrs(self, host, attrs): for attr in attrs: host.add_option_or_parameter(attr) def add_subnet(self, subnet): self.subnets.add(subnet) def add_host(self, host): self.hosts.add(host) def add_group(self, group): self.groups.add(group) def add_option(self, option): self.options.add(option) def add_parameter(self, parameter): self.parameters.add(parameter) def add_class(self, dhcp_class): self.classes.add(dhcp_class) def add_subclass(self, name, mac): for _class in self.classes: if _class.name == name: _class.add_subclass(mac) return True return False def __eq__(self, other): return self.hosts == other.hosts and \ self.subnets == other.subnets and \ self.groups == other.groups and \ self.classes == other.classes def diff(self, other): if not (self == other): first_subnets = self.subnets - other.subnets second_subnets = other.subnets - self.subnets first_hosts = self.hosts - other.hosts second_hosts = other.hosts - self.hosts first_groups = self.groups - other.groups second_groups = other.groups - self.groups first_classes = self.classes - other.classes second_classes = other.classes - self.classes if first_subnets: print '### Subnets found only in the first config ###' for subnet in first_subnets: stdout.write(str(subnet)) if second_subnets: print '### Subnets found only in the second config ###' for subnet in second_subnets: stdout.write(str(subnet)) if first_hosts: print '### Hosts found only in the first config ###' for host in first_hosts: stdout.write(str(host)) if second_hosts: print '### Hosts found only in the second config ###' for host in second_hosts: stdout.write(str(host)) if first_groups: print '### Groups found only in the first config ###' for group in first_groups: stdout.write(str(group)) if second_groups: print '### Groups found only in the second config ###' for group in second_groups: stdout.write(str(group)) if first_classes: print '### Classes found only in the first config ###' for klass in first_classes: stdout.write(str(klass)) if second_classes: print '### Classes found only in the second config ###' for klass in second_classes: stdout.write(str(klass))
from parsley import wrapGrammar from ometa.grammar import OMeta from ometa.runtime import OMetaBase from cyder.search.compiler.invparsley import grammar name = 'CyDSL' B = OMeta.makeGrammar(grammar, name=name).createParserClass(OMetaBase, globals()) class ICompiler(B): def directive(self, d, v): raise NotImplemented() def regexpr(self, r): raise NotImplemented() def text(self, t): raise NotImplemented() def compile(self, initial, values): raise NotImplemented() def OR_op(self, a, b): raise NotImplemented() def AND_op(self, a, b): raise NotImplemented()
def __init__(self, sender): self._sender = sender self._messageSetGrammar = OMeta(grammar_source).parseGrammar('messageSetGrammar') self._parsers = {}
def makeProtocol(source, sender, receiver, bindings=None, name='Grammar'): if bindings is None: bindings = {} grammar = OMeta(source).parseGrammar(name) return functools.partial(ParserProtocol, grammar, sender, receiver, bindings)
from os import path from ometa.grammar import OMeta from ometa.runtime import OMetaBase from parsley import wrapGrammar from activate import cy_path with open(cy_path('cyder/search/compiler/search.parsley')) as g: B = OMeta.makeGrammar(g.read()).createParserClass(OMetaBase, globals()) class ICompiler(B): def directive(self, d, v): raise NotImplemented() def mac_addr(self, addr): raise NotImplemented() def regexpr(self, r): raise NotImplemented() def text(self, t): raise NotImplemented() def compile(self, initial, values): raise NotImplemented() def OR_op(self, a, b): raise NotImplemented()
def compile(self, grammar, globals=None): g = OMeta(grammar) tree = g.parseGrammar('TestGrammar') return TrampolinedInterpWrapper(tree, globals)
from parsley import wrapGrammar from ometa.grammar import OMeta from ometa.runtime import OMetaBase from core.search.compiler.invparsley import grammar name = 'InvDSL' B = OMeta.makeGrammar(grammar, name=name).createParserClass( OMetaBase, globals() ) class ICompiler(B): def directive(self, d, v): raise NotImplemented() def regexpr(self, r): raise NotImplemented() def text(self, t): raise NotImplemented() def compile(self, initial, values): raise NotImplemented() def OR_op(self, a, b): raise NotImplemented() def AND_op(self, a, b): raise NotImplemented()
def __init__(self, source): self.source = source self.grammar = OMeta(source).parseGrammar('grammar') self.rules = decomposeGrammar(self.grammar)
def getGrammar(pkg, name): base = os.path.dirname(os.path.abspath(pkg.__file__)) src = open(os.path.join(base, name + ".parsley")).read() return OMeta(src).parseGrammar(name)
def makeSAMProtocol(senderFactory, receiverFactory): g = OMeta(grammar.samGrammarSource).parseGrammar('Grammar') return functools.partial(SAMParserProtocol, g, senderFactory, receiverFactory, {})
def _parseGrammar(self, grammar, name="Grammar"): return OMeta(grammar).parseGrammar(name)
class PythonParser(OMeta.makeGrammar(g, name="PythonParser") .createParserClass(OMetaBase, globals())): depth = 0 parens = 0 keywords = [ "and", "as", "assert", "break", "continue", "def", "del", "elif", "else", "except", "exec", "finally", "for", "from", "global", "if", "import", "in", "is", "lambda", "not", "object", "or", "pass", "print", "raise", "return", "try", "while", "with", "yield", ] def __init__(self, *args, **kwargs): super(PythonParser, self).__init__(*args, **kwargs) self.indents = [] def rule_until(self, rule, token): """ Parse up until a given token, using the given rule. The token may be multiple characters or a single character. """ m = self.input try: result = [] while True: try: s = self.input for char in token: v, e = self.exactly(char) return result, e except ParseError: self.input = s v, e = self.apply(rule) result.append(v) except ParseError, pe: self.input = m raise pe.withMessage(expected("%s until" % rule, token))