def test_url(self): import rbnf.zero as ze ze_exp = ze.compile(""" pyimport rbnf.std.common.[recover_codes Tokenizer] import std.common.[Space] pattern ::= (~ ('/' | Space))+ as seq rewrite seq url ::= ('https' | 'http') as prefix '://' as slash pattern as head ('/' pattern)* as tail ['/'] rewrite def stream(): yield prefix yield slash yield from head yield from tail stream() text ::= (url | ~url)+ as urls rewrite tuple(recover_codes(url) for url in urls if not isinstance(url, Tokenizer)) lexer_helper := R'.' """, use='text') text = """ <html lang="en"> <head> <meta charset="utf-8"> <link rel="dns-prefetch" href="https://assets-cdn.github.com"> <link rel="dns-prefetch" href="https://avatars0.githubusercontent.com"> <link rel="dns-prefetch" href="https://avatars1.githubusercontent.com"> <link rel="dns-prefetch" href="https://avatars2.githubusercontent.com"> <link rel="dns-prefetch" href="https://avatars3.githubusercontent.com"> <link rel="dns-prefetch" href="https://github-cloud.s3.amazonaws.com"> <link rel="dns-prefetch" href="https://user-images.githubusercontent.com/"> <link crossorigin="anonymous" media="all" integrity="sha512-PkbtxdWDpLChpxtWQ0KbvJoef4XMYPq5pfd/ZmylYZTzXYpCfGwN9d+bsSKcmOJLwTkfjFkfj5wz3poDrhJoSQ==" rel="stylesheet" href="https://assets-cdn.github.com/assets/frameworks-f6e6ce21346c0d2eb22def1e8534afcb.css" /> <link crossorigin="anonymous" media="all" integrity="sha512-LHNZGPA72iEyT2UIFOpxTPnfDcJ1Ecx8MKZgMzCJzkqfID/5niECnSBbRtDc4LDgbI3YDHu5dgs5mQiMmum6cA==" rel="stylesheet" href="https://assets-cdn.github.com/assets/github-caf1b1f61473986b3fdfa6e73e76a94f.css" /> <meta name="viewport" content="width=device-width"> .... """ expected = ( 'https://assets-cdn.github.com">', 'https://avatars0.githubusercontent.com">', 'https://avatars1.githubusercontent.com">', 'https://avatars2.githubusercontent.com">', 'https://avatars3.githubusercontent.com">', 'https://github-cloud.s3.amazonaws.com">', 'https://user-images.githubusercontent.com/">', 'https://assets-cdn.github.com/assets/frameworks-f6e6ce21346c0d2eb22def1e8534afcb.css"', 'https://assets-cdn.github.com/assets/github-caf1b1f61473986b3fdfa6e73e76a94f.css"' ) self.assertEqual(expected, ze_exp.match(text).result)
def test_ze(self): zero_exp = ze.compile(""" # test import poly.[*] # 1234""", use='Poly') expected = [(0, 7), (1, 3), (2, 7), (5, -7), (11, 7)] self.assertEqual( zero_exp.match( "2x^2 + 3 + 4 - 7 x^5 + 4 x + 5 x ^2 - x + 7 x ^ 11").result, expected)
def test_predicate(): ze_exp = ze.compile(""" [python] import predicate_helpers.[*] lexer_helper := R'.' a ::= (_{is_ok})+ b ::= (_+){not_ok} """, use='a') assert len(ze_exp.match("123234").result.item) == 2 ze_exp = ze.compile(""" [python] import predicate_helpers.[*] lexer_helper := R'.' a ::= (_{is_ok})+ b ::= (_+){not_ok} """, use='b') assert ze_exp.match("123234").result == None print(ze_exp.dumps())
def test_simple_xml(self): import rbnf.zero as ze ze_exp = ze.compile(""" import std.common.[Name Space] # import `Name` and `Space` from $RBNF_HOME/std/common XML ::= | '<' t1=Name '/' '>' | '<' t1=Name '>' (XML | (seq << ~('<' '/' Name '>')))* '<' '/' t2=Name '>' with 't2' not in state.ctx or t1.value == t2.value rewrite t1.value, seq if seq else () """) print(ze_exp.match('<a> b </a>').result)
def test_doctrans(self): ze_exp = ze.compile(""" pyimport rbnf.std.common.[recover_codes] LexerHelper := R'.' Formula ::= '`' (~'`')+ as content '`' rewrite " :math:`"+recover_codes(content)+'` ' Unit ::= Formula as formula | _ as other rewrite formula if not other else other.value Text ::= Unit+ as seq rewrite ''.join(seq) """) self.assertEqual( ze_exp.match('abcdefg `\lambda + 1` + 1').result, 'abcdefg :math:`\lambda + 1` + 1') ze_exp.lang.as_fixed() self.assertEqual( ze_exp.match('abcdefg `\lambda + 1` + 1').result, 'abcdefg :math:`\lambda + 1` + 1')
from Redy.Tools.PathLib import Path from .visitor import * import rbnf.zero as ze import sys import io import warnings import readline from .std import std warnings.filterwarnings('ignore') # with Path('./rmalt').into('malt.rbnf').open('r') as f: # rbnf = f.read() ze_exp = ze.compile('import rmalt.malt.[*]', use='Grammar') KeyWords = ['let', 'include', 'not', 'cond', 'true', 'false', 'nil', 'or', 'and'] class Completer: def __init__(self): self.choices = [] self.previous = [] @feature def err_write(info): if constexpr[isinstance(sys.stderr, io.BufferedWriter)]: if isinstance(info, str): info = info.encode()
import rbnf.zero as ze import timeit ze_exp = ze.compile(""" [python] import rbnf.std.common.[recover_codes] pattern := R'[^/\sa-z]+' url ::= (('https:' | 'http:') '//' pattern+ ('/' pattern)* ['/']) as result -> result text ::= (url to [urls]| ~url)+ rewrite tuple(recover_codes(each) for each in urls) pattern := R'[a-zA-Z0-9_]+' space := R'\s+' """, use='text') with open('prof_compiled.py', 'w') as f: f.write(ze_exp.dumps()) f.write('ulang.as_fixed()\n') f.write('_impl = ulang.implementation\n') f.write('from rbnf.core.State import State\n') f.write('_ze_exp = text.match\n') f.write('lexer = ulang.lexer\n') f.write('ze_exp = lambda text: _ze_exp(tuple(lexer(text)), State(_impl))')
import rbnf.zero as ze import timeit ze_exp = ze.compile(""" pyimport rbnf.std.common.[recover_codes Tokenizer] import std.common.[Space] pattern ::= (~ ('/' | Space))+ as seq rewrite seq url ::= ('https' | 'http') as prefix '://' as slash pattern as head ('/' pattern)* as tail ['/'] rewrite def stream(): yield prefix yield slash yield from head yield from tail stream() text ::= (url | ~url)+ as urls rewrite tuple(recover_codes(url) for url in urls if not isinstance(url, Tokenizer)) lexer_helper := R'.' """, use='text') text = """ <html lang="en"> <head> <meta charset="utf-8">
import rbnf.zero as ze from Redy.Tools.PathLib import Path with Path("./task.rbnf").open('r') as f: ze_exp = ze.compile(f.read(), use='Test') with open(Path("./data.xml").__str__(), encoding='utf8') as f: text = f.read() result = ze_exp.match(text).result for each in result: print(each)
import rbnf.zero as ze ze_exp = ze.compile('import mlisp.mlisp.[*]', use='grammar') s = ze_exp.match(""" (def f (lambda (x, y) (+ x y))) (let s 1) (let f 1.0) (def Point (struct (x:i32 y:i32))) """) for each in s.tokens: print(each) print(s.result)
import rbnf.zero as ze from visitor import visit ze_exp = ze.compile('import calc.[*]', use='Add') def loop(): while True: inp = input('calc> ') if inp == 'exit': print('good bye!') break try: matched = ze_exp.match(inp) print('=> ', visit(matched.result)) except Exception as e: print(repr(e)) loop()
import rbnf.zero as ze zero_exp = ze.compile(""" import std.common.[Number Name Space] ignore [Space] Numeric ::= Number as integer ['.' Number as floating] rewrite float(integer.value + '.' + floating.value) if floating else int(integer.value) Term ::= [Numeric as coef] Name as root ['^' Number as power] | Numeric as coef with not root or root.value == 'x' rewrite coef if coef else 1, int(power.value) if power else 1 if root else 0 Add ::= ('-' as neg | '+') Term as term rewrite coef, power = term -coef if neg else coef, power Poly ::= ['-' as neg] Term as head Add* as seq rewrite class Polynomial(dict): def __missing__(self, k): return 0 mapping = Polynomial() coef, power = head mapping[power] = -coef if neg else coef if any(seq): for coef, power in seq: mapping[power] += coef sorted(mapping.items(), key=lambda kv: kv[0])""") print( zero_exp.match( "2x^2 + 3 + 4 - 7 x^5 + 4 x + 5 x ^2 - x + 7 x ^ 11").result)
ze_exp = ze.compile(""" import std.common.[Space DoubleQuotedStr Str] [python] import wisepy.cmd_ast.[*] ignore [Space Newline] arg ::= Str as str | DoubleQuotedStr as str | pattern as pat | quote as expr | closure as expr | placeholder as expr rewrite if expr: return expr if str: return eval(str.value) pat.value placeholder ::= '$' arg as arg rewrite PlaceHolder(arg) quote ::= '`' command as cmd '`' rewrite Quote(cmd) flag ::= '--' pattern as key with key.value.isidentifier() rewrite (key.value, True) must ::= '-' pattern as key arg as value with key.value.isidentifier() rewrite (key.value, value) closure ::= '{' [command to [stmts] (';' command to [stmts])*] [';'] '}' rewrite Closure(stmts) command ::= arg as instruction ( (arg as last) to [args] | (flag as last) to [kwargs] | (must as last) to [kwargs] )* ['|' command as and_then] rewrite while isinstance(last, Cmd) and isinstance(last.last, Cmd): last = last.last ret = Cmd(instruction, args, kwargs, last) if and_then: args = and_then.args or () ret = Cmd(and_then.inst, (ret, *args), and_then.kwargs, and_then.last) ret pattern := R'[^`\s\{\}\;]+' Newline := '\n' """)
import rbnf.zero as ze import sys, os from rbnf.easy import build_parser from Redy.Tools.PathLib import Path pwd = Path(__file__).parent().__str__() sys.path.append(pwd) os.chdir(pwd) ze_exp = ze.compile('import simple_ml.[*]', use='Grammar') def test_simple_ml(): _test_simple_ml() ze_exp.lang.as_fixed() _test_simple_ml() parse = build_parser(ze_exp.lang, opt=True) print(parse("let x = 1 in x")) def _test_simple_ml(): print( repr( ze_exp.match(""" let s = fn x: 'a -> x in let d = s "1" in let k = fn x: 'a -> x in k; """).result)) print(
import rbnf.zero as ze def success(result: ze.ResultDescription): return len(result.tokens) <= result.state.end_index ze_exp = ze.compile('import rml.rml.[*]', use='ModuleDef') assert success(ze_exp.match(""" module S where let (x, y) = 1 """)) assert not success( ze_exp.match(""" module S where let (x, y) = 1 in x = y """)) assert success( ze_exp.match(""" module S where let (x, y) = "s\\"" in x + y """)) assert not success( ze_exp.match(""" module S where
import rbnf.zero as ze ze.compile(""" pyimport rbnf.std.common.[recover_codes] LexerHelper := R'.' Formula ::= '`' (~'`')+ as content '`' rewrite " :math:`"+recover_codes(content)+'` ' Unit ::= Formula as formula | _ as other rewrite formula if not other else other.value Text ::= Unit+ as seq rewrite ''.join(seq) """)
import rbnf.zero as ze from Redy.Tools.PathLib import Path from bytecode import Bytecode from yapypy.extended_python.parser import parse from yapypy.extended_python.py_compile import py_compile ze_exp = ze.compile(r""" [python] import rbnf.std.common.[recover_codes] Space := ' ' NL := R'\n' Keyword := 'test:' 'prepare:' '>>>' 'title:' NoSwitch ::= ~Keyword Doctest ::= [(~'title:')* 'title:' name=(~NL)+] [(~'prepare:')* 'prepare:' (NoSwitch* '>>>' prepare_lines<<((~NL)+) NL+)*] (~'test:')* 'test:' (NoSwitch* '>>>' test_lines<<((~NL)+))* -> prepare_lines = recover_codes(sum(prepare_lines, [])) if prepare_lines else '' test = recover_codes(sum(test_lines, [])) if test_lines else '' return recover_codes(name) if name else None, prepare_lines, test lexer := R'.' TestCase ::= [it=Doctest] _* -> it or None """, use='TestCase') yapypy = Path('yapypy') def dedent_all(text: str): while text.startswith(' ') or text.startswith('\t'): text = dedent(text)
def run(filename: 'python file generated by `rbnf` command, or rbnf sour file', opt: 'optimize switch' = False): """ You can apply immediate tests on your parser. P.S: use `--opt` option takes longer starting time. """ from rbnf.easy import build_parser import importlib.util import traceback full_path = Path(filename) base, ext = os.path.splitext(str(full_path)) full_path_str = str(full_path) if not ext: if full_path.into('.py').exists(): full_path_str = base + '.py' elif Path(base).into('.rbnf'): full_path_str = base + '.rbnf' if full_path_str[-3:].lower() != '.py': with Path(full_path_str).open('r') as fr: ze_exp = ze.compile(fr.read(), filename=full_path_str) lang = ze_exp.lang else: spec = importlib.util.spec_from_file_location("runbnf", full_path_str) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) try: lang = next(each for each in mod.__dict__.values() if isinstance(each, Language)) except StopIteration: raise NameError("Found no language in {}".format(full_path_str)) parse = build_parser(lang, opt=bool(opt)) namespace = {} print(Purple('type `:i` to switch between python mode and parsing mode.')) print(Purple('The last result of parsing is stored as symbol `res`.')) while True: inp = input('runbnf> ') if not inp.strip(): continue elif inp.strip() == 'exit': break if inp.strip() == ':i': while True: inp = input('python> ') if inp.strip() == ':i': break try: try: res = eval(inp, namespace) if res is not None: print(res) namespace['_'] = res except SyntaxError: exec(inp, namespace) except Exception: traceback.print_exc() else: res = namespace['res'] = parse(inp) print( LightBlue( 'parsed result = res: ResultDescription{result, tokens, state}' )) print(res.result)