def test_url(self):
import rbnf.zero as ze
ze_exp = ze.compile("""
pyimport rbnf.std.common.[recover_codes Tokenizer]
import std.common.[Space]
pattern ::= (~ ('/' | Space))+ as seq
rewrite seq
url ::= ('https' | 'http') as prefix
'://' as slash
pattern as head
('/' pattern)* as tail
['/']
rewrite
def stream():
yield prefix
yield slash
yield from head
yield from tail
stream()
text ::= (url | ~url)+ as urls
rewrite
tuple(recover_codes(url) for url in urls if not isinstance(url, Tokenizer))
lexer_helper := R'.'
""",
use='text')
text = """
<html lang="en">
<head>
<meta charset="utf-8">
<link rel="dns-prefetch" href="https://assets-cdn.github.com">
<link rel="dns-prefetch" href="https://avatars0.githubusercontent.com">
<link rel="dns-prefetch" href="https://avatars1.githubusercontent.com">
<link rel="dns-prefetch" href="https://avatars2.githubusercontent.com">
<link rel="dns-prefetch" href="https://avatars3.githubusercontent.com">
<link rel="dns-prefetch" href="https://github-cloud.s3.amazonaws.com">
<link rel="dns-prefetch" href="https://user-images.githubusercontent.com/">
<link crossorigin="anonymous" media="all" integrity="sha512-PkbtxdWDpLChpxtWQ0KbvJoef4XMYPq5pfd/ZmylYZTzXYpCfGwN9d+bsSKcmOJLwTkfjFkfj5wz3poDrhJoSQ==" rel="stylesheet" href="https://assets-cdn.github.com/assets/frameworks-f6e6ce21346c0d2eb22def1e8534afcb.css" />
<link crossorigin="anonymous" media="all" integrity="sha512-LHNZGPA72iEyT2UIFOpxTPnfDcJ1Ecx8MKZgMzCJzkqfID/5niECnSBbRtDc4LDgbI3YDHu5dgs5mQiMmum6cA==" rel="stylesheet" href="https://assets-cdn.github.com/assets/github-caf1b1f61473986b3fdfa6e73e76a94f.css" />
<meta name="viewport" content="width=device-width">
....
"""
expected = (
'https://assets-cdn.github.com">',
'https://avatars0.githubusercontent.com">',
'https://avatars1.githubusercontent.com">',
'https://avatars2.githubusercontent.com">',
'https://avatars3.githubusercontent.com">',
'https://github-cloud.s3.amazonaws.com">',
'https://user-images.githubusercontent.com/">',
'https://assets-cdn.github.com/assets/frameworks-f6e6ce21346c0d2eb22def1e8534afcb.css"',
'https://assets-cdn.github.com/assets/github-caf1b1f61473986b3fdfa6e73e76a94f.css"'
)
self.assertEqual(expected, ze_exp.match(text).result)
def test_ze(self):
zero_exp = ze.compile("""
# test
import poly.[*] # 1234""",
use='Poly')
expected = [(0, 7), (1, 3), (2, 7), (5, -7), (11, 7)]
self.assertEqual(
zero_exp.match(
"2x^2 + 3 + 4 - 7 x^5 + 4 x + 5 x ^2 - x + 7 x ^ 11").result,
expected)
def test_predicate():
ze_exp = ze.compile("""
[python] import predicate_helpers.[*]
lexer_helper := R'.'
a ::= (_{is_ok})+
b ::= (_+){not_ok}
""",
use='a')
assert len(ze_exp.match("123234").result.item) == 2
ze_exp = ze.compile("""
[python] import predicate_helpers.[*]
lexer_helper := R'.'
a ::= (_{is_ok})+
b ::= (_+){not_ok}
""",
use='b')
assert ze_exp.match("123234").result == None
print(ze_exp.dumps())
def test_simple_xml(self):
import rbnf.zero as ze
ze_exp = ze.compile("""
import std.common.[Name Space]
# import `Name` and `Space` from $RBNF_HOME/std/common
XML ::=
| '<' t1=Name '/' '>'
| '<' t1=Name '>' (XML | (seq << ~('<' '/' Name '>')))* '<' '/' t2=Name '>'
with
't2' not in state.ctx or t1.value == t2.value
rewrite
t1.value, seq if seq else ()
""")
print(ze_exp.match('<a> b </a>').result)
def test_doctrans(self):
ze_exp = ze.compile("""
pyimport rbnf.std.common.[recover_codes]
LexerHelper := R'.'
Formula ::= '`' (~'`')+ as content '`'
rewrite " :math:`"+recover_codes(content)+'` '
Unit ::= Formula as formula | _ as other
rewrite
formula if not other else other.value
Text ::= Unit+ as seq
rewrite ''.join(seq)
""")
self.assertEqual(
ze_exp.match('abcdefg `\lambda + 1` + 1').result,
'abcdefg :math:`\lambda + 1` + 1')
ze_exp.lang.as_fixed()
self.assertEqual(
ze_exp.match('abcdefg `\lambda + 1` + 1').result,
'abcdefg :math:`\lambda + 1` + 1')
from Redy.Tools.PathLib import Path
from .visitor import *
import rbnf.zero as ze
import sys
import io
import warnings
import readline
from .std import std
warnings.filterwarnings('ignore')
# with Path('./rmalt').into('malt.rbnf').open('r') as f:
# rbnf = f.read()
ze_exp = ze.compile('import rmalt.malt.[*]', use='Grammar')
KeyWords = ['let', 'include', 'not', 'cond', 'true', 'false', 'nil', 'or', 'and']
class Completer:
def __init__(self):
self.choices = []
self.previous = []
@feature
def err_write(info):
if constexpr[isinstance(sys.stderr, io.BufferedWriter)]:
if isinstance(info, str):
info = info.encode()
import rbnf.zero as ze
import timeit
ze_exp = ze.compile("""
[python] import rbnf.std.common.[recover_codes]
pattern := R'[^/\sa-z]+'
url ::= (('https:' | 'http:') '//' pattern+ ('/' pattern)* ['/']) as result
-> result
text ::= (url to [urls]| ~url)+
rewrite
tuple(recover_codes(each) for each in urls)
pattern := R'[a-zA-Z0-9_]+'
space := R'\s+'
""",
use='text')
with open('prof_compiled.py', 'w') as f:
f.write(ze_exp.dumps())
f.write('ulang.as_fixed()\n')
f.write('_impl = ulang.implementation\n')
f.write('from rbnf.core.State import State\n')
f.write('_ze_exp = text.match\n')
f.write('lexer = ulang.lexer\n')
f.write('ze_exp = lambda text: _ze_exp(tuple(lexer(text)), State(_impl))')
import rbnf.zero as ze
import timeit
ze_exp = ze.compile("""
pyimport rbnf.std.common.[recover_codes Tokenizer]
import std.common.[Space]
pattern ::= (~ ('/' | Space))+ as seq
rewrite
seq
url ::= ('https' | 'http') as prefix
'://' as slash
pattern as head
('/' pattern)* as tail
['/']
rewrite
def stream():
yield prefix
yield slash
yield from head
yield from tail
stream()
text ::= (url | ~url)+ as urls
rewrite
tuple(recover_codes(url) for url in urls if not isinstance(url, Tokenizer))
lexer_helper := R'.'
""", use='text')
text = """
<html lang="en">
<head>
<meta charset="utf-8">
import rbnf.zero as ze
from Redy.Tools.PathLib import Path
with Path("./task.rbnf").open('r') as f:
ze_exp = ze.compile(f.read(), use='Test')
with open(Path("./data.xml").__str__(), encoding='utf8') as f:
text = f.read()
result = ze_exp.match(text).result
for each in result:
print(each)
import rbnf.zero as ze
ze_exp = ze.compile('import mlisp.mlisp.[*]', use='grammar')
s = ze_exp.match("""
(def f
(lambda (x, y)
(+ x y)))
(let s 1)
(let f 1.0)
(def Point
(struct
(x:i32 y:i32)))
""")
for each in s.tokens:
print(each)
print(s.result)
import rbnf.zero as ze
from visitor import visit
ze_exp = ze.compile('import calc.[*]', use='Add')
def loop():
while True:
inp = input('calc> ')
if inp == 'exit':
print('good bye!')
break
try:
matched = ze_exp.match(inp)
print('=> ', visit(matched.result))
except Exception as e:
print(repr(e))
loop()
import rbnf.zero as ze
zero_exp = ze.compile("""
import std.common.[Number Name Space]
ignore [Space]
Numeric ::= Number as integer ['.' Number as floating]
rewrite float(integer.value + '.' + floating.value) if floating else int(integer.value)
Term ::= [Numeric as coef] Name as root ['^' Number as power] | Numeric as coef
with not root or root.value == 'x'
rewrite coef if coef else 1, int(power.value) if power else 1 if root else 0
Add ::= ('-' as neg | '+') Term as term
rewrite
coef, power = term
-coef if neg else coef, power
Poly ::= ['-' as neg] Term as head Add* as seq
rewrite
class Polynomial(dict):
def __missing__(self, k):
return 0
mapping = Polynomial()
coef, power = head
mapping[power] = -coef if neg else coef
if any(seq):
for coef, power in seq:
mapping[power] += coef
sorted(mapping.items(), key=lambda kv: kv[0])""")
print(
zero_exp.match(
"2x^2 + 3 + 4 - 7 x^5 + 4 x + 5 x ^2 - x + 7 x ^ 11").result)
ze_exp = ze.compile("""
import std.common.[Space DoubleQuotedStr Str]
[python] import wisepy.cmd_ast.[*]
ignore [Space Newline]
arg ::= Str as str | DoubleQuotedStr as str | pattern as pat | quote as expr | closure as expr | placeholder as expr
rewrite
if expr:
return expr
if str:
return eval(str.value)
pat.value
placeholder ::= '$' arg as arg
rewrite
PlaceHolder(arg)
quote ::= '`' command as cmd '`'
rewrite
Quote(cmd)
flag ::= '--' pattern as key
with
key.value.isidentifier()
rewrite
(key.value, True)
must ::= '-' pattern as key arg as value
with
key.value.isidentifier()
rewrite
(key.value, value)
closure ::= '{' [command to [stmts] (';' command to [stmts])*] [';'] '}'
rewrite Closure(stmts)
command ::= arg as instruction ( (arg as last) to [args]
| (flag as last) to [kwargs]
| (must as last) to [kwargs]
)*
['|' command as and_then]
rewrite
while isinstance(last, Cmd) and isinstance(last.last, Cmd):
last = last.last
ret = Cmd(instruction, args, kwargs, last)
if and_then:
args = and_then.args or ()
ret = Cmd(and_then.inst, (ret, *args), and_then.kwargs, and_then.last)
ret
pattern := R'[^`\s\{\}\;]+'
Newline := '\n'
""")
import rbnf.zero as ze
import sys, os
from rbnf.easy import build_parser
from Redy.Tools.PathLib import Path
pwd = Path(__file__).parent().__str__()
sys.path.append(pwd)
os.chdir(pwd)
ze_exp = ze.compile('import simple_ml.[*]', use='Grammar')
def test_simple_ml():
_test_simple_ml()
ze_exp.lang.as_fixed()
_test_simple_ml()
parse = build_parser(ze_exp.lang, opt=True)
print(parse("let x = 1 in x"))
def _test_simple_ml():
print(
repr(
ze_exp.match("""
let s = fn x: 'a -> x in
let d = s "1" in
let k = fn x: 'a -> x in
k;
""").result))
print(
import rbnf.zero as ze
def success(result: ze.ResultDescription):
return len(result.tokens) <= result.state.end_index
ze_exp = ze.compile('import rml.rml.[*]', use='ModuleDef')
assert success(ze_exp.match("""
module S where
let (x, y) = 1
"""))
assert not success(
ze_exp.match("""
module S where
let (x, y) = 1
in x = y
"""))
assert success(
ze_exp.match("""
module S where
let (x, y) = "s\\""
in x + y
"""))
assert not success(
ze_exp.match("""
module S where
import rbnf.zero as ze
ze.compile("""
pyimport rbnf.std.common.[recover_codes]
LexerHelper := R'.'
Formula ::= '`' (~'`')+ as content '`'
rewrite " :math:`"+recover_codes(content)+'` '
Unit ::= Formula as formula | _ as other
rewrite
formula if not other else other.value
Text ::= Unit+ as seq
rewrite ''.join(seq)
""")
import rbnf.zero as ze
from Redy.Tools.PathLib import Path
from bytecode import Bytecode
from yapypy.extended_python.parser import parse
from yapypy.extended_python.py_compile import py_compile
ze_exp = ze.compile(r"""
[python] import rbnf.std.common.[recover_codes]
Space := ' '
NL := R'\n'
Keyword := 'test:' 'prepare:' '>>>' 'title:'
NoSwitch ::= ~Keyword
Doctest ::= [(~'title:')* 'title:' name=(~NL)+]
[(~'prepare:')* 'prepare:' (NoSwitch* '>>>' prepare_lines<<((~NL)+) NL+)*]
(~'test:')* 'test:' (NoSwitch* '>>>' test_lines<<((~NL)+))*
->
prepare_lines = recover_codes(sum(prepare_lines, [])) if prepare_lines else ''
test = recover_codes(sum(test_lines, [])) if test_lines else ''
return recover_codes(name) if name else None, prepare_lines, test
lexer := R'.'
TestCase ::= [it=Doctest] _* -> it or None
""",
use='TestCase')
yapypy = Path('yapypy')
def dedent_all(text: str):
while text.startswith(' ') or text.startswith('\t'):
text = dedent(text)
def run(filename: 'python file generated by `rbnf` command, or rbnf sour file',
opt: 'optimize switch' = False):
"""
You can apply immediate tests on your parser.
P.S: use `--opt` option takes longer starting time.
"""
from rbnf.easy import build_parser
import importlib.util
import traceback
full_path = Path(filename)
base, ext = os.path.splitext(str(full_path))
full_path_str = str(full_path)
if not ext:
if full_path.into('.py').exists():
full_path_str = base + '.py'
elif Path(base).into('.rbnf'):
full_path_str = base + '.rbnf'
if full_path_str[-3:].lower() != '.py':
with Path(full_path_str).open('r') as fr:
ze_exp = ze.compile(fr.read(), filename=full_path_str)
lang = ze_exp.lang
else:
spec = importlib.util.spec_from_file_location("runbnf", full_path_str)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
try:
lang = next(each for each in mod.__dict__.values()
if isinstance(each, Language))
except StopIteration:
raise NameError("Found no language in {}".format(full_path_str))
parse = build_parser(lang, opt=bool(opt))
namespace = {}
print(Purple('type `:i` to switch between python mode and parsing mode.'))
print(Purple('The last result of parsing is stored as symbol `res`.'))
while True:
inp = input('runbnf> ')
if not inp.strip():
continue
elif inp.strip() == 'exit':
break
if inp.strip() == ':i':
while True:
inp = input('python> ')
if inp.strip() == ':i':
break
try:
try:
res = eval(inp, namespace)
if res is not None:
print(res)
namespace['_'] = res
except SyntaxError:
exec(inp, namespace)
except Exception:
traceback.print_exc()
else:
res = namespace['res'] = parse(inp)
print(
LightBlue(
'parsed result = res: ResultDescription{result, tokens, state}'
))
print(res.result)
