def get_parser(start, path):
peg = pg.grammar(path)
if start is None:
parser = pg.generate(peg)
else:
parser = pg.generate(peg, start=start)
return parser
def parse(line, src=''):
start, path = start_option(line.strip())
peg = pg.grammar(path)
if start is None:
parser = pg.generate(peg)
else:
parser = pg.generate(peg, start=start)
tree = parser(src, urn='(stdin)')
print(repr(tree))
return graph.draw_graph(tree)
def parse_example(peg, line):
if '@@example' in peg and len(peg['@@example']) > 0:
name, doc = peg['@@example'][-1]
parser = pg.generate(peg, start=str(name))
print(f'example: {name}', str(line))
return parser(str(doc))
else:
parser = pg.generate(peg)
print('Input:', line)
return parser(line)
def extract(line, cell=''):
files, options = parse_option(line.split())
peg = load_grammar(files, options)
start = options.get('--start', peg.start())
start = f'extract_{start}'
if start not in peg:
peg.addExtract(start)
parser = pg.generate(peg, start=start)
file_or_text = options.get('--from', cell)
def __init__(self, grammar='../pegtree/puppy2.tpeg'):
TransCompiler.__init__(self)
peg = pg.grammar(grammar)
self.parser = pg.generate(peg)
self.pycode = PythonCode(grammar)
self.env = {}
self.vocmap = VocabMap(self.env)
self.pycode.set_vocmap(self.vocmap)
self.isBlockMode = True
self.isAllowMath = False
def test_math(self):
peg = grammar("math.tpeg")
parser = generate(peg)
tree = parser('1+2*3')
self.assertEqual(str(tree), '1+2*3')
self.assertEqual(repr(tree),
"[#Add [#Int '1'][#Mul [#Int '2'][#Int '3']]]")
tree = parser('1*2+3')
self.assertEqual(str(tree), '1*2+3')
self.assertEqual(repr(tree),
"[#Add [#Mul [#Int '1'][#Int '2']][#Int '3']]")
def test_example(peg, start=None):
if '@@example' not in peg:
return
parsers = {}
for testcase in peg['@@example']:
name, doc = testcase
if not name in peg:
continue
if start is not None and name != start:
continue
if not name in parsers:
parsers[name] = pg.generate(peg, start=name)
tree = parsers[name](doc.inputs_, doc.urn_, doc.spos_, doc.epos_)
ok = doc.inputs_[doc.spos_:tree.epos_]
fail = doc.inputs_[tree.epos_:doc.epos_]
display(
HTML(
f'<b>{name}</b> {ok}<span style="background-color:#FFCACA;">{fail}</span>'
))
v = graph.draw_graph(tree)
v.format = 'SVG'
display(SVG(v.render()))
from os import read
import random
import sys
import pegtree as pg
import argparse
import csv
from pegtree.optimizer import optimize
peg = pg.grammar('yk.tpeg')
parse = pg.generate(peg)
parser = argparse.ArgumentParser(description='yk for Parameter Handling')
parser.add_argument('--notConv', action='store_true') # Python のトークナイズのみ
parser.add_argument('--diff',
action='store_true') # 変数名 (name) とリテラル (val) に異なるものを付与
parser.add_argument('--shuffle',
action='store_true') # 特殊トークンをランダムに付与 (順序を考慮しない)
parser.add_argument('--both', action='store_true') # shuffle ありとなしを両方追加
parser.add_argument('--files', nargs='*') # 入力ファイルを与える
args = parser.parse_args()
token_idx = list(range(1, 7))
def replace_as_special_parameter(s,
mapped,
token_idx=token_idx,
tag=None): # mapped => {'df': '<A>'}
if s in mapped:
import sys
import pegtree as pg
tpeg = pg.grammar('pyline.pegtree')
parser = pg.generate(tpeg)
def fix(tree):
a = [tree.epos_]
for t in tree:
fix(t)
a.append(t.epos_)
for key in tree.keys():
a.append(fix(tree.get(key)).epos_)
tree.epos_ = max(a)
return tree
def readfile(filename, head=None):
with open(filename, 'r') as f:
for i, line in enumerate(f.readlines()):
tree = parser(line)
if len(tree) > 0:
for t in tree:
if t.tag_ != 'Name':
print(fix(t))
#print(repr(fix(t)))
if i == head:
break
def apply(self, dict_or_func=identity):
if isinstance(dict_or_func, dict):
if self.index in dict_or_func:
return dict_or_func[self.index]
if self.w in dict_or_func:
return dict_or_func[self.w]
return self
else:
return dict_or_func(self)
def emit(self, buffers):
buffers.append(self.w)
peg = pg.grammar('tokibi.pegtree')
tokibi_parser = pg.generate(peg)
class TokibiReader(ParseTreeVisitor):
def __init__(self, synonyms=None):
ParseTreeVisitor.__init__(self)
self.indexes = {}
self.synonyms = {} if synonyms is None else synonyms
def parse(self, s):
tree = tokibi_parser(s)
self.indexes = {}
nexpr = self.visit(tree)
return nexpr, self.indexes
import sys
import pegtree as pg
tpeg = pg.grammar('pynmt.pegtree')
parser = pg.generate(tpeg, start = 'Checker')
parser2 = pg.generate(tpeg, start = 'Statement')
def check_syntax(s: str):
"""
Python の構文的に正しいかどうか
"""
tree = parser(s.strip())
# print(repr(tree))
# return not tree.isSyntaxError()
return tree.isSyntaxError()
def fix(tree):
a = [tree.epos_]
for t in tree:
fix(t)
a.append(t.epos_)
for key in tree.keys():
a.append(fix(tree.get(key)).epos_)
tree.epos_ = max(a)
return tree
def filter_syntax(s: str):
"""
Pythonの構文的に正しいところまで取り出す。
"""
tree = parser2(s.strip())
def __init__(self, grammar='puppy2.tpeg'):
self.buffers = []
peg = pg.grammar(grammar)
self.parser = pg.generate(peg)
import pegtree as pg
from pegtree import ParseTree
from pegtree.visitor import ParseTreeVisitor
import tree as ntree
import pprint
peg = pg.grammar('multiese.pegtree')
parser = pg.generate(peg)
def fix(tree):
a = [tree.epos_]
for t in tree:
fix(t)
a.append(t.epos_)
for key in tree.keys():
a.append(fix(tree.get(key)).epos_)
tree.epos_ = max(a)
return tree
class MultieseParser(ParseTreeVisitor):
def __init__(self):
ParseTreeVisitor.__init__(self)
def parse(self, s: str):
tree = parser(s)
node = self.visit(tree)
return node
def acceptChunk(self, tree: ParseTree):
def __init__(self, grammar='../pegtree/puppy2.tpeg'):
peg = pg.grammar(grammar)
self.parser = pg.generate(peg)
self.rootEnv = {}
def match(line, src):
peg = pg.grammar(src)
if '@error' not in peg:
parser = pg.generate(peg)
res = parser(line)
print(repr(res))
def __init__(self, grammar='../pegtree/puppy2.tpeg'):
TransCompiler.__init__(self)
peg = pg.grammar(grammar)
self.parser = pg.generate(peg)
self.vocmap = None
self.isTrimString = False
return ''
CEMPTY = CEmpty()
class CSlice(CExpr):
def __init__(self, recv, start=CEMPTY, stop=CEMPTY, step=CEMPTY):
CExpr.__init__(self, "[]", (toCExpr(recv),
toCExpr(start), toCExpr(stop), toCExpr(step)))
def format(self):
return f'{{}} [ {{}} : {{}} : {{}}]'
##
peg = pg.grammar('kotoha.pegtree')
snipet_parser = pg.generate(peg, start='Snipet')
class CodeParser(ParseTreeVisitor):
def __init__(self):
ParseTreeVisitor.__init__(self)
self.symbols = EMPTY
self.modules = STATIC_MODULE
# self.names = {}
# self.synonyms = {}
# self.newnames = set()
def parse(self, source, symbols=EMPTY):
self.symbols = symbols
tree = snipet_parser(source)
return self.visit(tree)
import pegtree as pg
puppy_peg = pg.grammar('puppy2.tpeg')
puppy_parser = pg.generate(puppy_peg)
METHODS = {}
FUNCDATA = {
'print': {
'stem': '表示する',
0: '空行を',
1: '{}を',
'end=': '{}を行末として',
'end=""': '改行なしで',
'seq=': '{}を区切り文字列として',
},
'+': {
'stem': '加えた値',
2: '{}を{}に',
},
'+=': {
'stem': '増やす',
2: '{}を{}だけ',
}
}
class Transpiler(object):
buffers: list
def __init__(self):
def test_int(self):
peg = grammar("math.tpeg")
parser = generate(peg)
tree = parser('1+')
self.assertEqual(str(tree), '1')
self.assertEqual(repr(tree), "[#Int '1']")
def getParser():
global CJParser
if CJParser is None:
CJParser = pg.generate(pg.grammar('cj0.tpeg'))
return CJParser
