def load_grammar(options, default=None):
global showingTPEG
expr = options.get('expression', None)
if expr is not None:
grammar = 'A = ' + expr
options['urn'] = f'(-e {repr(expr)})'
return pegtree.grammar(grammar, **options)
file = options.get('grammar', default)
if file is None:
print('Enter a TPEG grammar')
sb = []
try:
while True:
s = input()
if s == '' or s is None:
break
sb.append(s)
except:
pass
data = '\n'.join(sb)
options['urn'] = '(stdin)'
showingTPEG = False
return pegtree.grammar(data, **options)
if file == 'stdin.tpeg':
data = sys.stdin.read()
options['urn'] = file
return pegtree.grammar(data, **options)
options['urn'] = file
return pegtree.grammar(file, **options)
def load_grammar(options, default=None):
file = options.get('grammar', default)
if file is None:
raise CommandUsageError()
if file == 'stdin.tpeg':
data = sys.stdin.read()
options['basepath'] = file
return pegtree.grammar(data, **options)
return pegtree.grammar(file, **options)
def get_parser(start, path):
peg = pg.grammar(path)
if start is None:
parser = pg.generate(peg)
else:
parser = pg.generate(peg, start=start)
return parser
def parse(line, src=''):
start, path = start_option(line.strip())
peg = pg.grammar(path)
if start is None:
parser = pg.generate(peg)
else:
parser = pg.generate(peg, start=start)
tree = parser(src, urn='(stdin)')
print(repr(tree))
return graph.draw_graph(tree)
def __init__(self, grammar='../pegtree/puppy2.tpeg'):
TransCompiler.__init__(self)
peg = pg.grammar(grammar)
self.parser = pg.generate(peg)
self.pycode = PythonCode(grammar)
self.env = {}
self.vocmap = VocabMap(self.env)
self.pycode.set_vocmap(self.vocmap)
self.isBlockMode = True
self.isAllowMath = False
def test_math(self):
peg = grammar("math.tpeg")
parser = generate(peg)
tree = parser('1+2*3')
self.assertEqual(str(tree), '1+2*3')
self.assertEqual(repr(tree),
"[#Add [#Int '1'][#Mul [#Int '2'][#Int '3']]]")
tree = parser('1*2+3')
self.assertEqual(str(tree), '1*2+3')
self.assertEqual(repr(tree),
"[#Add [#Mul [#Int '1'][#Int '2']][#Int '3']]")
def pasm(grammar_file, configuration=''):
"""pasm by converting pegtree grammar_file
Args:
grammar_file (str): PegTree grammar file
configuration (str, optional): JSON-style configuration.
"""
start, path = start_option(grammar_file.strip())
peg = pg.grammar(path)
from .nezcc import parsec
if configuration != '':
config = json.loads(configuration)
else:
config = {}
parsec(peg, **config)
def peg(path_to_save: str, grammar: str):
"""display syntax trees of examples defined in a grammar of code cell
Args:
path_to_save (str): file to save (optional)
grammar (str): a PegTree grammar with example
"""
peg = pg.grammar(grammar)
if '@@error' in peg:
return
test_example(peg, None)
file = path_to_save.strip()
if file.endswith('peg') or file.endswith('.pegtree'):
with open(file, 'w') as f:
f.write(grammar)
print(f'wrote {file}')
def apply(self, dict_or_func=identity):
if isinstance(dict_or_func, dict):
if self.index in dict_or_func:
return dict_or_func[self.index]
if self.w in dict_or_func:
return dict_or_func[self.w]
return self
else:
return dict_or_func(self)
def emit(self, buffers):
buffers.append(self.w)
peg = pg.grammar('tokibi.pegtree')
tokibi_parser = pg.generate(peg)
class TokibiReader(ParseTreeVisitor):
def __init__(self, synonyms=None):
ParseTreeVisitor.__init__(self)
self.indexes = {}
self.synonyms = {} if synonyms is None else synonyms
def parse(self, s):
tree = tokibi_parser(s)
self.indexes = {}
nexpr = self.visit(tree)
return nexpr, self.indexes
import sys
import pegtree as pg
tpeg = pg.grammar('pynmt.pegtree')
parser = pg.generate(tpeg, start = 'Checker')
parser2 = pg.generate(tpeg, start = 'Statement')
def check_syntax(s: str):
"""
Python の構文的に正しいかどうか
"""
tree = parser(s.strip())
# print(repr(tree))
# return not tree.isSyntaxError()
return tree.isSyntaxError()
def fix(tree):
a = [tree.epos_]
for t in tree:
fix(t)
a.append(t.epos_)
for key in tree.keys():
a.append(fix(tree.get(key)).epos_)
tree.epos_ = max(a)
return tree
def filter_syntax(s: str):
"""
Pythonの構文的に正しいところまで取り出す。
"""
tree = parser2(s.strip())
if buffer is None:
ss.append(emit(p, verb.THEN))
else:
buffer.append(emit(p))
alias = longer(alias, alt(self.cat))
for p in params:
ss.append(emit(p))
#print(type(self.pieces[-1]), self.pieces)
ss.append(emit(self.pieces[-1], mode, alias, buffer))
return ' '.join(ss)
##
peg = pg.grammar('tokibi.tpeg')
tokibi_parser = pg.generate(peg)
class TokibiReader(ParseTreeVisitor):
def __init__(self, synonyms=None):
ParseTreeVisitor.__init__(self)
self.indexes = {}
self.synonyms = {} if synonyms is None else synonyms
def parse(self, s):
tree = tokibi_parser(s)
self.indexes = {}
nexpr = self.visit(tree)
return nexpr, self.indexes
def __init__(self, grammar='puppy2.tpeg'):
self.buffers = []
peg = pg.grammar(grammar)
self.parser = pg.generate(peg)
import pegtree as pg
puppy_peg = pg.grammar('puppy2.tpeg')
puppy_parser = pg.generate(puppy_peg)
METHODS = {}
FUNCDATA = {
'print': {
'stem': '表示する',
0: '空行を',
1: '{}を',
'end=': '{}を行末として',
'end=""': '改行なしで',
'seq=': '{}を区切り文字列として',
},
'+': {
'stem': '加えた値',
2: '{}を{}に',
},
'+=': {
'stem': '増やす',
2: '{}を{}だけ',
}
}
class Transpiler(object):
buffers: list
def __init__(self):
import pegtree as pg
from pegtree import ParseTree
from pegtree.visitor import ParseTreeVisitor
import tree as ntree
import pprint
peg = pg.grammar('multiese.pegtree')
parser = pg.generate(peg)
def fix(tree):
a = [tree.epos_]
for t in tree:
fix(t)
a.append(t.epos_)
for key in tree.keys():
a.append(fix(tree.get(key)).epos_)
tree.epos_ = max(a)
return tree
class MultieseParser(ParseTreeVisitor):
def __init__(self):
ParseTreeVisitor.__init__(self)
def parse(self, s: str):
tree = parser(s)
node = self.visit(tree)
return node
def acceptChunk(self, tree: ParseTree):
def __init__(self, grammar='../pegtree/puppy2.tpeg'):
TransCompiler.__init__(self)
peg = pg.grammar(grammar)
self.parser = pg.generate(peg)
self.vocmap = None
self.isTrimString = False
import sys
import pegtree as pg
tpeg = pg.grammar('pyline.pegtree')
parser = pg.generate(tpeg)
def fix(tree):
a = [tree.epos_]
for t in tree:
fix(t)
a.append(t.epos_)
for key in tree.keys():
a.append(fix(tree.get(key)).epos_)
tree.epos_ = max(a)
return tree
def readfile(filename, head=None):
with open(filename, 'r') as f:
for i, line in enumerate(f.readlines()):
tree = parser(line)
if len(tree) > 0:
for t in tree:
if t.tag_ != 'Name':
print(fix(t))
#print(repr(fix(t)))
if i == head:
break
def example(line, src=''):
start, path = start_option(line.strip())
peg = pg.grammar(path)
test_example(peg, start)
def match(line, src):
peg = pg.grammar(src)
if '@error' not in peg:
parser = pg.generate(peg)
res = parser(line)
print(repr(res))
def load_grammar(files, options):
peg = pg.grammar(files[0])
return peg
from os import read
import random
import sys
import pegtree as pg
import argparse
import csv
from pegtree.optimizer import optimize
peg = pg.grammar('yk.tpeg')
parse = pg.generate(peg)
parser = argparse.ArgumentParser(description='yk for Parameter Handling')
parser.add_argument('--notConv', action='store_true') # Python のトークナイズのみ
parser.add_argument('--diff',
action='store_true') # 変数名 (name) とリテラル (val) に異なるものを付与
parser.add_argument('--shuffle',
action='store_true') # 特殊トークンをランダムに付与 (順序を考慮しない)
parser.add_argument('--both', action='store_true') # shuffle ありとなしを両方追加
parser.add_argument('--files', nargs='*') # 入力ファイルを与える
args = parser.parse_args()
token_idx = list(range(1, 7))
def replace_as_special_parameter(s,
mapped,
token_idx=token_idx,
tag=None): # mapped => {'df': '<A>'}
if s in mapped:
def __init__(self, grammar='../pegtree/puppy2.tpeg'):
peg = pg.grammar(grammar)
self.parser = pg.generate(peg)
self.rootEnv = {}
CEMPTY = CEmpty()
class CSlice(CExpr):
def __init__(self, recv, start=CEMPTY, stop=CEMPTY, step=CEMPTY):
CExpr.__init__(
self, "[]",
(toCExpr(recv), toCExpr(start), toCExpr(stop), toCExpr(step)))
def format(self):
return f'{{}} [ {{}} : {{}} : {{}}]'
##
peg = pg.grammar('kotoha.tpeg')
parser = pg.generate(peg)
snipet_parser = pg.generate(peg, start='Snipet')
class Reader(ParseTreeVisitor):
rules: dict
indexes: dict
newnames: set
def __init__(self, rules):
ParseTreeVisitor.__init__(self)
self.rules = rules
self.symbols = EMPTY
self.names = {}
self.modules = STATIC_MODULE
def format(self):
return ''
CEMPTY = CEmpty()
class CSlice(CExpr):
def __init__(self, recv, start=CEMPTY, stop=CEMPTY, step=CEMPTY):
CExpr.__init__(self, "[]", (toCExpr(recv),
toCExpr(start), toCExpr(stop), toCExpr(step)))
def format(self):
return f'{{}} [ {{}} : {{}} : {{}}]'
##
peg = pg.grammar('kotoha.pegtree')
snipet_parser = pg.generate(peg, start='Snipet')
class CodeParser(ParseTreeVisitor):
def __init__(self):
ParseTreeVisitor.__init__(self)
self.symbols = EMPTY
self.modules = STATIC_MODULE
# self.names = {}
# self.synonyms = {}
# self.newnames = set()
def parse(self, source, symbols=EMPTY):
self.symbols = symbols
tree = snipet_parser(source)
def test_int(self):
peg = grammar("math.tpeg")
parser = generate(peg)
tree = parser('1+')
self.assertEqual(str(tree), '1')
self.assertEqual(repr(tree), "[#Int '1']")
def getParser():
global CJParser
if CJParser is None:
CJParser = pg.generate(pg.grammar('cj0.tpeg'))
return CJParser
