def test_expand1(self):
g = Lark("""start: a
?a: b
b: "x"
""")
r = g.parse('x')
self.assertEqual(r.children[0].data, "b")
g = Lark("""start: a
?a: b -> c
b: "x"
""")
r = g.parse('x')
self.assertEqual(r.children[0].data, "c")
g = Lark("""start: a
?a: B -> c
B: "x"
""")
self.assertEqual(r.children[0].data, "c")
g = Lark("""start: a
?a: b b -> c
b: "x"
""")
r = g.parse('xx')
self.assertEqual(r.children[0].data, "c")
def test_earley_prioritization_sum(self):
"Tests effect of priority on result"
grammar = """
start: ab_ b_ a_ | indirection
indirection: a_ bb_ a_
a_: "a"
b_: "b"
ab_: "ab"
bb_.1: "bb"
"""
l = Lark(grammar, ambiguity='resolve__antiscore_sum')
res = l.parse('abba')
self.assertEqual(''.join(child.data for child in res.children),
'ab_b_a_')
grammar = """
start: ab_ b_ a_ | indirection
indirection: a_ bb_ a_
a_: "a"
b_: "b"
ab_.1: "ab"
bb_: "bb"
"""
l = Lark(grammar, ambiguity='resolve__antiscore_sum')
res = l.parse('abba')
self.assertEqual(''.join(child.data for child in res.children),
'indirection')
grammar = """
start: ab_ b_ a_ | indirection
indirection: a_ bb_ a_
a_.2: "a"
b_.1: "b"
ab_.3: "ab"
bb_.3: "bb"
"""
l = Lark(grammar, ambiguity='resolve__antiscore_sum')
res = l.parse('abba')
self.assertEqual(''.join(child.data for child in res.children),
'ab_b_a_')
grammar = """
start: ab_ b_ a_ | indirection
indirection: a_ bb_ a_
a_.1: "a"
b_.1: "b"
ab_.4: "ab"
bb_.3: "bb"
"""
l = Lark(grammar, ambiguity='resolve__antiscore_sum')
res = l.parse('abba')
self.assertEqual(''.join(child.data for child in res.children),
'indirection')
def test_same_ast(self):
"Tests that Earley and LALR parsers produce equal trees"
g = Lark("""start: "(" name_list ("," "*" NAME)? ")"
name_list: NAME | name_list "," NAME
NAME: /\w+/ """, parser='lalr')
l = g.parse('(a,b,c,*x)')
g = Lark("""start: "(" name_list ("," "*" NAME)? ")"
name_list: NAME | name_list "," NAME
NAME: /\w/+ """)
l2 = g.parse('(a,b,c,*x)')
assert l == l2, '%s != %s' % (l.pretty(), l2.pretty())
def puzzle1():
rules, lines = open("day19.txt").read().split('\n\n')
rules = rules.translate(str.maketrans('0123456789', 'abcdefghij'))
parser = Lark(rules, start='a')
total = 0
for line in lines.splitlines():
try:
parser.parse(line)
total += 1
except LarkError:
pass
return total
def test_earley2(self):
grammar = """
start: statement+
statement: "r"
| "c" /[a-z]/+
%ignore " "
"""
program = """c b r"""
l = Lark(grammar, parser='earley', lexer=LEXER)
l.parse(program)
def test_fruitflies_ambig(self):
grammar = """
start: noun verb noun -> simple
| noun verb "like" noun -> comparative
noun: adj? NOUN
verb: VERB
adj: ADJ
NOUN: "flies" | "bananas" | "fruit"
VERB: "like" | "flies"
ADJ: "fruit"
%import common.WS
%ignore WS
"""
parser = Lark(grammar, ambiguity='explicit', lexer=LEXER)
tree = parser.parse('fruit flies like bananas')
expected = Tree('_ambig', [
Tree('comparative', [
Tree('noun', ['fruit']),
Tree('verb', ['flies']),
Tree('noun', ['bananas'])
]),
Tree('simple', [
Tree('noun', [Tree('adj', ['fruit']), 'flies']),
Tree('verb', ['like']),
Tree('noun', ['bananas'])
])
])
# self.assertEqual(tree, expected)
self.assertEqual(tree.data, expected.data)
self.assertEqual(set(tree.children), set(expected.children))
def test_propagate_positions(self):
g = Lark("""start: a
a: "a"
""", propagate_positions=True)
r = g.parse('a')
self.assertEqual( r.children[0].line, 1 )
def test_earley_scanless(self):
g = Lark("""start: A "b" c
A: "a"+
c: "abc"
""",
parser="earley",
lexer=None)
x = g.parse('aaaababc')
def test_earley(self):
g = Lark("""start: A "b" c
A: "a"+
c: "abc"
""",
parser="earley",
lexer=LEXER)
x = g.parse('aaaababc')
def test_anon_in_scanless(self):
# Fails an Earley implementation without special handling for empty rules,
# or re-processing of already completed rules.
g = Lark(r"""start: B
B: ("ab"|/[^b]/)*
""", lexer=LEXER)
self.assertEqual( g.parse('abc').children[0], 'abc')
def test_earley4(self):
grammar = """
start: A A?
A: "a"+
"""
l = Lark(grammar, parser='earley', lexer=LEXER)
res = l.parse("aaa")
assert set(res.children) == {'aa', 'a'} or res.children == ['aaa']
def test_earley4(self):
grammar = """
start: A A?
A: "a"+
"""
l = Lark(grammar, parser='earley', lexer=LEXER)
res = l.parse("aaa")
self.assertEqual(res.children, ['aaa'])
def test_term_ambig_resolve(self):
grammar = r"""
!start: NAME+
NAME: /\w+/
%ignore " "
"""
text = """foo bar"""
parser = Lark(grammar)
tree = parser.parse(text)
self.assertEqual(tree.children, ['foo', 'bar'])
def test_earley3(self):
"Tests prioritization and disambiguation for pseudo-terminals (there should be only one result)"
grammar = """
start: A A
A: "a"+
"""
l = Lark(grammar, parser='earley', lexer=LEXER)
res = l.parse("aaa")
self.assertEqual(res.children, ['aa', 'a'])
def test_earley3(self):
"""Tests prioritization and disambiguation for pseudo-terminals (there should be only one result)
By default, `+` should immitate regexp greedy-matching
"""
grammar = """
start: A A
A: "a"+
"""
l = Lark(grammar, parser='earley', lexer=LEXER)
res = l.parse("aaa")
self.assertEqual(set(res.children), {'aa', 'a'})
def test_ambiguity1(self):
grammar = """
start: cd+ "e"
!cd: "c"
| "d"
| "cd"
"""
l = Lark(grammar, parser='earley', ambiguity='explicit', lexer=LEXER)
x = l.parse('cde')
assert x.data == '_ambig', x
assert len(x.children) == 2
def test_earley_prioritization(self):
"Tests effect of priority on result"
grammar = """
start: a | b
a.1: "a"
b.2: "a"
"""
l = Lark(grammar, parser='earley', lexer='standard')
res = l.parse("a")
self.assertEqual(res.children[0].data, 'b')
grammar = """
start: a | b
a.2: "a"
b.1: "a"
"""
l = Lark(grammar, parser='earley', lexer='standard')
res = l.parse("a")
self.assertEqual(res.children[0].data, 'a')
def test_not_all_derivations(self):
grammar = """
start: cd+ "e"
!cd: "c"
| "d"
| "cd"
"""
l = Lark(grammar, parser='earley', ambiguity='explicit', lexer=LEXER, earley__all_derivations=False)
x = l.parse('cde')
assert x.data != '_ambig', x
assert len(x.children) == 1
def parse(mod, fname, tests):
test_results = TestResults()
mod.__dict__.update(BUILTINS)
parser = Lark(open(path),
parser='lalr',
postlex=GrammarIndenter(),
transformer=ParseTreeToAST(mod, test_results, fname)
)
tree = parser.parse(tests)
execute(tree)
return test_results.passed, test_results.errors
def process(self):
parser = Lark(grammar, propagate_positions = True, parser = 'lalr')
lines = self.source.split('\n')
parsed = None
try:
parsed = parser.parse(self.source)
except UnexpectedToken as e:
self.report_message(e, 'error',
f'unexpected token \'{e.token}\'',
f'was expecting {expected_to_human_readable(e.expected)} here')
except UnexpectedCharacters as e:
self.report_message(e, 'error',
f'unexpected character \'{lines[e.line - 1][e.column - 1]}\'',
f'was expecting {expected_to_human_readable(e.allowed)} here')
except UnexpectedEOF as e:
self.report_message(eof, 'error',
f'unexpected end of file',
f'was expecting {expected_to_human_readable(e.expected)} here')
self.tokens = IdlTransformer().transform(parsed)
for t in self.tokens:
if type(t) is Enum:
if self.type_registry.is_known_type(t.name):
self.report_message(t.type, 'error', f'name {t.name} is already in use.', '')
subtype = self.type_registry.get_type(t.type.name)
if not subtype:
self.report_message(t.type, 'error', f'unknown type for this {t.mode} block', f'{t.type.name} is not a known type', True)
if subtype.identity is not TypeIdentity.INTEGER:
self.report_message(t.type, 'error', f'{t.name} {t.mode} block\'s type is not an integer', f'{t.type.name} is not an integer')
self.type_registry.register_type(
Type(t.name,
TypeIdentity.CONSTS if t.mode == 'consts' else TypeIdentity.ENUM,
fixed_size = subtype.fixed_size,
signed = subtype.signed,
subtype = subtype)
)
t.type = self.type_registry.get_type(t.name)
if type(t) is Struct:
if self.type_registry.is_known_type(t.name):
self.report_message(t.type, 'error', f'name {t.name} is already in use.', '')
self.type_registry.register_type(
Type(t.name,
TypeIdentity.STRUCT,
dynamic = True)
)
def test_earley_explicit_ambiguity(self):
# This was a sneaky bug!
grammar = """
start: a b | ab
a: "a"
b: "b"
ab: "ab"
"""
parser = Lark(grammar, parser='earley', lexer=LEXER, ambiguity='explicit')
ambig_tree = parser.parse('ab')
self.assertEqual( ambig_tree.data, '_ambig')
self.assertEqual( len(ambig_tree.children), 2)
def test_earley_repeating_empty(self):
# This was a sneaky bug!
grammar = """
!start: "a" empty empty "b"
empty: empty2
empty2:
"""
parser = Lark(grammar, parser='earley', lexer=LEXER)
res = parser.parse('ab')
empty_tree = Tree('empty', [Tree('empty2', [])])
self.assertSequenceEqual(res.children, ['a', empty_tree, empty_tree, 'b'])
def test_ambiguity2(self):
grammar = """
ANY: /[a-zA-Z0-9 ]+/
a.2: "A" b+
b.2: "B"
c: ANY
start: (a|c)*
"""
l = Lark(grammar, parser='earley', lexer=LEXER)
res = l.parse('ABX')
expected = Tree('start',
[Tree('a', [Tree('b', [])]),
Tree('c', ['X'])])
self.assertEqual(res, expected)
def test_embedded_transformer(self):
class T(Transformer):
def a(self, children):
return "<a>"
def b(self, children):
return "<b>"
def c(self, children):
return "<c>"
# Test regular
g = Lark("""start: a
a : "x"
""", parser='lalr')
r = T().transform(g.parse("x"))
self.assertEqual( r.children, ["<a>"] )
g = Lark("""start: a
a : "x"
""", parser='lalr', transformer=T())
r = g.parse("x")
self.assertEqual( r.children, ["<a>"] )
# Test Expand1
g = Lark("""start: a
?a : b
b : "x"
""", parser='lalr')
r = T().transform(g.parse("x"))
self.assertEqual( r.children, ["<b>"] )
g = Lark("""start: a
?a : b
b : "x"
""", parser='lalr', transformer=T())
r = g.parse("x")
self.assertEqual( r.children, ["<b>"] )
# Test Expand1 -> Alias
g = Lark("""start: a
?a : b b -> c
b : "x"
""", parser='lalr')
r = T().transform(g.parse("xx"))
self.assertEqual( r.children, ["<c>"] )
g = Lark("""start: a
?a : b b -> c
b : "x"
""", parser='lalr', transformer=T())
r = g.parse("xx")
self.assertEqual( r.children, ["<c>"] )
def test_explicit_ambiguity2(self):
grammar = r"""
start: NAME+
NAME: /\w+/
%ignore " "
"""
text = """cat"""
parser = Lark(grammar, start='start', ambiguity='explicit')
tree = parser.parse(text)
self.assertEqual(tree.data, '_ambig')
combinations = {
tuple(str(s) for s in t.children)
for t in tree.children
}
self.assertEqual(combinations, {('cat', ), ('ca', 't'),
('c', 'at'), ('c', 'a', 't')})
def add_mismatch_offsets(
netlist_in: Union[Path, str],
netlist_out: Optional[Union[Path, str]] = None,
debug: bool = False,
) -> None:
if isinstance(netlist_in, str):
netlist_in = Path(netlist_in)
if netlist_in.suffix in ['.cdl', '.sp', '.spf']:
parser = Lark(grammar_cdl, parser='lalr')
scs = False
elif netlist_in.suffix in ['.scs', '.net']:
parser = Lark(grammar_scs, parser='lalr')
scs = True
else:
raise ValueError(
f'Unknown netlist suffix={netlist_in.suffix}. Use ".cdl" or ".scs".'
)
lines = read_spectre_cdl_unwrap(netlist_in)
lines[-1] += '\n'
tree = parser.parse('\n'.join(lines))
if debug:
pydot__tree_to_png(tree, "test0.png")
obj_list = CktTransformer().transform(tree).children
obj_list[-1].last = True
if netlist_out is None:
netlist_out: Path = netlist_in.with_name(netlist_in.stem + 'out')
if isinstance(netlist_out, str):
netlist_out: Path = Path(netlist_out)
full_netlist = ''
used_names = []
offset_map = {}
for obj in obj_list:
full_netlist += obj.netlist(used_names, offset_map, scs)
for key, val in offset_map.items():
print(f'{val}: 0.0')
with open_file(netlist_out, 'w') as f:
f.write(full_netlist)
class IPDSL():
def __init__(self):
self.parser = Lark(ipl_grammar)
self.yolo = None
self.agender = None
self.emotion = None
self.food = None
self.pspnet = None
self.original_img = None
self.img_path = ''
self.msg = ''
self.persons = []
self.objects = []
pass
def interpreter(self, s, img, img_path=''):
if s.data == 'tuning':
tmp = 1 if s.children[0] == 'increase' else -1
attr = s.children[1]
val = int(s.children[2])
print(tmp, attr, val)
elif s.data == 'set_filter':
filter_name = s.children[0]
return img_helper.color_filter(img, filter_name)
elif s.data == 'flip':
flip_dir = s.children[0]
if flip_dir == 'vertically':
return img_helper.flip_top(img)
elif flip_dir == 'horizontally':
return img_helper.flip_left(img)
elif s.data == 'detect':
print(img_path)
if self.yolo is None:
self.yolo = Yolo()
self.agender = Agender()
self.emotion = Emotion()
if img_path != self.img_path:
self.img_path = img_path
self.persons = []
self.objects = []
objects = self.yolo.detect(img_path)
faces = self.agender.detect(img_path)
emotions = self.emotion.detect(img_path)
font_path = "/Users/woffee/www/language_design/prototype/font_consolas/CONSOLA.TTF"
font = ImageFont.truetype(font_path, 20)
fw = 11
fh = 16
msg = []
draw = ImageDraw.Draw(img)
for i, box in enumerate(objects):
l = box['left']
t = box['top']
b = box['bottom']
r = box['right']
label = box['class']
self.objects.append({
'left': l,
'top': t,
'bottom': b,
'right': r,
'class': label
})
if label == 'person':
self.persons.append({
'left': l,
'top': t,
'bottom': b,
'right': r,
'gender': '',
'age': 0,
'emotion': ''
})
else:
draw.rectangle(((l, t), (r, b)), outline='blue')
txt_width = fw * len(label)
draw.rectangle(((l, t), (l + txt_width, t + fh)),
fill="blue")
draw.text((l, t), label, font=font)
for i, box in enumerate(faces):
l = box['left']
t = box['top']
b = box['bottom']
r = box['right']
gender = 'male' if box['gender'] < 0.5 else 'female'
age = box['age']
label = gender + ", %.2f" % age
print(" * Agender: " + label)
# msg.append(" * Agender: " + label)
score = 0
for i, p in enumerate(self.persons):
area = self.computeArea(l, t, r, b, p['left'], p['top'],
p['right'], p['bottom'])
s = area / ((r - l) * (b - t))
if s > 0.5:
self.persons[i]['age'] = age
self.persons[i]['gender'] = gender
for i, box in enumerate(emotions):
l = box['left']
t = box['top']
b = box['bottom']
r = box['right']
emo = box['emotion']
print(" * Emotion: " + emo)
# msg.append(" * Emotion: " + emo)
for i, p in enumerate(self.persons):
area = self.computeArea(l, t, r, b, p['left'], p['top'],
p['right'], p['bottom'])
if (r - l) * (b - t) > 0:
s = area / ((r - l) * (b - t))
if s > 0.5:
self.persons[i]['emotion'] = emo
# draw.rectangle(((l, t), (r, b)), outline='yellow')
# txt_width = fw * len(emo)
# draw.rectangle(((l, t), (l + txt_width, t + fh)), fill="black")
# draw.text((l, t), emo, font=font)
for i, box in enumerate(self.persons):
l = box['left']
t = box['top']
b = box['bottom']
r = box['right']
draw.rectangle(((l, t), (r, b)), outline='blue')
gender_age = box['gender']
if box['age'] > 0:
gender_age = gender_age + ", age %.2f" % box['age']
emotion = box['emotion']
txt_width = fw * len(gender_age)
draw.rectangle(((l, t), (l + txt_width, t + fh)), fill="blue")
draw.text((l, t), gender_age, font=font)
txt_width = fw * len(emotion)
draw.rectangle(((l, t + fh), (l + txt_width, t + fh * 2)),
fill="blue")
draw.text((l, t + fh), emotion, font=font)
self.msg = " * done"
elif s.data == 'how_many':
str = s.children[0]
str = self.remove_s(str)
num = 0
for obj in self.objects:
if obj['class'] == str:
num += 1
self.msg = " * %d %s(s)" % (num, str)
elif s.data == 'tag_objects':
img = self.original_img.copy()
str = s.children[0]
str = self.remove_s(str)
msg = []
draw = ImageDraw.Draw(img)
font_path = "/Users/woffee/www/language_design/prototype/font_consolas/CONSOLA.TTF"
font = ImageFont.truetype(font_path, 20)
fw = 11
fh = 16
for i, box in enumerate(self.objects):
if str == box['class']:
l = box['left']
t = box['top']
b = box['bottom']
r = box['right']
label = box['class']
print(" * Yolo: " + label)
msg.append(" * Yolo: " + label)
draw.rectangle(((l, t), (r, b)), outline='blue')
txt_width = fw * len(label)
draw.rectangle(((l, t), (l + txt_width, t + fh)),
fill="blue")
draw.text((l, t), label, font=font)
self.msg = "\n".join(msg)
elif s.data == 'show_statistics':
statistics = {}
for o in self.objects:
if o['class'] not in statistics.keys():
statistics[o['class']] = 1
else:
statistics[o['class']] += 1
msg = []
for k in statistics.keys():
v = statistics[k]
msg.append(" * %d %s(s)" % (v, k))
self.msg = "\n".join(msg)
elif s.data == 'detect_food':
self.detect_food(img_path)
elif s.data == 'image_segmentation':
img = self.image_segmentation(img_path)
return img
def process(self, img, cmd, img_path):
ast = self.parser.parse(cmd)
print(ast.pretty())
for s in ast.children:
img = self.interpreter(s, img, img_path)
return img
def computeArea(self, A, B, C, D, E, F, G, H):
width = min(C, G) - max(A, E)
height = min(D, H) - max(B, F)
# width = width if width > 0 else 0
# height = height if height > 0 else 0
# return (C-A)*(D-B) + (G-E)*(H-F) - width*height
return width * height
def remove_s(self, str):
dic = {
'persons': 'person',
'dogs': 'dog',
'cats': 'cat',
'bicycles': 'bicycle',
'birds': 'bird',
'boats': 'boat',
'cars': 'car',
'chairs': 'chair',
'cows': 'cow',
'diningtables': 'diningtable',
'horses': 'horse',
'motorbikes': 'motorbike',
'pottedplants': 'pottedplant',
'sheep': 'sheep',
'sofas': 'sofa',
'trains': 'train',
'tvmonitors': 'tvmonitor',
'trucks': 'truck',
}
if str in dic.keys():
str = dic[str]
return str
def detect_food(self, img_path):
if self.food is None:
self.food = Food()
score, name = self.food.detect(img_path)
if score > 0.1:
self.msg = " * " + name
else:
self.msg = " * no food"
def image_segmentation(self, img_path):
if self.pspnet is None:
self.pspnet = MyPSPNET()
arr = self.pspnet.detect(img_path)
arr = arr.astype(np.uint8)
img = Image.fromarray(arr)
self.msg = " * done"
return img
case: "case" test ":" suite // test is not quite correct.
%ignore /[\t \f]+/ // WS
%ignore /\\[\t \f]*\r?\n/ // LINE_CONT
%ignore COMMENT
"""
parser = Lark(GRAMMAR,
parser='lalr',
start=['single_input', 'file_input', 'eval_input'],
postlex=PythonIndenter())
tree = parser.parse(r"""
def name(n):
match n:
case 1:
print("one")
case 2:
print("two")
case _:
print("number is too big")
""",
start='file_input')
# Remove the 'python3__' prefix that was added to the implicitly imported rules.
for t in tree.iter_subtrees():
t.data = t.data.rsplit('__', 1)[-1]
print(tree.pretty())
