def test_expand1(self): g = Lark("""start: a ?a: b b: "x" """) r = g.parse('x') self.assertEqual(r.children[0].data, "b") g = Lark("""start: a ?a: b -> c b: "x" """) r = g.parse('x') self.assertEqual(r.children[0].data, "c") g = Lark("""start: a ?a: B -> c B: "x" """) self.assertEqual(r.children[0].data, "c") g = Lark("""start: a ?a: b b -> c b: "x" """) r = g.parse('xx') self.assertEqual(r.children[0].data, "c")
def test_earley_prioritization_sum(self): "Tests effect of priority on result" grammar = """ start: ab_ b_ a_ | indirection indirection: a_ bb_ a_ a_: "a" b_: "b" ab_: "ab" bb_.1: "bb" """ l = Lark(grammar, ambiguity='resolve__antiscore_sum') res = l.parse('abba') self.assertEqual(''.join(child.data for child in res.children), 'ab_b_a_') grammar = """ start: ab_ b_ a_ | indirection indirection: a_ bb_ a_ a_: "a" b_: "b" ab_.1: "ab" bb_: "bb" """ l = Lark(grammar, ambiguity='resolve__antiscore_sum') res = l.parse('abba') self.assertEqual(''.join(child.data for child in res.children), 'indirection') grammar = """ start: ab_ b_ a_ | indirection indirection: a_ bb_ a_ a_.2: "a" b_.1: "b" ab_.3: "ab" bb_.3: "bb" """ l = Lark(grammar, ambiguity='resolve__antiscore_sum') res = l.parse('abba') self.assertEqual(''.join(child.data for child in res.children), 'ab_b_a_') grammar = """ start: ab_ b_ a_ | indirection indirection: a_ bb_ a_ a_.1: "a" b_.1: "b" ab_.4: "ab" bb_.3: "bb" """ l = Lark(grammar, ambiguity='resolve__antiscore_sum') res = l.parse('abba') self.assertEqual(''.join(child.data for child in res.children), 'indirection')
def test_same_ast(self): "Tests that Earley and LALR parsers produce equal trees" g = Lark("""start: "(" name_list ("," "*" NAME)? ")" name_list: NAME | name_list "," NAME NAME: /\w+/ """, parser='lalr') l = g.parse('(a,b,c,*x)') g = Lark("""start: "(" name_list ("," "*" NAME)? ")" name_list: NAME | name_list "," NAME NAME: /\w/+ """) l2 = g.parse('(a,b,c,*x)') assert l == l2, '%s != %s' % (l.pretty(), l2.pretty())
def puzzle1(): rules, lines = open("day19.txt").read().split('\n\n') rules = rules.translate(str.maketrans('0123456789', 'abcdefghij')) parser = Lark(rules, start='a') total = 0 for line in lines.splitlines(): try: parser.parse(line) total += 1 except LarkError: pass return total
def test_earley2(self): grammar = """ start: statement+ statement: "r" | "c" /[a-z]/+ %ignore " " """ program = """c b r""" l = Lark(grammar, parser='earley', lexer=LEXER) l.parse(program)
def test_fruitflies_ambig(self): grammar = """ start: noun verb noun -> simple | noun verb "like" noun -> comparative noun: adj? NOUN verb: VERB adj: ADJ NOUN: "flies" | "bananas" | "fruit" VERB: "like" | "flies" ADJ: "fruit" %import common.WS %ignore WS """ parser = Lark(grammar, ambiguity='explicit', lexer=LEXER) tree = parser.parse('fruit flies like bananas') expected = Tree('_ambig', [ Tree('comparative', [ Tree('noun', ['fruit']), Tree('verb', ['flies']), Tree('noun', ['bananas']) ]), Tree('simple', [ Tree('noun', [Tree('adj', ['fruit']), 'flies']), Tree('verb', ['like']), Tree('noun', ['bananas']) ]) ]) # self.assertEqual(tree, expected) self.assertEqual(tree.data, expected.data) self.assertEqual(set(tree.children), set(expected.children))
def test_propagate_positions(self): g = Lark("""start: a a: "a" """, propagate_positions=True) r = g.parse('a') self.assertEqual( r.children[0].line, 1 )
def test_earley_scanless(self): g = Lark("""start: A "b" c A: "a"+ c: "abc" """, parser="earley", lexer=None) x = g.parse('aaaababc')
def test_earley(self): g = Lark("""start: A "b" c A: "a"+ c: "abc" """, parser="earley", lexer=LEXER) x = g.parse('aaaababc')
def test_anon_in_scanless(self): # Fails an Earley implementation without special handling for empty rules, # or re-processing of already completed rules. g = Lark(r"""start: B B: ("ab"|/[^b]/)* """, lexer=LEXER) self.assertEqual( g.parse('abc').children[0], 'abc')
def test_earley4(self): grammar = """ start: A A? A: "a"+ """ l = Lark(grammar, parser='earley', lexer=LEXER) res = l.parse("aaa") assert set(res.children) == {'aa', 'a'} or res.children == ['aaa']
def test_earley4(self): grammar = """ start: A A? A: "a"+ """ l = Lark(grammar, parser='earley', lexer=LEXER) res = l.parse("aaa") self.assertEqual(res.children, ['aaa'])
def test_term_ambig_resolve(self): grammar = r""" !start: NAME+ NAME: /\w+/ %ignore " " """ text = """foo bar""" parser = Lark(grammar) tree = parser.parse(text) self.assertEqual(tree.children, ['foo', 'bar'])
def test_earley3(self): "Tests prioritization and disambiguation for pseudo-terminals (there should be only one result)" grammar = """ start: A A A: "a"+ """ l = Lark(grammar, parser='earley', lexer=LEXER) res = l.parse("aaa") self.assertEqual(res.children, ['aa', 'a'])
def test_earley3(self): """Tests prioritization and disambiguation for pseudo-terminals (there should be only one result) By default, `+` should immitate regexp greedy-matching """ grammar = """ start: A A A: "a"+ """ l = Lark(grammar, parser='earley', lexer=LEXER) res = l.parse("aaa") self.assertEqual(set(res.children), {'aa', 'a'})
def test_ambiguity1(self): grammar = """ start: cd+ "e" !cd: "c" | "d" | "cd" """ l = Lark(grammar, parser='earley', ambiguity='explicit', lexer=LEXER) x = l.parse('cde') assert x.data == '_ambig', x assert len(x.children) == 2
def test_earley_prioritization(self): "Tests effect of priority on result" grammar = """ start: a | b a.1: "a" b.2: "a" """ l = Lark(grammar, parser='earley', lexer='standard') res = l.parse("a") self.assertEqual(res.children[0].data, 'b') grammar = """ start: a | b a.2: "a" b.1: "a" """ l = Lark(grammar, parser='earley', lexer='standard') res = l.parse("a") self.assertEqual(res.children[0].data, 'a')
def test_not_all_derivations(self): grammar = """ start: cd+ "e" !cd: "c" | "d" | "cd" """ l = Lark(grammar, parser='earley', ambiguity='explicit', lexer=LEXER, earley__all_derivations=False) x = l.parse('cde') assert x.data != '_ambig', x assert len(x.children) == 1
def parse(mod, fname, tests): test_results = TestResults() mod.__dict__.update(BUILTINS) parser = Lark(open(path), parser='lalr', postlex=GrammarIndenter(), transformer=ParseTreeToAST(mod, test_results, fname) ) tree = parser.parse(tests) execute(tree) return test_results.passed, test_results.errors
def process(self): parser = Lark(grammar, propagate_positions = True, parser = 'lalr') lines = self.source.split('\n') parsed = None try: parsed = parser.parse(self.source) except UnexpectedToken as e: self.report_message(e, 'error', f'unexpected token \'{e.token}\'', f'was expecting {expected_to_human_readable(e.expected)} here') except UnexpectedCharacters as e: self.report_message(e, 'error', f'unexpected character \'{lines[e.line - 1][e.column - 1]}\'', f'was expecting {expected_to_human_readable(e.allowed)} here') except UnexpectedEOF as e: self.report_message(eof, 'error', f'unexpected end of file', f'was expecting {expected_to_human_readable(e.expected)} here') self.tokens = IdlTransformer().transform(parsed) for t in self.tokens: if type(t) is Enum: if self.type_registry.is_known_type(t.name): self.report_message(t.type, 'error', f'name {t.name} is already in use.', '') subtype = self.type_registry.get_type(t.type.name) if not subtype: self.report_message(t.type, 'error', f'unknown type for this {t.mode} block', f'{t.type.name} is not a known type', True) if subtype.identity is not TypeIdentity.INTEGER: self.report_message(t.type, 'error', f'{t.name} {t.mode} block\'s type is not an integer', f'{t.type.name} is not an integer') self.type_registry.register_type( Type(t.name, TypeIdentity.CONSTS if t.mode == 'consts' else TypeIdentity.ENUM, fixed_size = subtype.fixed_size, signed = subtype.signed, subtype = subtype) ) t.type = self.type_registry.get_type(t.name) if type(t) is Struct: if self.type_registry.is_known_type(t.name): self.report_message(t.type, 'error', f'name {t.name} is already in use.', '') self.type_registry.register_type( Type(t.name, TypeIdentity.STRUCT, dynamic = True) )
def test_earley_explicit_ambiguity(self): # This was a sneaky bug! grammar = """ start: a b | ab a: "a" b: "b" ab: "ab" """ parser = Lark(grammar, parser='earley', lexer=LEXER, ambiguity='explicit') ambig_tree = parser.parse('ab') self.assertEqual( ambig_tree.data, '_ambig') self.assertEqual( len(ambig_tree.children), 2)
def test_earley_repeating_empty(self): # This was a sneaky bug! grammar = """ !start: "a" empty empty "b" empty: empty2 empty2: """ parser = Lark(grammar, parser='earley', lexer=LEXER) res = parser.parse('ab') empty_tree = Tree('empty', [Tree('empty2', [])]) self.assertSequenceEqual(res.children, ['a', empty_tree, empty_tree, 'b'])
def test_ambiguity2(self): grammar = """ ANY: /[a-zA-Z0-9 ]+/ a.2: "A" b+ b.2: "B" c: ANY start: (a|c)* """ l = Lark(grammar, parser='earley', lexer=LEXER) res = l.parse('ABX') expected = Tree('start', [Tree('a', [Tree('b', [])]), Tree('c', ['X'])]) self.assertEqual(res, expected)
def test_embedded_transformer(self): class T(Transformer): def a(self, children): return "<a>" def b(self, children): return "<b>" def c(self, children): return "<c>" # Test regular g = Lark("""start: a a : "x" """, parser='lalr') r = T().transform(g.parse("x")) self.assertEqual( r.children, ["<a>"] ) g = Lark("""start: a a : "x" """, parser='lalr', transformer=T()) r = g.parse("x") self.assertEqual( r.children, ["<a>"] ) # Test Expand1 g = Lark("""start: a ?a : b b : "x" """, parser='lalr') r = T().transform(g.parse("x")) self.assertEqual( r.children, ["<b>"] ) g = Lark("""start: a ?a : b b : "x" """, parser='lalr', transformer=T()) r = g.parse("x") self.assertEqual( r.children, ["<b>"] ) # Test Expand1 -> Alias g = Lark("""start: a ?a : b b -> c b : "x" """, parser='lalr') r = T().transform(g.parse("xx")) self.assertEqual( r.children, ["<c>"] ) g = Lark("""start: a ?a : b b -> c b : "x" """, parser='lalr', transformer=T()) r = g.parse("xx") self.assertEqual( r.children, ["<c>"] )
def test_explicit_ambiguity2(self): grammar = r""" start: NAME+ NAME: /\w+/ %ignore " " """ text = """cat""" parser = Lark(grammar, start='start', ambiguity='explicit') tree = parser.parse(text) self.assertEqual(tree.data, '_ambig') combinations = { tuple(str(s) for s in t.children) for t in tree.children } self.assertEqual(combinations, {('cat', ), ('ca', 't'), ('c', 'at'), ('c', 'a', 't')})
def add_mismatch_offsets( netlist_in: Union[Path, str], netlist_out: Optional[Union[Path, str]] = None, debug: bool = False, ) -> None: if isinstance(netlist_in, str): netlist_in = Path(netlist_in) if netlist_in.suffix in ['.cdl', '.sp', '.spf']: parser = Lark(grammar_cdl, parser='lalr') scs = False elif netlist_in.suffix in ['.scs', '.net']: parser = Lark(grammar_scs, parser='lalr') scs = True else: raise ValueError( f'Unknown netlist suffix={netlist_in.suffix}. Use ".cdl" or ".scs".' ) lines = read_spectre_cdl_unwrap(netlist_in) lines[-1] += '\n' tree = parser.parse('\n'.join(lines)) if debug: pydot__tree_to_png(tree, "test0.png") obj_list = CktTransformer().transform(tree).children obj_list[-1].last = True if netlist_out is None: netlist_out: Path = netlist_in.with_name(netlist_in.stem + 'out') if isinstance(netlist_out, str): netlist_out: Path = Path(netlist_out) full_netlist = '' used_names = [] offset_map = {} for obj in obj_list: full_netlist += obj.netlist(used_names, offset_map, scs) for key, val in offset_map.items(): print(f'{val}: 0.0') with open_file(netlist_out, 'w') as f: f.write(full_netlist)
class IPDSL(): def __init__(self): self.parser = Lark(ipl_grammar) self.yolo = None self.agender = None self.emotion = None self.food = None self.pspnet = None self.original_img = None self.img_path = '' self.msg = '' self.persons = [] self.objects = [] pass def interpreter(self, s, img, img_path=''): if s.data == 'tuning': tmp = 1 if s.children[0] == 'increase' else -1 attr = s.children[1] val = int(s.children[2]) print(tmp, attr, val) elif s.data == 'set_filter': filter_name = s.children[0] return img_helper.color_filter(img, filter_name) elif s.data == 'flip': flip_dir = s.children[0] if flip_dir == 'vertically': return img_helper.flip_top(img) elif flip_dir == 'horizontally': return img_helper.flip_left(img) elif s.data == 'detect': print(img_path) if self.yolo is None: self.yolo = Yolo() self.agender = Agender() self.emotion = Emotion() if img_path != self.img_path: self.img_path = img_path self.persons = [] self.objects = [] objects = self.yolo.detect(img_path) faces = self.agender.detect(img_path) emotions = self.emotion.detect(img_path) font_path = "/Users/woffee/www/language_design/prototype/font_consolas/CONSOLA.TTF" font = ImageFont.truetype(font_path, 20) fw = 11 fh = 16 msg = [] draw = ImageDraw.Draw(img) for i, box in enumerate(objects): l = box['left'] t = box['top'] b = box['bottom'] r = box['right'] label = box['class'] self.objects.append({ 'left': l, 'top': t, 'bottom': b, 'right': r, 'class': label }) if label == 'person': self.persons.append({ 'left': l, 'top': t, 'bottom': b, 'right': r, 'gender': '', 'age': 0, 'emotion': '' }) else: draw.rectangle(((l, t), (r, b)), outline='blue') txt_width = fw * len(label) draw.rectangle(((l, t), (l + txt_width, t + fh)), fill="blue") draw.text((l, t), label, font=font) for i, box in enumerate(faces): l = box['left'] t = box['top'] b = box['bottom'] r = box['right'] gender = 'male' if box['gender'] < 0.5 else 'female' age = box['age'] label = gender + ", %.2f" % age print(" * Agender: " + label) # msg.append(" * Agender: " + label) score = 0 for i, p in enumerate(self.persons): area = self.computeArea(l, t, r, b, p['left'], p['top'], p['right'], p['bottom']) s = area / ((r - l) * (b - t)) if s > 0.5: self.persons[i]['age'] = age self.persons[i]['gender'] = gender for i, box in enumerate(emotions): l = box['left'] t = box['top'] b = box['bottom'] r = box['right'] emo = box['emotion'] print(" * Emotion: " + emo) # msg.append(" * Emotion: " + emo) for i, p in enumerate(self.persons): area = self.computeArea(l, t, r, b, p['left'], p['top'], p['right'], p['bottom']) if (r - l) * (b - t) > 0: s = area / ((r - l) * (b - t)) if s > 0.5: self.persons[i]['emotion'] = emo # draw.rectangle(((l, t), (r, b)), outline='yellow') # txt_width = fw * len(emo) # draw.rectangle(((l, t), (l + txt_width, t + fh)), fill="black") # draw.text((l, t), emo, font=font) for i, box in enumerate(self.persons): l = box['left'] t = box['top'] b = box['bottom'] r = box['right'] draw.rectangle(((l, t), (r, b)), outline='blue') gender_age = box['gender'] if box['age'] > 0: gender_age = gender_age + ", age %.2f" % box['age'] emotion = box['emotion'] txt_width = fw * len(gender_age) draw.rectangle(((l, t), (l + txt_width, t + fh)), fill="blue") draw.text((l, t), gender_age, font=font) txt_width = fw * len(emotion) draw.rectangle(((l, t + fh), (l + txt_width, t + fh * 2)), fill="blue") draw.text((l, t + fh), emotion, font=font) self.msg = " * done" elif s.data == 'how_many': str = s.children[0] str = self.remove_s(str) num = 0 for obj in self.objects: if obj['class'] == str: num += 1 self.msg = " * %d %s(s)" % (num, str) elif s.data == 'tag_objects': img = self.original_img.copy() str = s.children[0] str = self.remove_s(str) msg = [] draw = ImageDraw.Draw(img) font_path = "/Users/woffee/www/language_design/prototype/font_consolas/CONSOLA.TTF" font = ImageFont.truetype(font_path, 20) fw = 11 fh = 16 for i, box in enumerate(self.objects): if str == box['class']: l = box['left'] t = box['top'] b = box['bottom'] r = box['right'] label = box['class'] print(" * Yolo: " + label) msg.append(" * Yolo: " + label) draw.rectangle(((l, t), (r, b)), outline='blue') txt_width = fw * len(label) draw.rectangle(((l, t), (l + txt_width, t + fh)), fill="blue") draw.text((l, t), label, font=font) self.msg = "\n".join(msg) elif s.data == 'show_statistics': statistics = {} for o in self.objects: if o['class'] not in statistics.keys(): statistics[o['class']] = 1 else: statistics[o['class']] += 1 msg = [] for k in statistics.keys(): v = statistics[k] msg.append(" * %d %s(s)" % (v, k)) self.msg = "\n".join(msg) elif s.data == 'detect_food': self.detect_food(img_path) elif s.data == 'image_segmentation': img = self.image_segmentation(img_path) return img def process(self, img, cmd, img_path): ast = self.parser.parse(cmd) print(ast.pretty()) for s in ast.children: img = self.interpreter(s, img, img_path) return img def computeArea(self, A, B, C, D, E, F, G, H): width = min(C, G) - max(A, E) height = min(D, H) - max(B, F) # width = width if width > 0 else 0 # height = height if height > 0 else 0 # return (C-A)*(D-B) + (G-E)*(H-F) - width*height return width * height def remove_s(self, str): dic = { 'persons': 'person', 'dogs': 'dog', 'cats': 'cat', 'bicycles': 'bicycle', 'birds': 'bird', 'boats': 'boat', 'cars': 'car', 'chairs': 'chair', 'cows': 'cow', 'diningtables': 'diningtable', 'horses': 'horse', 'motorbikes': 'motorbike', 'pottedplants': 'pottedplant', 'sheep': 'sheep', 'sofas': 'sofa', 'trains': 'train', 'tvmonitors': 'tvmonitor', 'trucks': 'truck', } if str in dic.keys(): str = dic[str] return str def detect_food(self, img_path): if self.food is None: self.food = Food() score, name = self.food.detect(img_path) if score > 0.1: self.msg = " * " + name else: self.msg = " * no food" def image_segmentation(self, img_path): if self.pspnet is None: self.pspnet = MyPSPNET() arr = self.pspnet.detect(img_path) arr = arr.astype(np.uint8) img = Image.fromarray(arr) self.msg = " * done" return img
case: "case" test ":" suite // test is not quite correct. %ignore /[\t \f]+/ // WS %ignore /\\[\t \f]*\r?\n/ // LINE_CONT %ignore COMMENT """ parser = Lark(GRAMMAR, parser='lalr', start=['single_input', 'file_input', 'eval_input'], postlex=PythonIndenter()) tree = parser.parse(r""" def name(n): match n: case 1: print("one") case 2: print("two") case _: print("number is too big") """, start='file_input') # Remove the 'python3__' prefix that was added to the implicitly imported rules. for t in tree.iter_subtrees(): t.data = t.data.rsplit('__', 1)[-1] print(tree.pretty())