def test_str(self): self.assertEqual(tokenize('"a\\"b\\_c"'), [Token('STRING', '"a\\"b\\_c"'), Token('NL', '')]) self.assertEqual(tokenize("'a\\'b\\_c'"), [Token('STRING', "'a\\'b\\_c'"), Token('NL', '')])
def test_intermediate_dedent_to_zero(self): tokens = tokenize('\n a\n b\n c\n d\nabc\n a') token_types = [t.type for t in tokens] expected = [ 'NL', 'INDENT', 'ID', 'NL', 'INDENT', 'ID', 'NL', 'INDENT', 'ID', 'NL', 'INDENT', 'ID', 'NL', 'DEDENT', 'DEDENT', 'DEDENT', 'DEDENT', 'ID', 'NL', 'INDENT', 'ID', 'NL', 'DEDENT', ] self.assertEqual(token_types, expected)
def test_comment_misindent(self): tokens = tokenize('\n a\n #b\n c\n # d\n e') token_types = [t.type for t in tokens] expected = [ 'NL', 'INDENT', 'ID', 'NL', 'ID', 'NL', 'ID', 'NL', 'DEDENT' ] self.assertEqual(token_types, expected)
def test_multiple_dedent(self): tokens = tokenize('\n \n \n \n \n \n') token_types = [t.type for t in tokens] expected = [ 'NL', 'INDENT', 'NL', 'INDENT', 'NL', 'INDENT', 'NL', 'INDENT', 'NL', 'DEDENT', 'DEDENT', 'NL', 'DEDENT', 'DEDENT' ] self.assertEqual(token_types, expected)
def test_entrypoints5(self): tokens = tokenize( '\nentrypoint c:\n # f\nentrypoint d: \n\t\t# f\n\td') token_types = [t.type for t in tokens] expected = [ 'NL', 'INDENT', 'ID', 'ID', 'COLON', 'NL', 'ID', 'ID', 'COLON', 'NL', 'ID', 'NL', 'DEDENT' ] self.assertEqual(token_types, expected)
def process_file(filename, output_dir, output_name, actor_gen, optimizer_flags, **kwargs): init_logger(filename) if_ = Path(filename) of = output_dir / output_name if output_name else output_dir / if_.with_suffix( '.bfevfl').name name = if_.with_suffix('').name if not if_.exists(): emit_error('file not found, skipping') raise LogError() with if_.open('rt') as f: evfl = f.read() setup_logger(evfl) tokens = tokenize(evfl) roots, actors = parse(tokens, actor_gen, exported_tco=optimizer_flags['exported_tco'], **kwargs) if optimizer_flags['merge_duplicate']: optimize_merge_identical(roots) if optimizer_flags['short_event_names']: optimize_names(roots, make_compact_renamer) else: optimize_names(roots, make_counter_renamer) nodes: Set[Node] = set() entrypoints = set(r.name for r in roots) for root in roots: for node in find_postorder(root): if node in nodes: continue if isinstance(node, ActionNode): node.action.mark_used() elif isinstance(node, SwitchNode): node.query.mark_used() elif isinstance(node, SubflowNode): if node.ns == '': if node.called_root_name not in entrypoints: emit_error( f'subflow call for {node.called_root_name} but matching flow/entrypoint not found' ) raise LogError() nodes.add(node) bfevfl = File(name, actors, list(nodes)) with of.open('wb') as f: f.write(bfevfl.prepare_bitstream().bytes)
def test_entrypoints2(self): tokens = tokenize('\na\nentrypoint b:\n\tc') token_types = [t.type for t in tokens] expected = [ 'NL', 'ID', 'NL', 'INDENT', 'ID', 'ID', 'COLON', 'NL', 'ID', 'NL', 'DEDENT', ] self.assertEqual(token_types, expected)
def test_files(self): self.maxDiff = None set_log_level(level=logging.ERROR) for c, err in TestParser.CASES: with self.subTest(msg=f'test_{c}'): with open(f'{TestParser.TEST_DIR}/{c}.evfl', 'rt') as ef: evfl = ef.read() tokens = tokenize(evfl) if err is not None: with self.assertRaises(err): parse(tokens, self.generate_actor) else: rn, actors = parse(tokens, self.generate_actor) actual = [str(x) for x in extract_and_sort_nodes(rn)] expected = [] with open(f'{TestParser.TEST_DIR}/{c}.out', 'rt') as ef: for line in ef: line = line.strip() if line: expected.append(line) self.assertEqual(actual, expected)
def test_bad_dedent(self): with self.assertRaises(LexerError): tokenize('\n\t\n\t \n ')
def test_paren_unopened2(self): with self.assertRaises(LexerError): tokenize(']')
def test_comment_indent_1(self): tokens = tokenize('\n # \na # b\n # c') token_types = [t.type for t in tokens] expected = ['NL', 'ID', 'NL'] self.assertEqual(token_types, expected)
def test_paren_unclosed2(self): with self.assertRaises(LexerError): tokenize('[')
def test_id(self): with self.assertRaises(LexerError): tokenize('-abc') with self.assertRaises(LexerError): tokenize('_abc') with self.assertRaises(LexerError): tokenize('abc-') with self.assertRaises(LexerError): tokenize('abc_') self.assertEqual( tokenize('abc_d0-3'), [Token('ID', 'abc_d0-3'), Token('NL', '')]) self.assertEqual( tokenize('`1.233 444\\` ee`'), [Token('QUOTE_ID', '`1.233 444\\` ee`'), Token('NL', '')]) with self.assertRaises(LexerError): tokenize('01-')
def test_example(self): src = ''' \ flow Test(a: int = 5, b: float = 3.0): # comment if SubflowResults[+5] in (1, 2): System.EventFlags['a'] = 3 else: fork: branch: System.EventFlags["b"] = 7 branch: System.EventFlags["c"] = 7 branch: pass''' expected = [ 'ID', 'ID', 'LPAREN', 'ID', 'COLON', 'ID', 'ASSIGN', 'INT', 'COMMA', 'ID', 'COLON', 'ID', 'ASSIGN', 'FLOAT', 'RPAREN', 'COLON', 'NL', 'INDENT', 'ID', 'ID', 'LSQUARE', 'INT', 'RSQUARE', 'ID', 'LPAREN', 'INT', 'COMMA', 'INT', 'RPAREN', 'COLON', 'NL', 'INDENT', 'ID', 'DOT', 'ID', 'LSQUARE', 'STRING', 'RSQUARE', 'ASSIGN', 'INT', 'NL', 'DEDENT', 'ID', 'COLON', 'NL', 'INDENT', 'ID', 'COLON', 'NL', 'INDENT', 'ID', 'COLON', 'NL', 'INDENT', 'ID', 'DOT', 'ID', 'LSQUARE', 'STRING', 'RSQUARE', 'ASSIGN', 'INT', 'NL', 'DEDENT', 'ID', 'COLON', 'NL', 'INDENT', 'ID', 'DOT', 'ID', 'LSQUARE', 'STRING', 'RSQUARE', 'ASSIGN', 'INT', 'NL', 'DEDENT', 'ID', 'COLON', 'NL', 'INDENT', 'ID', 'NL', 'DEDENT', 'DEDENT', 'DEDENT', 'DEDENT', ] tokens = tokenize(src) token_types = [t.type for t in tokens] self.assertEqual(token_types, expected)
def test_paren_mismatch_nested2(self): with self.assertRaises(LexerError): tokenize('([[()])]')
def test_paren_mismatch2(self): with self.assertRaises(LexerError): tokenize('[)')
def test_entrypoints4(self): with self.assertRaises(LexerError): tokenize('\nentrypoint e:\n')
def test_indent_dedent_simple(self): tokens = tokenize('\n \n') token_types = [t.type for t in tokens] self.assertEqual(token_types, ['NL', 'INDENT', 'NL', 'DEDENT'])
def test_paren_mismatch_nested1(self): with self.assertRaises(LexerError): tokenize('[(([])])')
def test_keyword_start_id(self): tokens = tokenize('\nentrypoint_e:\n') token_types = [t.type for t in tokens] expected = ['NL', 'ID', 'COLON', 'NL'] self.assertEqual(token_types, expected)