예제 #1
0
def lex(chars):
    stream = Stream(chars)
    ret = []
    while stream.next is not None:
        c = stream.get()
        if c in whitespace:
            if c == '\n':
                line()
            pass
        elif c in operation:
            ret.append(('operation', c))
        elif c in string:
            ret.append(('string', get_string(c, stream)))
        elif c in digits:
            ret.append(('number', get_num(c, stream)))
        elif c in symchars:
            ret.append(('symbol', get_symbol(c, stream)))
        elif c in special:
            ret.append((c, ''))
        elif c in comment:
            pass_comment(stream)
        else:
            unexpected_token(c)
    return ret
class TestPyMorseStreamSlow(unittest.TestCase):

    def setUp(self):
        print("Starting test...")
        self.freq = 10
        self._server = SocketWriter(freq = self.freq) # First tests: producer at 10 Hz

        self.com = ComChannel("localhost")
        self.stream = Stream(self.com, 61000, maxlength = 3)
        
    def test_get(self):

        self.assertEqual(self.stream.get(), [0])
        self.assertEqual(self.stream.get(), [1])
        
    def test_last(self):

        self.assertIsNone(self.stream.last())
        self.stream.get()
        self.assertEqual(self.stream.last(), [0])
        self.stream.get()
        self.assertEqual(self.stream.last(), [1])

    def test_last_timed(self):

        d = 1/float(self.freq)
        self.assertIsNone(self.stream.last())
        time.sleep(d + d * 0.5)
        self.assertEqual(self.stream.last(), [0])
        time.sleep(d)
        self.assertEqual(self.stream.last(), [1])
        time.sleep(d)
        time.sleep(d)
        self.assertEqual(self.stream.last(), [3])

    def test_last_n(self):

        d = 1/float(self.freq)
        self.assertIsNone(self.stream.last())
        self.assertIsNone(self.stream.last(2))
        self.assertIsNone(self.stream.last(10))

        time.sleep(d + d * 0.5)

        self.assertEqual(self.stream.last(1), [[0]])
        self.assertEqual(self.stream.last(2), [[0]]) # if only one item available, should be fine to ask for two

        time.sleep(d)

        self.assertEqual(self.stream.last(1), [[1]]) # make sure we return the last one
        self.assertEqual(self.stream.last(2), [[0],[1]]) # make sure the order is right (from older to most recent)

        time.sleep(d)

        self.assertEqual(self.stream.last(3), [[0],[1],[2]])

        time.sleep(d)

        self.assertEqual(self.stream.last(3), [[1], [2],[3]])
        self.assertEqual(self.stream.last(4), [[1], [2],[3]]) # make sure the buffer lenght (here, 3) is used

    def test_subscribe(self):

        d = 1/float(self.freq)
        self.ok = False
        self.i = 0

        self.stream.subscribe(self.on_data)

        time.sleep(d + d * 0.5)
        self.assertTrue(self.ok)

        self.ok = False
        time.sleep(d)
        self.assertTrue(self.ok)

    def test_unsubscribe(self):

        d = 1/float(self.freq)
        self.ok = False
        self.i = 0

        self.stream.subscribe(self.on_data)
        time.sleep(d + d * 0.5)
        self.assertTrue(self.ok)

        self.ok = False

        self.stream.unsubscribe(self.on_data)
        time.sleep(d)
        self.assertFalse(self.ok)

    def on_data(self, record):
        
        self.ok = True
        self.assertEqual(record, [self.i])
        self.i += 1
        
    def tearDown(self):
        self.com.close()
        self._server.close()
예제 #3
0
class TestPyMorseStreamSlow(unittest.TestCase):
    def setUp(self):
        print("Starting test...")
        self.freq = 10
        self._server = SocketWriter(
            freq=self.freq)  # First tests: producer at 10 Hz

        self.com = ComChannel("localhost")
        self.stream = Stream(self.com, 61000, maxlength=3)

    def test_get(self):

        self.assertEqual(self.stream.get(), [0])
        self.assertEqual(self.stream.get(), [1])

    def test_last(self):

        self.assertIsNone(self.stream.last())
        self.stream.get()
        self.assertEqual(self.stream.last(), [0])
        self.stream.get()
        self.assertEqual(self.stream.last(), [1])

    def test_last_timed(self):

        d = 1 / float(self.freq)
        self.assertIsNone(self.stream.last())
        time.sleep(d + d * 0.5)
        self.assertEqual(self.stream.last(), [0])
        time.sleep(d)
        self.assertEqual(self.stream.last(), [1])
        time.sleep(d)
        time.sleep(d)
        self.assertEqual(self.stream.last(), [3])

    def test_last_n(self):

        d = 1 / float(self.freq)
        self.assertIsNone(self.stream.last())
        self.assertIsNone(self.stream.last(2))
        self.assertIsNone(self.stream.last(10))

        time.sleep(d + d * 0.5)

        self.assertEqual(self.stream.last(1), [[0]])
        self.assertEqual(
            self.stream.last(2),
            [[0]])  # if only one item available, should be fine to ask for two

        time.sleep(d)

        self.assertEqual(self.stream.last(1),
                         [[1]])  # make sure we return the last one
        self.assertEqual(
            self.stream.last(2),
            [[0], [1]
             ])  # make sure the order is right (from older to most recent)

        time.sleep(d)

        self.assertEqual(self.stream.last(3), [[0], [1], [2]])

        time.sleep(d)

        self.assertEqual(self.stream.last(3), [[1], [2], [3]])
        self.assertEqual(
            self.stream.last(4),
            [[1], [2], [3]])  # make sure the buffer lenght (here, 3) is used

    def test_subscribe(self):

        d = 1 / float(self.freq)
        self.ok = False
        self.i = 0

        self.stream.subscribe(self.on_data)

        time.sleep(d + d * 0.5)
        self.assertTrue(self.ok)

        self.ok = False
        time.sleep(d)
        self.assertTrue(self.ok)

    def test_unsubscribe(self):

        d = 1 / float(self.freq)
        self.ok = False
        self.i = 0

        self.stream.subscribe(self.on_data)
        time.sleep(d + d * 0.5)
        self.assertTrue(self.ok)

        self.ok = False

        self.stream.unsubscribe(self.on_data)
        time.sleep(d)
        self.assertFalse(self.ok)

    def on_data(self, record):

        self.ok = True
        self.assertEqual(record, [self.i])
        self.i += 1

    def tearDown(self):
        self.com.close()
        self._server.close()
for fname in files:
    print "OPENING NEW FILE:", fname

    f = open(indir + fname)
    data = f.read()
    f.close()

    newf = open(outdir + fname, 'w')
    newf.write('\n')

    stream = Stream(data)
    while True:

        #read '('
        while True:
            char = stream.get()
            if char == '(' or stream.eof():
                break

        if stream.eof():
            break

        root = Node()
        root.read(stream)

        #insert the structure into standard PTB
        sline = slines[supto].strip()
        factors = []
        while sline != "---":
            print "inserting:", sline
            label, start, end = sline.split()
예제 #5
0
파일: tokenizer.py 프로젝트: narupo/cap
class Tokenizer:
    class ParseError(RuntimeError):
        pass

    class ModuleError(RuntimeError):
        pass

    def parse(self, src):
        self.strm = Stream(src)
        self.tokens = []

        s = self.strm
        m = 'text block'
        buf = ''
        while not s.eof():
            c = s.get()
            if m == 'text block':
                if c == '{' and s.cur() == '@':
                    if len(buf):
                        self.tokens.append(Token(kind='text-block', value=buf))
                        buf = ''
                    s.get()
                    self.tokens.append(Token(kind='lbraceat', value='{@'))
                    m = 'code block'
                elif c == '{' and s.cur() == '{':
                    if len(buf):
                        self.tokens.append(Token(kind='text-block', value=buf))
                        buf = ''
                    s.get()
                    self.tokens.append(Token(kind='ldbrace', value='{{'))
                    m = 'ref block'
                else:
                    buf += c
            elif m == 'ref block':
                if c == '}' and s.cur() == '}':
                    s.get()
                    self.tokens.append(Token(kind='rdbrace', value='}}'))
                    m = 'text block'                    
                elif self.is_identifier_char(c):
                    s.prev()
                    self.read_identifier()
                elif c == '(':
                    self.tokens.append(Token(kind='lparen', value='('))
                elif c == ')':
                    self.tokens.append(Token(kind='rparen', value=')'))
                elif c == '.':
                    self.tokens.append(Token(kind='operator', value='.'))
                elif c == '"':
                    s.prev()
                    self.read_string()
            elif m == 'code block':
                if c == '@' and s.cur() == '}':
                    s.get()
                    self.tokens.append(Token(kind='rbraceat', value='@}'))
                    m = 'text block'
                elif c == '.':
                    self.tokens.append(Token(kind='operator', value='.'))
                elif c == ',':
                    self.tokens.append(Token(kind='comma', value=','))
                elif c == '(':
                    self.tokens.append(Token(kind='lparen', value='('))
                elif c == ')':
                    self.tokens.append(Token(kind='rparen', value=')'))
                elif c == ':':
                    self.tokens.append(Token(kind='colon', value=':'))
                elif c == '=':
                    s.prev()
                    self.read_assign()
                elif c == '!':
                    s.prev()
                    self.read_not()
                elif c == '<':
                    s.prev()
                    self.read_lt()
                elif c == '>':
                    s.prev()
                    self.read_gt()
                elif c == '+':
                    s.prev()
                    self.read_add()
                elif c == '-':
                    s.prev()
                    self.read_sub()
                elif c == '*':
                    self.tokens.append(Token(kind='operator', value='*'))
                elif c == '/':
                    self.tokens.append(Token(kind='operator', value='/'))
                elif c == '"':
                    s.prev()
                    self.read_string()
                elif c.isdigit():
                    s.prev()
                    self.read_digit()
                elif self.is_identifier_char(c):
                    s.prev()
                    self.read_identifier()
                elif c.isspace():
                    pass
                else:
                    raise Tokenizer.ParseError('unsupprted character %s' % c)

        if len(buf):
            self.tokens.append(Token(kind='text-block', value=buf))
            buf = ''

        return self.tokens

    def read_add(self):
        c = self.strm.get()
        if c != '+':
            raise Tokenizer.ModuleError('not found "+"')

        c = self.strm.get()
        if c == '+':
            self.tokens.append(Token(kind='operator', value='++'))
        else:
            self.strm.prev()
            self.tokens.append(Token(kind='operator', value='+'))

    def read_sub(self):
        c = self.strm.get()
        if c != '-':
            raise Tokenizer.ModuleError('not found "-"')

        c = self.strm.get()
        if c == '-':
            self.tokens.append(Token(kind='operator', value='--'))
        else:
            self.strm.prev()
            self.tokens.append(Token(kind='operator', value='-'))

    def read_gt(self):
        c = self.strm.get()
        if c != '>':
            raise Tokenizer.ModuleError('not found ">"')

        c = self.strm.get()
        if c == '=':
            self.tokens.append(Token(kind='comp_op', value='>='))
        else:
            self.strm.prev()
            self.tokens.append(Token(kind='comp_op', value='>'))

    def read_lt(self):
        c = self.strm.get()
        if c != '<':
            raise Tokenizer.ModuleError('not found "<"')

        c = self.strm.get()
        if c == '=':
            self.tokens.append(Token(kind='comp_op', value='<='))
        else:
            self.strm.prev()
            self.tokens.append(Token(kind='comp_op', value='<'))

    def read_not(self):
        c = self.strm.get()
        if c != '!':
            raise Tokenizer.ModuleError('not found "!"')

        c = self.strm.get()
        if c == '=':
            self.tokens.append(Token(kind='comp_op', value='!='))
        else:
            self.strm.prev()
            self.tokens.append(Token(kind='comp_op', value='!'))

    def read_digit(self):
        buf = ''
        while not self.strm.eof():
            c = self.strm.get()
            if not c.isdigit():
                self.strm.prev()
                break
            buf += c

        if not len(buf):
            raise Tokenizer.ModuleError('not found digit')

        self.tokens.append(Token(kind='digit', value=int(buf)))

    def read_assign(self):
        c = self.strm.get()
        if c != '=':
            raise Tokenizer.ModuleError('not found "="')

        c = self.strm.get()
        if c == '=':
            self.tokens.append(Token(kind='comp_op', value='=='))
        else:
            self.strm.prev()
            self.tokens.append(Token(kind='operator', value='='))

    def is_identifier_char(self, c):
        return c.isalpha() or c.isdigit() or c == '_'

    def read_identifier(self):
        t = Token(kind='identifier', value='')
        while not self.strm.eof():
            c = self.strm.get()
            if not self.is_identifier_char(c):
                self.strm.prev()
                break
            else:
                t.value += c

        if not len(t.value):
            raise Tokenizer.ModuleError('not found identifier') # impossible

        if t.value == 'import':
            t.kind = 'import'
        elif t.value == 'if':
            t.kind = 'if'
        elif t.value == 'elif':
            t.kind = 'elif'
        elif t.value == 'else':
            t.kind = 'else'
        elif t.value == 'end':
            t.kind = 'end'

        self.tokens.append(t)

    def read_string(self):
        c = self.strm.get()
        if c != '"':
            raise Tokenizer.ModuleError("not found '"'')

        t = Token(kind='string', value='')
        while not self.strm.eof():
            c = self.strm.get()
            if c == '"':
                break
            else:
                t.value += c

        self.tokens.append(t)

    def read_lbrace(self):
        c = self.strm.get()
        if c != '{':
            raise Tokenizer.ModuleError('not found "{"')

        c = self.strm.get()
        if c == '{':
            self.tokens.append(Token(kind='ldbrace', value='{{'))
        elif c == '@':
            self.tokens.append(Token(kind='lbraceat', value='{@'))
        else:
            self.strm.prev()
            self.tokens.append(Token(kind='lbrace', value='{'))

    def read_rbrace(self):
        c = self.strm.get()
        if c != '}':
            raise Tokenizer.ModuleError('not found "}"')

        c = self.strm.get()
        if c == '}':
            self.tokens.append(Token(kind='rdbrace', value='}}'))
        else:
            self.strm.prev()
            self.tokens.append(Token(kind='rbrace', value='}'))

    def read_at(self):
        c = self.strm.get()
        if c != '@':
            raise Tokenizer.ModuleError('not found "@"')

        c = self.strm.get()
        if c == '}':
            self.tokens.append(Token(kind='rbraceat', value='@}'))
        else:
            raise Tokenizer.ParseError('invalid token. "@" is not supported')
예제 #6
0
파일: ast.py 프로젝트: narupo/cap
class AST:
    class ModuleError(RuntimeError):
        pass

    class SyntaxError(SyntaxError):
        pass

    class ImportError(ImportError):
        pass

    class NameError(NameError):
        pass

    class ReferenceError(RuntimeError):
        pass

    class TypeError(TypeError):
        pass

    def parse(self, tokens, debug=False):
        self.debug_parse = debug
        self.strm = Stream(tokens)
        self.root = self.block(dep=0)

    def traverse(self, debug=False, opts=None):
        self.debug_traverse = debug
        self.opts = opts
        self.context = Context()
        self._traverse(self.root, dep=0)
        return self.context

    def show_traverse(self, node, dep):
        if self.debug_traverse:
            print(('_'*dep) + str(dep) + ' ' + str(node).split('.')[1].split(' ')[0])

    def _traverse(self, node, dep):
        if node is None:
            return
        self.show_traverse(node, dep)

        if isinstance(node, BlockNode):
            self._traverse(node.text_block, dep=dep+1)
            self._traverse(node.code_block, dep=dep+1)
            self._traverse(node.ref_block, dep=dep+1)
            self._traverse(node.block, dep=dep+1)

        elif isinstance(node, TextBlockNode):
            self.context.buffer += node.text

        elif isinstance(node, CodeBlockNode):
            self._traverse(node.formula, dep=dep+1)

        elif isinstance(node, RefBlockNode):
            self.traverse_ref_block(node, dep=dep+1)

        elif isinstance(node, FormulaNode):
            if node.comparison:
                self.context.last_expr_val = self._traverse(node.comparison, dep=dep+1)
                return self.context.last_expr_val
            elif node.assign_expr:
                self._traverse(node.assign_expr, dep=dep+1)
            elif node.import_:
                self._traverse(node.import_, dep=dep+1)
            elif node.caller:
                self._traverse(node.caller, dep=dep+1)
            elif node.if_:
                self._traverse(node.if_, dep=dep+1)

            if node.formula:
                self._traverse(node.formula, dep=dep+1)
            elif node.block:
                self._traverse(node.block, dep=dep+1)

        elif isinstance(node, ComparisonNode):
            if node.expr and node.comparison:
                lval = self._traverse(node.expr, dep=dep+1)
                rval = self._traverse(node.comparison, dep=dep+1)
                if node.op == '>':
                    return lval > rval
                elif node.op == '<':
                    return lval < rval
                elif node.op == '>=':
                    return lval >= rval
                elif node.op == '<=':
                    return lval <= rval
                elif node.op == '==':
                    return lval == rval
                elif node.op == '!=':
                    return lval != rval
                else:
                    raise AST.ModuleError('unsupported comparison operator "%s"' % node.op)

            elif node.expr:
                return self._traverse(node.expr, dep=dep+1)
            else:
                raise AST.ModuleError('unsupported node %s in comparison' % str(node))

        elif isinstance(node, IfNode):
            result = self._traverse(node.comparison, dep=dep+1)
            if result:
                if node.formula:
                    self._traverse(node.formula, dep=dep+1)
                elif node.block:
                    self._traverse(node.block, dep=dep+1)
            else:
                if node.elif_:
                    self._traverse(node.elif_, dep=dep+1)
                elif node.else_:
                    self._traverse(node.else_, dep=dep+1)

        elif isinstance(node, ElseNode):
            if node.block:
                self._traverse(node.block, dep=dep+1)
            elif node.formula:
                self._traverse(node.formula, dep=dep+1)

        elif isinstance(node, ExprNode):
            if node.term and node.expr:
                lval = self._traverse(node.term, dep=dep+1)
                rval = self._traverse(node.expr, dep=dep+1)
                if node.op == '+':
                    return lval + rval
                elif node.op == '-':
                    return lval - rval
                else:
                    raise AST.ModuleError('unsupported operation "%s" in traverse expr' % node.op)
            elif node.term:
                return self._traverse(node.term, dep=dep+1)
            else:
                raise AST.ModuleError('programming error. impossible case in traverse expr')

        elif isinstance(node, TermNode):
            if node.factor and node.term:
                lval = self._traverse(node.factor, dep=dep+1)
                rval = self._traverse(node.term, dep=dep+1)
                if node.op == '*':
                    return lval * rval
                elif node.op == '/':
                    return lval / rval
                else:
                    raise AST.ModuleError('unsupported operation "%s" in traverse term' % node.op)
            elif node.factor:
                return self._traverse(node.factor, dep=dep+1)
            else:
                raise AST.ModuleError('programming error. impossible case in traverse term')

        elif isinstance(node, FactorNode):
            if node.expr:
                return self._traverse(node.expr, dep=dep+1)
            elif node.identifier != None:
                if node.identifier not in self.context.syms.keys():
                    raise AST.ReferenceError('%s is not defined' % node.identifier)
                return self.context.syms[node.identifier]
            elif node.digit:
                return self._traverse(node.digit, dep=dep+1)
            elif node.string != None:
                return node.string
            elif node.callable:
                return self._traverse(node.callable, dep=dep+1)

        elif isinstance(node, DigitNode):
            return node.value

        elif isinstance(node, AssignExprNode):
            return self.traverse_assign_expr(node, dep+1)

        elif isinstance(node, ImportNode):
            self.traverse_import(node, dep+1)

        elif isinstance(node, CallerNode):
            self.traverse_caller(node, dep+1)

        elif isinstance(node, CallableNode):
            package = node.caller_list.identifier
            if package == 'opts':
                if node.caller_list.caller_list.identifier == 'get':
                    if self.opts and node.args.arg.string in self.opts.keys():
                        return self.opts[node.args.arg.string]
                    else:
                        return ''

        else:
            raise AST.ModuleError('impossible. not supported node', type(node))

    def traverse_assign_expr(self, node, dep):
        lhs = node.assign_operand_lhs
        op = node.assign_operator
        rhs = node.assign_operand_rhs
        assexpr = node.assign_expr
        if lhs is None and rhs:
            if rhs.expr:
                return self._traverse(rhs.expr, dep+1)
            elif rhs.callable:
                return self._traverse(rhs.callable, dep+1)
            elif rhs.string != None:
                return rhs.string
            elif rhs.identifier != None:
                if rhs.identifier not in self.context.syms.keys():
                    raise AST.ReferenceError('"%s" is not defined' % rhs.identifier)
        elif lhs and op and assexpr:
            if op.operator == '=':
                self.context.syms[lhs.identifier] = self._traverse(assexpr, dep=dep+1)
                self.context.last_expr_val = self.context.syms[lhs.identifier]
                return self.context.syms[lhs.identifier]
            else:
                raise AST.ModuleError('unsupported operator "%s"' % op.operator)
        else:
            raise AST.ModuleError('invalid case in traverse assign expr')
             
    def traverse_ref_block(self, node, dep):
        if node.identifier != None:
            if node.identifier in self.context.syms.keys():
                self.context.buffer += str(self.context.syms[node.identifier])
        elif node.callable:
            result = self._traverse(node.callable, dep=dep+1)
            self.context.buffer += str(result)

    def traverse_import(self, node, dep):
        if node.identifier == 'alias':
            self.context.imported_alias = True 
        elif node.identifier == 'config':
            self.context.imported_config = True
        else:
            raise AST.ImportError('can not import package "%s"' % node.identifier)

    def traverse_caller(self, node, dep):
        first = node.identifiers[0]
        if first == 'alias':
            self.call_alias_package(node)
        elif first == 'config':
            self.call_config_package(node)
        else:
            raise AST.NameError('"%s" is not defined' % first)

    def call_alias_package(self, node):
        if not self.context.imported_alias:
            raise AST.ImportError('alias is not imported')

        method = node.identifiers[1]
        if method == 'set':
            if len(node.args) != 2:
                AST.SyntaxError('alias.set need two arguments')
            name = node.args[0]
            cmd = node.args[1]
            self.context.alias_map[name] = cmd

    def call_config_package(self, node):
        if not self.context.imported_config:
            raise AST.ImportError('config is not imported')

        method = node.identifiers[1]
        if method == 'set':
            if len(node.args) != 2:
                AST.SyntaxError('config.set need two arguments')
            name = node.args[0]
            cmd = node.args[1]
            self.context.config_map[name] = cmd

    def show_parse(self, name, dep):
        if self.debug_parse:
            t = self.strm.cur()
            print(dep, name + ': ' + str(t))

    def block(self, dep=0):
        self.show_parse('block', dep=dep)
        if self.strm.eof():
            return None
        
        t = self.strm.cur()
        if t.kind in ('rbraceat', 'end', 'if', 'elif', 'else'):
            return None

        node = BlockNode()

        node.code_block = self.code_block(dep=dep+1)
        if node.code_block is None:
            node.ref_block = self.ref_block(dep=dep+1)
            if node.ref_block is None:
                node.text_block = self.text_block(dep=dep+1)

        node.block = self.block(dep=dep+1)
        return node

    def ref_block(self, dep=0):
        self.show_parse('ref_block', dep=dep)
        if self.strm.eof():
            return None

        node = RefBlockNode()

        t = self.strm.get()
        if t.kind != 'ldbrace':
            self.strm.prev()
            return None

        t = self.strm.get()
        if t.kind == 'identifier':
            t2 = self.strm.cur()
            if t2 == Stream.EOF:
                raise AST.SyntaxError('reference block not closed')
            elif t2.value in ('.', '('):
                self.strm.prev()
                node.callable = self.callable()
            else:
                node.identifier = t.value
        elif t.kind == 'rdbrace':
            return node

        t = self.strm.get()
        if t == Stream.EOF or t.kind != 'rdbrace':
            raise AST.SyntaxError('not found "rdbrace"')

        return node

    def callable(self, dep=0):
        self.show_parse('callable', dep=dep)
        if self.strm.eof():
            return None

        node = CallableNode()
        node.caller_list = self.caller_list()

        t = self.strm.get()
        if t.value != '(':
            raise AST.SyntaxError('not found "(" in callable')
        
        node.args = self.args()

        t = self.strm.get()
        if t.value != ')':
            raise AST.SyntaxError('not found ")" in callable')
        
        return node

    def caller_list(self, dep=0):
        self.show_parse('caller_list', dep=dep)
        if self.strm.eof():
            return None

        node = CallerListNode()
        t = self.strm.get()
        if t == Stream.EOF:
            raise AST.SyntaxError('reached EOF in caller list')
        elif t.kind == 'identifier':
            node.identifier = t.value
        else:
            raise AST.SyntaxError('invalid token "%s"' % t)

        t = self.strm.get()
        if t.value == '.':
            node.caller_list = self.caller_list()
        else:
            self.strm.prev()

        return node

    def args(self, dep=0):
        self.show_parse('args', dep=dep)
        if self.strm.eof():
            return None

        node = ArgsNode()
        node.arg = self.arg()
        t = self.strm.get()
        if t.value == ',':
            node.args = self.args()
        else:
            self.strm.prev()

        return node

    def arg(self, dep=0):
        self.show_parse('arg', dep=dep)
        if self.strm.eof():
            return None

        node = ArgNode()
        t = self.strm.get()
        if t == Stream.EOF:
            raise AST.SyntaxError('reached EOF in argument')
        elif t.kind == 'digit':
            node.digit = DigitNode()
            node.digit.value = t.value
        elif t.kind == 'string':
            node.string = t.value
        elif t.kind == 'identifier':
            node.identifier = t.value
        else:
            raise AST.SyntaxError('not supported argument type "%s"' % t)

        return node

    def code_block(self, dep=0):
        self.show_parse('code_block', dep=dep)
        if self.strm.eof():
            return None

        t = self.strm.get()
        if t.kind != 'lbraceat':
            self.strm.prev()
            return None

        node = CodeBlockNode()
        node.formula = self.formula(dep=dep+1)

        t = self.strm.get()
        if t == Stream.EOF:
            pass
        elif t.kind in ('end'):
            self.strm.prev()
            return node
        elif t.kind in ('ldbrace', 'else', 'elif'):
            self.strm.prev()

        return node

    def text_block(self, dep=0):
        self.show_parse('text_block', dep=dep)
        if self.strm.eof():
            return None
        
        t = self.strm.get()
        if t.kind != 'text-block':
            self.strm.prev()
            return None

        node = TextBlockNode()
        node.text = t.value

        return node

    def formula(self, dep=0):
        self.show_parse('formula', dep=dep)
        if self.strm.eof():
            return None
        
        t = self.strm.cur()
        if t.kind in ('rbraceat', 'ldbrace', 'end', 'elif', 'else'):
            return None
        if t.kind in ('colon'):
            raise AST.SyntaxError('found "%s". invalid formula.' % t)

        node = FormulaNode()
        t = self.strm.cur()
        if t.kind == 'import':
            node.import_ = self.import_(dep=dep+1)
        elif t.kind == 'if':
            node.if_ = self.if_(dep=dep+1)
        else:
            node.caller = self.caller(dep=dep+1)
            if node.caller is None:
                t2 = self.strm.cur(1)
                if t2.value == '=':
                    node.assign_expr = self.assign_expr(dep=dep+1)
                    if not self.strm.eof() and node.assign_expr is None:
                        return None
                else:
                    node.comparison = self.comparison(dep=dep+1)
                    if not self.strm.eof() and node.comparison is None:
                        return None

        t = self.strm.get()
        if t == Stream.EOF:
            return node

        if t.kind == 'rbraceat' and self.strm.cur() != Stream.EOF:
            node.block = self.block(dep=dep+1)
            t = self.strm.get()
            if t == Stream.EOF:
                pass
            elif t.kind == 'lbraceat':
                pass
            else:
                self.strm.prev()
        else:
            self.strm.prev()
            node.formula = self.formula(dep=dep+1)

        return node

    def comparison(self, dep=0):
        self.show_parse('comparison', dep=dep)
        if self.strm.eof():
            return None

        node = ComparisonNode()
        node.expr = self.expr(dep=dep+1)
        t = self.strm.get()
        if t == Stream.EOF:
            return node
        elif t.kind != 'comp_op':
            self.strm.prev()
            return node
        node.op = t.value

        node.comparison = self.comparison(dep=dep+1)
        return node

    def if_(self, dep=0, first_symbol='if'):
        self.show_parse('if', dep=dep)
        if self.strm.eof():
            return None

        t = self.strm.get()
        if t.kind != first_symbol:
            return None

        node = IfNode()
        node.comparison = self.comparison(dep=dep+1)
        if node.comparison is None:
            raise AST.SyntaxError('invalid if statement. not found comparison')

        t = self.strm.get()
        if t.kind != 'colon':
            raise AST.SyntaxError('invalid if statement. not found colon')

        t = self.strm.get()
        if t.kind == 'rbraceat': # @}
            node.block = self.block(dep=dep+1)
        else:
            self.strm.prev()
            node.formula = self.formula(dep=dep+1)

        t = self.strm.get()
        if t == Stream.EOF:
            raise AST.SyntaxError('reached EOF in if statement')

        if t.kind != 'lbraceat':
            self.strm.prev()

        t = self.strm.get()
        if t.kind == 'end':
            pass
        elif t.kind == 'elif':
            self.strm.prev()
            node.elif_ = self.if_(dep=dep+1, first_symbol='elif')
        elif t.kind == 'else':
            self.strm.prev()
            node.else_ = self.else_(dep=dep+1)
        else:
            raise AST.SyntaxError('not ended in if statement. token is %s' % t)

        return node

    def else_(self, dep=0):
        self.show_parse('else', dep=dep)
        if self.strm.eof():
            return None

        node = ElseNode()

        t = self.strm.get()
        if t.kind != 'else':
            raise AST.SyntaxError('invalid else statement. not found "else"')

        t = self.strm.get()
        if t.kind != 'colon':
            raise AST.SyntaxError('invalid else statement. not found colon')

        t = self.strm.get()
        if t.kind == 'rbraceat':
            node.block = self.block(dep=dep+1)
        else:
            self.strm.prev()
            node.formula = self.formula(dep=dep+1)

        t = self.strm.get()
        if t == Stream.EOF:
            return node
        if t.kind != 'lbraceat':
            self.strm.prev()

        t = self.strm.get()
        if t.kind == 'end':
            pass
        else:
            self.strm.prev()

        return node

    def expr(self, dep=0):
        self.show_parse('expr', dep=dep)
        if self.strm.eof():
            return None
        
        node = ExprNode()
        node.term = self.term(dep=dep+1)

        t = self.strm.get()
        if t == Stream.EOF:
            return node
        elif t.value not in ('+', '-'):
            self.strm.prev()
            return node
        node.op = t.value

        node.expr = self.expr(dep=dep+1)
        return node

    def term(self, dep=0):
        self.show_parse('term', dep=dep)
        if self.strm.eof():
            return None

        node = TermNode()
        node.factor = self.factor(dep=dep+1)

        t = self.strm.get()
        if t == Stream.EOF:
            return node
        elif t.value not in ('*', '/'):
            self.strm.prev()
            return node
        node.op = t.value

        node.term = self.term(dep=dep+1)
        return node

    def factor(self, dep=0):
        self.show_parse('factor', dep=dep)
        if self.strm.eof():
            return None

        node = FactorNode()
        t = self.strm.get()
        if t == Stream.EOF:
            raise AST.SyntaxError('reached EOF in factor')
        elif t.kind == 'lparen':
            node.expr = self.expr(dep=dep+1)
            t = self.strm.get()
            if t == Stream.EOF:
                raise AST.SyntaxError('not found rparen in factor. reached EOF')
            if t.kind != 'rparen':
                raise AST.SyntaxError('not found rparen in factor. token is %s' % t)
        elif t.kind == 'digit':
            node.digit = DigitNode()
            node.digit.value = t.value
        elif t.kind == 'identifier':
            self.strm.prev()
            if self.is_callable(dep=dep+1):
                node.callable = self.callable(dep=dep+1)
            else:
                t = self.strm.get()
                node.identifier = t.value
        elif t.kind == 'string':
            node.string = t.value
        else:
            self.strm.prev()
            return None

        return node 

    def is_callable(self, dep):
        self.show_parse('is_callable', dep=dep)
        if self.strm.eof():
            return None

        ret = True
        i = self.strm.index

        while not self.strm.eof():
            t = self.strm.get()
            if t == Stream.EOF:
                ret = False
                break
            elif t.kind == 'identifier':
                pass
            elif t.value == '.':
                pass
            elif t.value == '(':
                break
            else:
                ret = False
                break

        self.strm.index = i
        return ret


    def digit(self, dep=0):
        self.show_parse('digit', dep=dep)
        if self.strm.eof():
            return None

        t = self.strm.get()
        if t.kind != 'digit':
            raise AST.ModuleError('not found digit')

        node = DigitNode()
        node.value = t.value

        return node

    def assign_expr(self, dep=0):
        self.show_parse('assign_expr', dep=dep)
        if self.strm.eof():
            return None

        node = AssignExprNode()
        i = self.strm.index
        node.assign_operand_lhs = self.assign_operand_lhs(dep=dep+1)
        if node.assign_operand_lhs:
            node.assign_operator = self.assign_operator(dep=dep+1)
            if node.assign_operator is None:
                node.assign_operand_lhs = None
                self.strm.index = i
                node.assign_operand_rhs = self.assign_operand_rhs(dep=dep+1)
            else:
                node.assign_expr = self.assign_expr(dep=dep+1)
        else:
            node.assign_operand_rhs = self.assign_operand_rhs(dep=dep+1)

        return node

    def assign_operand_rhs(self, dep=0):
        self.show_parse('assign_operand_rhs', dep=dep)
        if self.strm.eof():
            return None

        node = AssignOperandRhsNode()
        t = self.strm.get()
        if t.kind == 'string':
            node.string = t.value
        elif t.kind == 'identifier':
            self.strm.prev()
            if self.is_callable(dep=dep+1):
                node.callable = self.callable()
            else:
                node.expr = self.expr(dep=dep+1)
                if node.expr is None:
                    t = self.strm.get()
                    node.identifier = t.value
        else:
            self.strm.prev()
            node.expr = self.expr(dep=dep+1)
            if node.expr is None:
                return None

        return node
        
    def assign_operator(self, dep=0):
        self.show_parse('assign_operator', dep=dep)
        if self.strm.eof():
            return None

        t = self.strm.get()
        if t.value != '=':
            self.strm.prev()
            return None

        node = AssignOperatorNode()
        node.operator = t.value
        return node
        
    def assign_operand_lhs(self, dep=0):
        self.show_parse('assign_operand_lhs', dep=dep)
        if self.strm.eof():
            return None
        elif self.is_callable(dep=dep+1):
            return None

        node = AssignOperandLhsNode()
        t = self.strm.get()
        if t.kind != 'identifier':
            self.strm.prev()
            return None

        node.identifier = t.value
        return node
                
    def import_(self, dep=0):
        self.show_parse('import', dep=dep)
        if self.strm.eof():
            return None
        
        node = ImportNode()
        t = self.strm.get()
        if t.kind != 'import':
            self.strm.prev()
            return None

        t = self.strm.get()
        if t.kind != 'identifier':
            raise AST.SyntaxError('invalid import statement')

        node.identifier = t.value
        return node

    def caller(self, dep=0):
        """
        ここがしんどい
        もっと抽象化するべき
        """
        self.show_parse('caller', dep=dep)
        if self.strm.eof():
            return None
        
        # find lparen
        i = self.strm.index
        found = False
        while not self.strm.eof():
            t = self.strm.get()
            if t.kind == 'identifier':
                pass
            elif t.kind == 'operator' and t.value == '.':
                pass
            elif t.kind == 'lparen':
                found = True
                break
            else:
                break
        self.strm.index = i
        if not found:
            return None

        node = CallerNode()
        save_index = self.strm.index

        while not self.strm.eof():
            t = self.strm.get()
            if t.kind == 'identifier':
                node.identifiers.append(t.value)
            elif t.kind == 'lparen':
                break

        if not len(node.identifiers):
            self.strm.index = save_index
            return None

        while not self.strm.eof():
            t = self.strm.get()
            if t.kind == 'string':
                node.args.append(t.value)
            elif t.kind == 'rparen':
                break

        return node