def lex(chars):
stream = Stream(chars)
ret = []
while stream.next is not None:
c = stream.get()
if c in whitespace:
if c == '\n':
line()
pass
elif c in operation:
ret.append(('operation', c))
elif c in string:
ret.append(('string', get_string(c, stream)))
elif c in digits:
ret.append(('number', get_num(c, stream)))
elif c in symchars:
ret.append(('symbol', get_symbol(c, stream)))
elif c in special:
ret.append((c, ''))
elif c in comment:
pass_comment(stream)
else:
unexpected_token(c)
return ret
class TestPyMorseStreamSlow(unittest.TestCase):
def setUp(self):
print("Starting test...")
self.freq = 10
self._server = SocketWriter(freq = self.freq) # First tests: producer at 10 Hz
self.com = ComChannel("localhost")
self.stream = Stream(self.com, 61000, maxlength = 3)
def test_get(self):
self.assertEqual(self.stream.get(), [0])
self.assertEqual(self.stream.get(), [1])
def test_last(self):
self.assertIsNone(self.stream.last())
self.stream.get()
self.assertEqual(self.stream.last(), [0])
self.stream.get()
self.assertEqual(self.stream.last(), [1])
def test_last_timed(self):
d = 1/float(self.freq)
self.assertIsNone(self.stream.last())
time.sleep(d + d * 0.5)
self.assertEqual(self.stream.last(), [0])
time.sleep(d)
self.assertEqual(self.stream.last(), [1])
time.sleep(d)
time.sleep(d)
self.assertEqual(self.stream.last(), [3])
def test_last_n(self):
d = 1/float(self.freq)
self.assertIsNone(self.stream.last())
self.assertIsNone(self.stream.last(2))
self.assertIsNone(self.stream.last(10))
time.sleep(d + d * 0.5)
self.assertEqual(self.stream.last(1), [[0]])
self.assertEqual(self.stream.last(2), [[0]]) # if only one item available, should be fine to ask for two
time.sleep(d)
self.assertEqual(self.stream.last(1), [[1]]) # make sure we return the last one
self.assertEqual(self.stream.last(2), [[0],[1]]) # make sure the order is right (from older to most recent)
time.sleep(d)
self.assertEqual(self.stream.last(3), [[0],[1],[2]])
time.sleep(d)
self.assertEqual(self.stream.last(3), [[1], [2],[3]])
self.assertEqual(self.stream.last(4), [[1], [2],[3]]) # make sure the buffer lenght (here, 3) is used
def test_subscribe(self):
d = 1/float(self.freq)
self.ok = False
self.i = 0
self.stream.subscribe(self.on_data)
time.sleep(d + d * 0.5)
self.assertTrue(self.ok)
self.ok = False
time.sleep(d)
self.assertTrue(self.ok)
def test_unsubscribe(self):
d = 1/float(self.freq)
self.ok = False
self.i = 0
self.stream.subscribe(self.on_data)
time.sleep(d + d * 0.5)
self.assertTrue(self.ok)
self.ok = False
self.stream.unsubscribe(self.on_data)
time.sleep(d)
self.assertFalse(self.ok)
def on_data(self, record):
self.ok = True
self.assertEqual(record, [self.i])
self.i += 1
def tearDown(self):
self.com.close()
self._server.close()
class TestPyMorseStreamSlow(unittest.TestCase):
def setUp(self):
print("Starting test...")
self.freq = 10
self._server = SocketWriter(
freq=self.freq) # First tests: producer at 10 Hz
self.com = ComChannel("localhost")
self.stream = Stream(self.com, 61000, maxlength=3)
def test_get(self):
self.assertEqual(self.stream.get(), [0])
self.assertEqual(self.stream.get(), [1])
def test_last(self):
self.assertIsNone(self.stream.last())
self.stream.get()
self.assertEqual(self.stream.last(), [0])
self.stream.get()
self.assertEqual(self.stream.last(), [1])
def test_last_timed(self):
d = 1 / float(self.freq)
self.assertIsNone(self.stream.last())
time.sleep(d + d * 0.5)
self.assertEqual(self.stream.last(), [0])
time.sleep(d)
self.assertEqual(self.stream.last(), [1])
time.sleep(d)
time.sleep(d)
self.assertEqual(self.stream.last(), [3])
def test_last_n(self):
d = 1 / float(self.freq)
self.assertIsNone(self.stream.last())
self.assertIsNone(self.stream.last(2))
self.assertIsNone(self.stream.last(10))
time.sleep(d + d * 0.5)
self.assertEqual(self.stream.last(1), [[0]])
self.assertEqual(
self.stream.last(2),
[[0]]) # if only one item available, should be fine to ask for two
time.sleep(d)
self.assertEqual(self.stream.last(1),
[[1]]) # make sure we return the last one
self.assertEqual(
self.stream.last(2),
[[0], [1]
]) # make sure the order is right (from older to most recent)
time.sleep(d)
self.assertEqual(self.stream.last(3), [[0], [1], [2]])
time.sleep(d)
self.assertEqual(self.stream.last(3), [[1], [2], [3]])
self.assertEqual(
self.stream.last(4),
[[1], [2], [3]]) # make sure the buffer lenght (here, 3) is used
def test_subscribe(self):
d = 1 / float(self.freq)
self.ok = False
self.i = 0
self.stream.subscribe(self.on_data)
time.sleep(d + d * 0.5)
self.assertTrue(self.ok)
self.ok = False
time.sleep(d)
self.assertTrue(self.ok)
def test_unsubscribe(self):
d = 1 / float(self.freq)
self.ok = False
self.i = 0
self.stream.subscribe(self.on_data)
time.sleep(d + d * 0.5)
self.assertTrue(self.ok)
self.ok = False
self.stream.unsubscribe(self.on_data)
time.sleep(d)
self.assertFalse(self.ok)
def on_data(self, record):
self.ok = True
self.assertEqual(record, [self.i])
self.i += 1
def tearDown(self):
self.com.close()
self._server.close()
for fname in files:
print "OPENING NEW FILE:", fname
f = open(indir + fname)
data = f.read()
f.close()
newf = open(outdir + fname, 'w')
newf.write('\n')
stream = Stream(data)
while True:
#read '('
while True:
char = stream.get()
if char == '(' or stream.eof():
break
if stream.eof():
break
root = Node()
root.read(stream)
#insert the structure into standard PTB
sline = slines[supto].strip()
factors = []
while sline != "---":
print "inserting:", sline
label, start, end = sline.split()
class Tokenizer:
class ParseError(RuntimeError):
pass
class ModuleError(RuntimeError):
pass
def parse(self, src):
self.strm = Stream(src)
self.tokens = []
s = self.strm
m = 'text block'
buf = ''
while not s.eof():
c = s.get()
if m == 'text block':
if c == '{' and s.cur() == '@':
if len(buf):
self.tokens.append(Token(kind='text-block', value=buf))
buf = ''
s.get()
self.tokens.append(Token(kind='lbraceat', value='{@'))
m = 'code block'
elif c == '{' and s.cur() == '{':
if len(buf):
self.tokens.append(Token(kind='text-block', value=buf))
buf = ''
s.get()
self.tokens.append(Token(kind='ldbrace', value='{{'))
m = 'ref block'
else:
buf += c
elif m == 'ref block':
if c == '}' and s.cur() == '}':
s.get()
self.tokens.append(Token(kind='rdbrace', value='}}'))
m = 'text block'
elif self.is_identifier_char(c):
s.prev()
self.read_identifier()
elif c == '(':
self.tokens.append(Token(kind='lparen', value='('))
elif c == ')':
self.tokens.append(Token(kind='rparen', value=')'))
elif c == '.':
self.tokens.append(Token(kind='operator', value='.'))
elif c == '"':
s.prev()
self.read_string()
elif m == 'code block':
if c == '@' and s.cur() == '}':
s.get()
self.tokens.append(Token(kind='rbraceat', value='@}'))
m = 'text block'
elif c == '.':
self.tokens.append(Token(kind='operator', value='.'))
elif c == ',':
self.tokens.append(Token(kind='comma', value=','))
elif c == '(':
self.tokens.append(Token(kind='lparen', value='('))
elif c == ')':
self.tokens.append(Token(kind='rparen', value=')'))
elif c == ':':
self.tokens.append(Token(kind='colon', value=':'))
elif c == '=':
s.prev()
self.read_assign()
elif c == '!':
s.prev()
self.read_not()
elif c == '<':
s.prev()
self.read_lt()
elif c == '>':
s.prev()
self.read_gt()
elif c == '+':
s.prev()
self.read_add()
elif c == '-':
s.prev()
self.read_sub()
elif c == '*':
self.tokens.append(Token(kind='operator', value='*'))
elif c == '/':
self.tokens.append(Token(kind='operator', value='/'))
elif c == '"':
s.prev()
self.read_string()
elif c.isdigit():
s.prev()
self.read_digit()
elif self.is_identifier_char(c):
s.prev()
self.read_identifier()
elif c.isspace():
pass
else:
raise Tokenizer.ParseError('unsupprted character %s' % c)
if len(buf):
self.tokens.append(Token(kind='text-block', value=buf))
buf = ''
return self.tokens
def read_add(self):
c = self.strm.get()
if c != '+':
raise Tokenizer.ModuleError('not found "+"')
c = self.strm.get()
if c == '+':
self.tokens.append(Token(kind='operator', value='++'))
else:
self.strm.prev()
self.tokens.append(Token(kind='operator', value='+'))
def read_sub(self):
c = self.strm.get()
if c != '-':
raise Tokenizer.ModuleError('not found "-"')
c = self.strm.get()
if c == '-':
self.tokens.append(Token(kind='operator', value='--'))
else:
self.strm.prev()
self.tokens.append(Token(kind='operator', value='-'))
def read_gt(self):
c = self.strm.get()
if c != '>':
raise Tokenizer.ModuleError('not found ">"')
c = self.strm.get()
if c == '=':
self.tokens.append(Token(kind='comp_op', value='>='))
else:
self.strm.prev()
self.tokens.append(Token(kind='comp_op', value='>'))
def read_lt(self):
c = self.strm.get()
if c != '<':
raise Tokenizer.ModuleError('not found "<"')
c = self.strm.get()
if c == '=':
self.tokens.append(Token(kind='comp_op', value='<='))
else:
self.strm.prev()
self.tokens.append(Token(kind='comp_op', value='<'))
def read_not(self):
c = self.strm.get()
if c != '!':
raise Tokenizer.ModuleError('not found "!"')
c = self.strm.get()
if c == '=':
self.tokens.append(Token(kind='comp_op', value='!='))
else:
self.strm.prev()
self.tokens.append(Token(kind='comp_op', value='!'))
def read_digit(self):
buf = ''
while not self.strm.eof():
c = self.strm.get()
if not c.isdigit():
self.strm.prev()
break
buf += c
if not len(buf):
raise Tokenizer.ModuleError('not found digit')
self.tokens.append(Token(kind='digit', value=int(buf)))
def read_assign(self):
c = self.strm.get()
if c != '=':
raise Tokenizer.ModuleError('not found "="')
c = self.strm.get()
if c == '=':
self.tokens.append(Token(kind='comp_op', value='=='))
else:
self.strm.prev()
self.tokens.append(Token(kind='operator', value='='))
def is_identifier_char(self, c):
return c.isalpha() or c.isdigit() or c == '_'
def read_identifier(self):
t = Token(kind='identifier', value='')
while not self.strm.eof():
c = self.strm.get()
if not self.is_identifier_char(c):
self.strm.prev()
break
else:
t.value += c
if not len(t.value):
raise Tokenizer.ModuleError('not found identifier') # impossible
if t.value == 'import':
t.kind = 'import'
elif t.value == 'if':
t.kind = 'if'
elif t.value == 'elif':
t.kind = 'elif'
elif t.value == 'else':
t.kind = 'else'
elif t.value == 'end':
t.kind = 'end'
self.tokens.append(t)
def read_string(self):
c = self.strm.get()
if c != '"':
raise Tokenizer.ModuleError("not found '"'')
t = Token(kind='string', value='')
while not self.strm.eof():
c = self.strm.get()
if c == '"':
break
else:
t.value += c
self.tokens.append(t)
def read_lbrace(self):
c = self.strm.get()
if c != '{':
raise Tokenizer.ModuleError('not found "{"')
c = self.strm.get()
if c == '{':
self.tokens.append(Token(kind='ldbrace', value='{{'))
elif c == '@':
self.tokens.append(Token(kind='lbraceat', value='{@'))
else:
self.strm.prev()
self.tokens.append(Token(kind='lbrace', value='{'))
def read_rbrace(self):
c = self.strm.get()
if c != '}':
raise Tokenizer.ModuleError('not found "}"')
c = self.strm.get()
if c == '}':
self.tokens.append(Token(kind='rdbrace', value='}}'))
else:
self.strm.prev()
self.tokens.append(Token(kind='rbrace', value='}'))
def read_at(self):
c = self.strm.get()
if c != '@':
raise Tokenizer.ModuleError('not found "@"')
c = self.strm.get()
if c == '}':
self.tokens.append(Token(kind='rbraceat', value='@}'))
else:
raise Tokenizer.ParseError('invalid token. "@" is not supported')
class AST:
class ModuleError(RuntimeError):
pass
class SyntaxError(SyntaxError):
pass
class ImportError(ImportError):
pass
class NameError(NameError):
pass
class ReferenceError(RuntimeError):
pass
class TypeError(TypeError):
pass
def parse(self, tokens, debug=False):
self.debug_parse = debug
self.strm = Stream(tokens)
self.root = self.block(dep=0)
def traverse(self, debug=False, opts=None):
self.debug_traverse = debug
self.opts = opts
self.context = Context()
self._traverse(self.root, dep=0)
return self.context
def show_traverse(self, node, dep):
if self.debug_traverse:
print(('_'*dep) + str(dep) + ' ' + str(node).split('.')[1].split(' ')[0])
def _traverse(self, node, dep):
if node is None:
return
self.show_traverse(node, dep)
if isinstance(node, BlockNode):
self._traverse(node.text_block, dep=dep+1)
self._traverse(node.code_block, dep=dep+1)
self._traverse(node.ref_block, dep=dep+1)
self._traverse(node.block, dep=dep+1)
elif isinstance(node, TextBlockNode):
self.context.buffer += node.text
elif isinstance(node, CodeBlockNode):
self._traverse(node.formula, dep=dep+1)
elif isinstance(node, RefBlockNode):
self.traverse_ref_block(node, dep=dep+1)
elif isinstance(node, FormulaNode):
if node.comparison:
self.context.last_expr_val = self._traverse(node.comparison, dep=dep+1)
return self.context.last_expr_val
elif node.assign_expr:
self._traverse(node.assign_expr, dep=dep+1)
elif node.import_:
self._traverse(node.import_, dep=dep+1)
elif node.caller:
self._traverse(node.caller, dep=dep+1)
elif node.if_:
self._traverse(node.if_, dep=dep+1)
if node.formula:
self._traverse(node.formula, dep=dep+1)
elif node.block:
self._traverse(node.block, dep=dep+1)
elif isinstance(node, ComparisonNode):
if node.expr and node.comparison:
lval = self._traverse(node.expr, dep=dep+1)
rval = self._traverse(node.comparison, dep=dep+1)
if node.op == '>':
return lval > rval
elif node.op == '<':
return lval < rval
elif node.op == '>=':
return lval >= rval
elif node.op == '<=':
return lval <= rval
elif node.op == '==':
return lval == rval
elif node.op == '!=':
return lval != rval
else:
raise AST.ModuleError('unsupported comparison operator "%s"' % node.op)
elif node.expr:
return self._traverse(node.expr, dep=dep+1)
else:
raise AST.ModuleError('unsupported node %s in comparison' % str(node))
elif isinstance(node, IfNode):
result = self._traverse(node.comparison, dep=dep+1)
if result:
if node.formula:
self._traverse(node.formula, dep=dep+1)
elif node.block:
self._traverse(node.block, dep=dep+1)
else:
if node.elif_:
self._traverse(node.elif_, dep=dep+1)
elif node.else_:
self._traverse(node.else_, dep=dep+1)
elif isinstance(node, ElseNode):
if node.block:
self._traverse(node.block, dep=dep+1)
elif node.formula:
self._traverse(node.formula, dep=dep+1)
elif isinstance(node, ExprNode):
if node.term and node.expr:
lval = self._traverse(node.term, dep=dep+1)
rval = self._traverse(node.expr, dep=dep+1)
if node.op == '+':
return lval + rval
elif node.op == '-':
return lval - rval
else:
raise AST.ModuleError('unsupported operation "%s" in traverse expr' % node.op)
elif node.term:
return self._traverse(node.term, dep=dep+1)
else:
raise AST.ModuleError('programming error. impossible case in traverse expr')
elif isinstance(node, TermNode):
if node.factor and node.term:
lval = self._traverse(node.factor, dep=dep+1)
rval = self._traverse(node.term, dep=dep+1)
if node.op == '*':
return lval * rval
elif node.op == '/':
return lval / rval
else:
raise AST.ModuleError('unsupported operation "%s" in traverse term' % node.op)
elif node.factor:
return self._traverse(node.factor, dep=dep+1)
else:
raise AST.ModuleError('programming error. impossible case in traverse term')
elif isinstance(node, FactorNode):
if node.expr:
return self._traverse(node.expr, dep=dep+1)
elif node.identifier != None:
if node.identifier not in self.context.syms.keys():
raise AST.ReferenceError('%s is not defined' % node.identifier)
return self.context.syms[node.identifier]
elif node.digit:
return self._traverse(node.digit, dep=dep+1)
elif node.string != None:
return node.string
elif node.callable:
return self._traverse(node.callable, dep=dep+1)
elif isinstance(node, DigitNode):
return node.value
elif isinstance(node, AssignExprNode):
return self.traverse_assign_expr(node, dep+1)
elif isinstance(node, ImportNode):
self.traverse_import(node, dep+1)
elif isinstance(node, CallerNode):
self.traverse_caller(node, dep+1)
elif isinstance(node, CallableNode):
package = node.caller_list.identifier
if package == 'opts':
if node.caller_list.caller_list.identifier == 'get':
if self.opts and node.args.arg.string in self.opts.keys():
return self.opts[node.args.arg.string]
else:
return ''
else:
raise AST.ModuleError('impossible. not supported node', type(node))
def traverse_assign_expr(self, node, dep):
lhs = node.assign_operand_lhs
op = node.assign_operator
rhs = node.assign_operand_rhs
assexpr = node.assign_expr
if lhs is None and rhs:
if rhs.expr:
return self._traverse(rhs.expr, dep+1)
elif rhs.callable:
return self._traverse(rhs.callable, dep+1)
elif rhs.string != None:
return rhs.string
elif rhs.identifier != None:
if rhs.identifier not in self.context.syms.keys():
raise AST.ReferenceError('"%s" is not defined' % rhs.identifier)
elif lhs and op and assexpr:
if op.operator == '=':
self.context.syms[lhs.identifier] = self._traverse(assexpr, dep=dep+1)
self.context.last_expr_val = self.context.syms[lhs.identifier]
return self.context.syms[lhs.identifier]
else:
raise AST.ModuleError('unsupported operator "%s"' % op.operator)
else:
raise AST.ModuleError('invalid case in traverse assign expr')
def traverse_ref_block(self, node, dep):
if node.identifier != None:
if node.identifier in self.context.syms.keys():
self.context.buffer += str(self.context.syms[node.identifier])
elif node.callable:
result = self._traverse(node.callable, dep=dep+1)
self.context.buffer += str(result)
def traverse_import(self, node, dep):
if node.identifier == 'alias':
self.context.imported_alias = True
elif node.identifier == 'config':
self.context.imported_config = True
else:
raise AST.ImportError('can not import package "%s"' % node.identifier)
def traverse_caller(self, node, dep):
first = node.identifiers[0]
if first == 'alias':
self.call_alias_package(node)
elif first == 'config':
self.call_config_package(node)
else:
raise AST.NameError('"%s" is not defined' % first)
def call_alias_package(self, node):
if not self.context.imported_alias:
raise AST.ImportError('alias is not imported')
method = node.identifiers[1]
if method == 'set':
if len(node.args) != 2:
AST.SyntaxError('alias.set need two arguments')
name = node.args[0]
cmd = node.args[1]
self.context.alias_map[name] = cmd
def call_config_package(self, node):
if not self.context.imported_config:
raise AST.ImportError('config is not imported')
method = node.identifiers[1]
if method == 'set':
if len(node.args) != 2:
AST.SyntaxError('config.set need two arguments')
name = node.args[0]
cmd = node.args[1]
self.context.config_map[name] = cmd
def show_parse(self, name, dep):
if self.debug_parse:
t = self.strm.cur()
print(dep, name + ': ' + str(t))
def block(self, dep=0):
self.show_parse('block', dep=dep)
if self.strm.eof():
return None
t = self.strm.cur()
if t.kind in ('rbraceat', 'end', 'if', 'elif', 'else'):
return None
node = BlockNode()
node.code_block = self.code_block(dep=dep+1)
if node.code_block is None:
node.ref_block = self.ref_block(dep=dep+1)
if node.ref_block is None:
node.text_block = self.text_block(dep=dep+1)
node.block = self.block(dep=dep+1)
return node
def ref_block(self, dep=0):
self.show_parse('ref_block', dep=dep)
if self.strm.eof():
return None
node = RefBlockNode()
t = self.strm.get()
if t.kind != 'ldbrace':
self.strm.prev()
return None
t = self.strm.get()
if t.kind == 'identifier':
t2 = self.strm.cur()
if t2 == Stream.EOF:
raise AST.SyntaxError('reference block not closed')
elif t2.value in ('.', '('):
self.strm.prev()
node.callable = self.callable()
else:
node.identifier = t.value
elif t.kind == 'rdbrace':
return node
t = self.strm.get()
if t == Stream.EOF or t.kind != 'rdbrace':
raise AST.SyntaxError('not found "rdbrace"')
return node
def callable(self, dep=0):
self.show_parse('callable', dep=dep)
if self.strm.eof():
return None
node = CallableNode()
node.caller_list = self.caller_list()
t = self.strm.get()
if t.value != '(':
raise AST.SyntaxError('not found "(" in callable')
node.args = self.args()
t = self.strm.get()
if t.value != ')':
raise AST.SyntaxError('not found ")" in callable')
return node
def caller_list(self, dep=0):
self.show_parse('caller_list', dep=dep)
if self.strm.eof():
return None
node = CallerListNode()
t = self.strm.get()
if t == Stream.EOF:
raise AST.SyntaxError('reached EOF in caller list')
elif t.kind == 'identifier':
node.identifier = t.value
else:
raise AST.SyntaxError('invalid token "%s"' % t)
t = self.strm.get()
if t.value == '.':
node.caller_list = self.caller_list()
else:
self.strm.prev()
return node
def args(self, dep=0):
self.show_parse('args', dep=dep)
if self.strm.eof():
return None
node = ArgsNode()
node.arg = self.arg()
t = self.strm.get()
if t.value == ',':
node.args = self.args()
else:
self.strm.prev()
return node
def arg(self, dep=0):
self.show_parse('arg', dep=dep)
if self.strm.eof():
return None
node = ArgNode()
t = self.strm.get()
if t == Stream.EOF:
raise AST.SyntaxError('reached EOF in argument')
elif t.kind == 'digit':
node.digit = DigitNode()
node.digit.value = t.value
elif t.kind == 'string':
node.string = t.value
elif t.kind == 'identifier':
node.identifier = t.value
else:
raise AST.SyntaxError('not supported argument type "%s"' % t)
return node
def code_block(self, dep=0):
self.show_parse('code_block', dep=dep)
if self.strm.eof():
return None
t = self.strm.get()
if t.kind != 'lbraceat':
self.strm.prev()
return None
node = CodeBlockNode()
node.formula = self.formula(dep=dep+1)
t = self.strm.get()
if t == Stream.EOF:
pass
elif t.kind in ('end'):
self.strm.prev()
return node
elif t.kind in ('ldbrace', 'else', 'elif'):
self.strm.prev()
return node
def text_block(self, dep=0):
self.show_parse('text_block', dep=dep)
if self.strm.eof():
return None
t = self.strm.get()
if t.kind != 'text-block':
self.strm.prev()
return None
node = TextBlockNode()
node.text = t.value
return node
def formula(self, dep=0):
self.show_parse('formula', dep=dep)
if self.strm.eof():
return None
t = self.strm.cur()
if t.kind in ('rbraceat', 'ldbrace', 'end', 'elif', 'else'):
return None
if t.kind in ('colon'):
raise AST.SyntaxError('found "%s". invalid formula.' % t)
node = FormulaNode()
t = self.strm.cur()
if t.kind == 'import':
node.import_ = self.import_(dep=dep+1)
elif t.kind == 'if':
node.if_ = self.if_(dep=dep+1)
else:
node.caller = self.caller(dep=dep+1)
if node.caller is None:
t2 = self.strm.cur(1)
if t2.value == '=':
node.assign_expr = self.assign_expr(dep=dep+1)
if not self.strm.eof() and node.assign_expr is None:
return None
else:
node.comparison = self.comparison(dep=dep+1)
if not self.strm.eof() and node.comparison is None:
return None
t = self.strm.get()
if t == Stream.EOF:
return node
if t.kind == 'rbraceat' and self.strm.cur() != Stream.EOF:
node.block = self.block(dep=dep+1)
t = self.strm.get()
if t == Stream.EOF:
pass
elif t.kind == 'lbraceat':
pass
else:
self.strm.prev()
else:
self.strm.prev()
node.formula = self.formula(dep=dep+1)
return node
def comparison(self, dep=0):
self.show_parse('comparison', dep=dep)
if self.strm.eof():
return None
node = ComparisonNode()
node.expr = self.expr(dep=dep+1)
t = self.strm.get()
if t == Stream.EOF:
return node
elif t.kind != 'comp_op':
self.strm.prev()
return node
node.op = t.value
node.comparison = self.comparison(dep=dep+1)
return node
def if_(self, dep=0, first_symbol='if'):
self.show_parse('if', dep=dep)
if self.strm.eof():
return None
t = self.strm.get()
if t.kind != first_symbol:
return None
node = IfNode()
node.comparison = self.comparison(dep=dep+1)
if node.comparison is None:
raise AST.SyntaxError('invalid if statement. not found comparison')
t = self.strm.get()
if t.kind != 'colon':
raise AST.SyntaxError('invalid if statement. not found colon')
t = self.strm.get()
if t.kind == 'rbraceat': # @}
node.block = self.block(dep=dep+1)
else:
self.strm.prev()
node.formula = self.formula(dep=dep+1)
t = self.strm.get()
if t == Stream.EOF:
raise AST.SyntaxError('reached EOF in if statement')
if t.kind != 'lbraceat':
self.strm.prev()
t = self.strm.get()
if t.kind == 'end':
pass
elif t.kind == 'elif':
self.strm.prev()
node.elif_ = self.if_(dep=dep+1, first_symbol='elif')
elif t.kind == 'else':
self.strm.prev()
node.else_ = self.else_(dep=dep+1)
else:
raise AST.SyntaxError('not ended in if statement. token is %s' % t)
return node
def else_(self, dep=0):
self.show_parse('else', dep=dep)
if self.strm.eof():
return None
node = ElseNode()
t = self.strm.get()
if t.kind != 'else':
raise AST.SyntaxError('invalid else statement. not found "else"')
t = self.strm.get()
if t.kind != 'colon':
raise AST.SyntaxError('invalid else statement. not found colon')
t = self.strm.get()
if t.kind == 'rbraceat':
node.block = self.block(dep=dep+1)
else:
self.strm.prev()
node.formula = self.formula(dep=dep+1)
t = self.strm.get()
if t == Stream.EOF:
return node
if t.kind != 'lbraceat':
self.strm.prev()
t = self.strm.get()
if t.kind == 'end':
pass
else:
self.strm.prev()
return node
def expr(self, dep=0):
self.show_parse('expr', dep=dep)
if self.strm.eof():
return None
node = ExprNode()
node.term = self.term(dep=dep+1)
t = self.strm.get()
if t == Stream.EOF:
return node
elif t.value not in ('+', '-'):
self.strm.prev()
return node
node.op = t.value
node.expr = self.expr(dep=dep+1)
return node
def term(self, dep=0):
self.show_parse('term', dep=dep)
if self.strm.eof():
return None
node = TermNode()
node.factor = self.factor(dep=dep+1)
t = self.strm.get()
if t == Stream.EOF:
return node
elif t.value not in ('*', '/'):
self.strm.prev()
return node
node.op = t.value
node.term = self.term(dep=dep+1)
return node
def factor(self, dep=0):
self.show_parse('factor', dep=dep)
if self.strm.eof():
return None
node = FactorNode()
t = self.strm.get()
if t == Stream.EOF:
raise AST.SyntaxError('reached EOF in factor')
elif t.kind == 'lparen':
node.expr = self.expr(dep=dep+1)
t = self.strm.get()
if t == Stream.EOF:
raise AST.SyntaxError('not found rparen in factor. reached EOF')
if t.kind != 'rparen':
raise AST.SyntaxError('not found rparen in factor. token is %s' % t)
elif t.kind == 'digit':
node.digit = DigitNode()
node.digit.value = t.value
elif t.kind == 'identifier':
self.strm.prev()
if self.is_callable(dep=dep+1):
node.callable = self.callable(dep=dep+1)
else:
t = self.strm.get()
node.identifier = t.value
elif t.kind == 'string':
node.string = t.value
else:
self.strm.prev()
return None
return node
def is_callable(self, dep):
self.show_parse('is_callable', dep=dep)
if self.strm.eof():
return None
ret = True
i = self.strm.index
while not self.strm.eof():
t = self.strm.get()
if t == Stream.EOF:
ret = False
break
elif t.kind == 'identifier':
pass
elif t.value == '.':
pass
elif t.value == '(':
break
else:
ret = False
break
self.strm.index = i
return ret
def digit(self, dep=0):
self.show_parse('digit', dep=dep)
if self.strm.eof():
return None
t = self.strm.get()
if t.kind != 'digit':
raise AST.ModuleError('not found digit')
node = DigitNode()
node.value = t.value
return node
def assign_expr(self, dep=0):
self.show_parse('assign_expr', dep=dep)
if self.strm.eof():
return None
node = AssignExprNode()
i = self.strm.index
node.assign_operand_lhs = self.assign_operand_lhs(dep=dep+1)
if node.assign_operand_lhs:
node.assign_operator = self.assign_operator(dep=dep+1)
if node.assign_operator is None:
node.assign_operand_lhs = None
self.strm.index = i
node.assign_operand_rhs = self.assign_operand_rhs(dep=dep+1)
else:
node.assign_expr = self.assign_expr(dep=dep+1)
else:
node.assign_operand_rhs = self.assign_operand_rhs(dep=dep+1)
return node
def assign_operand_rhs(self, dep=0):
self.show_parse('assign_operand_rhs', dep=dep)
if self.strm.eof():
return None
node = AssignOperandRhsNode()
t = self.strm.get()
if t.kind == 'string':
node.string = t.value
elif t.kind == 'identifier':
self.strm.prev()
if self.is_callable(dep=dep+1):
node.callable = self.callable()
else:
node.expr = self.expr(dep=dep+1)
if node.expr is None:
t = self.strm.get()
node.identifier = t.value
else:
self.strm.prev()
node.expr = self.expr(dep=dep+1)
if node.expr is None:
return None
return node
def assign_operator(self, dep=0):
self.show_parse('assign_operator', dep=dep)
if self.strm.eof():
return None
t = self.strm.get()
if t.value != '=':
self.strm.prev()
return None
node = AssignOperatorNode()
node.operator = t.value
return node
def assign_operand_lhs(self, dep=0):
self.show_parse('assign_operand_lhs', dep=dep)
if self.strm.eof():
return None
elif self.is_callable(dep=dep+1):
return None
node = AssignOperandLhsNode()
t = self.strm.get()
if t.kind != 'identifier':
self.strm.prev()
return None
node.identifier = t.value
return node
def import_(self, dep=0):
self.show_parse('import', dep=dep)
if self.strm.eof():
return None
node = ImportNode()
t = self.strm.get()
if t.kind != 'import':
self.strm.prev()
return None
t = self.strm.get()
if t.kind != 'identifier':
raise AST.SyntaxError('invalid import statement')
node.identifier = t.value
return node
def caller(self, dep=0):
"""
ここがしんどい
もっと抽象化するべき
"""
self.show_parse('caller', dep=dep)
if self.strm.eof():
return None
# find lparen
i = self.strm.index
found = False
while not self.strm.eof():
t = self.strm.get()
if t.kind == 'identifier':
pass
elif t.kind == 'operator' and t.value == '.':
pass
elif t.kind == 'lparen':
found = True
break
else:
break
self.strm.index = i
if not found:
return None
node = CallerNode()
save_index = self.strm.index
while not self.strm.eof():
t = self.strm.get()
if t.kind == 'identifier':
node.identifiers.append(t.value)
elif t.kind == 'lparen':
break
if not len(node.identifiers):
self.strm.index = save_index
return None
while not self.strm.eof():
t = self.strm.get()
if t.kind == 'string':
node.args.append(t.value)
elif t.kind == 'rparen':
break
return node