Python token Examples

Example #1

def _parse_lll(tokens, pos):
    m, sv, o = tokens[pos].metadata, tokens[pos].val, []
    if sv not in ['(', '{', '[', '@', '@@']:
        return tokens[pos], pos + 1
    elif sv == '@':
        node, pos = _parse_lll(tokens, pos + 1)
        return parser.astnode('mload', [node], *m), pos
    elif sv == '@@':
        node, pos = _parse_lll(tokens, pos + 1)
        return parser.astnode('sload', [node], *m), pos
    elif sv == '(':
        pos, o, watch = pos + 1, [], ')'
    elif sv == '{':
        pos, o, watch = pos + 1, [parser.token('seq')], '}'
    elif sv == '[':
        sv, pos, o, watch = '[[', pos + 1, [parser.token('mstore')], ']'
    else:
        raise Exception(sv)
    while tokens[pos].val != watch:
        sub, pos = _parse_lll(tokens, pos)
        o.append(sub)
    pos += 1
    if sv in ['[', '[['] and tokens[pos].val != ']':
        sub, pos = _parse_lll(tokens, pos)
        o.append(sub)
    if len(o) == 0:
        o.append(parser.token('seq'))
    if len(o) == 3 and isinstance(o[0], parser.token) and o[0].val == 'mstore' and \
            isinstance(o[1], parser.astnode) and len(o[1].args) == 1 and \
            o[1].fun == 'mstore':
        o = [parser.token('sstore'), o[1].args[0], o[2]]
    return parser.astnode(o[0].val, o[1:], *m), pos

Example #2

File: lllparser.py Project: jamslevy/serpent

def _parse_lll(tokens, pos):
    m, sv, o = tokens[pos].metadata, tokens[pos].val, []
    if sv not in ['(', '{', '[', '@', '@@']:
        return tokens[pos], pos + 1
    elif sv == '@':
        node, pos = _parse_lll(tokens, pos+1)
        return parser.astnode('mload', [node], *m), pos
    elif sv == '@@':
        node, pos = _parse_lll(tokens, pos+1)
        return parser.astnode('sload', [node], *m), pos
    elif sv == '(':
        pos, o, watch = pos + 1, [], ')'
    elif sv == '{':
        pos, o, watch = pos + 1, [parser.token('seq')], '}'
    elif sv == '[':
        sv, pos, o, watch = '[[', pos + 1, [parser.token('mstore')], ']'
    else:
        raise Exception(sv)
    while tokens[pos].val != watch:
        sub, pos = _parse_lll(tokens, pos)
        o.append(sub)
    pos += 1
    if sv in ['[', '[['] and tokens[pos].val != ']':
        sub, pos = _parse_lll(tokens, pos)
        o.append(sub)
    if len(o) == 0:
        o.append(parser.token('seq'))
    if len(o) == 3 and isinstance(o[0], parser.token) and o[0].val == 'mstore' and \
            isinstance(o[1], parser.astnode) and len(o[1].args) == 1 and \
            o[1].fun == 'mstore':
        o = [parser.token('sstore'), o[1].args[0], o[2]]
    return parser.astnode(o[0].val, o[1:], *m), pos

Example #3

File: parse_subst.py Project: marekyggdrasil/proofofconcept

def expression(expr):
    expr = token(expr)

    def foo(callback):
        expr.callback = callback
        return expr

    return foo

Example #4

File: parse_subst.py Project: WojciechMula/pydvi2svg

def expression(expr):
    expr = token(expr)

    def foo(callback):
        expr.callback = callback
        return expr

    return foo

Example #5

File: parse_subst.py Project: stjordanis/pydvi2svg

		self.margin   = (0.0, 0.0, 0.0, 0.0)
		self.position = ('inherit', 1.0) # inherit, bottom
	def __str__(self):
		return "{%s}" % ", ".join(self.__dict__.keys())
	
	__repr__ = __str__


@expression(alt(quoted_string, xml_id, rect, point))
def target(l, s, r):
	if type(r[0]) is str:
		return [('string', r[0])]
	else:
		return r

arrow  = eat(token(re.compile(r'(?:->|=>|=)')))

@expression("(this)")
def this(l, s, r):
	return [None]

source = alt(quoted_string, this)

rule = seq(target, arrow, source, opt(position), opt(margin), opt(scale))
@expression(rule)
def rule(l, s, r):
	# target, source, (...other data...)
	data = record()
	for item in r[2:]:
		value = item[1:]
		if len(value) == 1:

Example #6

File: example_s_expression.py Project: takushi-m/python-parser-combinator

import re
import parser as p
import unittest

number = p.map(p.regexp(re.compile("[1-9][0-9]*|[0-9]")),lambda x:int(x))
symbol = p.regexp(re.compile("[^\s\[\()]"))
popen = p.token("(")
pclose = p.token(")")
ws = p.regexp(re.compile("\s*"))
atom = p.choice(number,symbol)
expression = p.lazy(lambda:p.choice(
    p.map(p.seq(ws,atom,ws),lambda parsed:parsed[1])
    ,p.map(p.seq(popen,p.many(expression),pclose),lambda parsed:parsed[1])
))

class TestSExpression(unittest.TestCase):
    def test_number(self):
        self.assertEqual(number("1",0), [True, 1, 1])
        self.assertEqual(number("123",0), [True, 123, 3])
        self.assertEqual(number("a1",0), [False, None, 0])

    def test_sexp(self):
        self.assertEqual(expression("(+ 1 2)",0),[True,["+",1,2],7])
        self.assertEqual(expression("(+ 1 (- 3 4))",0),[True,["+",1,["-",3,4]],13])


if __name__ == '__main__':
    unittest.main()

Example #7

File: parse_subst.py Project: WojciechMula/pydvi2svg

    def __str__(self):
        return "{%s}" % ", ".join(self.__dict__.keys())

    __repr__ = __str__


@expression(alt(quoted_string, xml_id, rect, point))
def target(l, s, r):
    if type(r[0]) is str:
        return [("string", r[0])]
    else:
        return r


arrow = eat(token(re.compile(r"(?:->|=>|=)")))


@expression("(this)")
def this(l, s, r):
    return [None]


source = alt(quoted_string, this)

rule = seq(target, arrow, source, opt(position), opt(margin), opt(scale))


@expression(rule)
def rule(l, s, r):
    # target, source, (...other data...)