def do_call(t,r=None):
r = get_tmp(r)
items = t.items
fnc = do(items[0])
a,b,c,d = p_filter(t.items[1:])
e = None
if len(b) != 0 or d != None:
e = do(Token(t.pos,'dict',None,[])); un_tmp(e);
for p in b:
p.items[0].type = 'string'
t1,t2 = do(p.items[0]),do(p.items[1])
#code('SET',e,t1,t2)
emitset(e,t1,t2)
free_tmp(t1); free_tmp(t2) #REG
if d: free_tmp(do(Token(t.pos,'call',None,[Token(t.pos,'name','merge'),Token(t.pos,'reg',e),d.items[0]]))) #REG
manage_seq('PARAMS',r,a)
if c != None:
t1,t2 = _do_string('*'),do(c.items[0])
#code('SET',r,t1,t2)
emitset(r,t1,t2)
free_tmp(t1); free_tmp(t2) #REG
if e != None:
t1 = _do_none()
#code('SET',r,t1,e)
emitset(r,t1,e)
free_tmp(t1) #REG
xml('CALL','ret',r,'func',fnc,'params',r)
free_tmp(fnc) #REG
return r
def do_class(t):
tok = t
items = t.items
parent = None
if items[0].type == 'name':
name = items[0].val
parent = Token(tok.pos, 'name', 'object')
else:
name = items[0].items[0].val
parent = items[0].items[1]
kls = do(Token(t.pos, 'dict', 0, []))
un_tmp(kls)
ts = _do_string(name)
code(GSET, ts, kls)
free_tmp(ts) #REG
free_tmp(
do(
Token(tok.pos, 'call', None, [
Token(tok.pos, 'name', 'setmeta'),
Token(tok.pos, 'reg', kls), parent
])))
for fc in items[1].items:
if fc.type != 'def': continue
do_def(fc, kls)
free_reg(kls) #REG
def do_call(t, r=None):
r = get_tmp(r)
items = t.items
fnc = do(items[0])
a, b, c, d = p_filter(t.items[1:])
e = None
if len(b) != 0 or d != None:
e = do(Token(t.pos, 'dict', None, []))
un_tmp(e)
for p in b:
p.items[0].type = 'string'
t1, t2 = do(p.items[0]), do(p.items[1])
code(SET, e, t1, t2)
free_tmp(t1)
free_tmp(t2) #REG
if d:
free_tmp(
do(
Token(t.pos, 'call', None, [
Token(t.pos, 'name', 'merge'),
Token(t.pos, 'reg', e), d.items[0]
]))) #REG
manage_seq(PARAMS, r, a)
if c != None:
t1, t2 = _do_string('*'), do(c.items[0])
code(SET, r, t1, t2)
free_tmp(t1)
free_tmp(t2) #REG
if e != None:
t1 = _do_none()
code(SET, r, t1, e)
free_tmp(t1) #REG
code(CALL, r, fnc, r)
free_tmp(fnc) #REG
return r
def do_import(t):
for mod in t.items:
mod.type = 'string'
v = do_call(
Token(t.pos, 'call', None, [Token(t.pos, 'name', 'import'), mod]))
mod.type = 'name'
do_set_ctx(mod, Token(t.pos, 'reg', v))
def do_class(t):
tok = t
items = t.items
parent = None
if items[0].type == 'name':
name = items[0].val
parent = Token(tok.pos, 'name', 'object')
else:
name = items[0].items[0].val
parent = items[0].items[1]
kls = do(Token(t.pos, 'dict', 0, []))
un_tmp(kls)
ts = _do_string(name)
code(GSET, ts, kls)
free_tmp(ts) #REG
free_tmp(
do(
Token(tok.pos, 'call', None, [
Token(tok.pos, 'name', 'setmeta'),
Token(tok.pos, 'reg', kls), parent
])))
for member in items[1].items:
if member.type == 'def': do_def(member, kls)
elif member.type == 'symbol' and member.val == '=':
do_classvar(member, kls)
else:
continue
free_reg(kls) #REG
def do_set_ctx(k, v):
if k.type == 'name':
if (D._globals and k.val not in D.vars) or (k.val in D.globals):
#c = do_string(k)
b = do(v)
#code('<GSET gVarName="'+k.val+'" reg="'+str(b)+'" />','','','')
emitgset(k.val, b)
#free_tmp(c)
free_tmp(b)
return
a = do_local(k)
b = do(v)
#code('<MOVE to="'+str(a)+'" from="'+str(b)+'" />', '', '', '')
emitmove(a, b)
free_tmp(b)
return a
elif k.type in ('tuple', 'list'):
if v.type in ('tuple', 'list'):
n, tmps = 0, []
for kk in k.items:
vv = v.items[n]
tmp = get_tmp()
tmps.append(tmp)
r = do(vv)
#code('< MOVE to="'+str(tmp)+'" from="'+str(r)+'" />', '', '', '')
emitmove(tmp, r)
free_tmp(r) #REG
n += 1
n = 0
for kk in k.items:
vv = v.items[n]
tmp = tmps[n]
free_tmp(do_set_ctx(kk, Token(vv.pos, 'reg', tmp))) #REG
n += 1
return
r = do(v)
un_tmp(r)
n, tmp = 0, Token(v.pos, 'reg', r)
for tt in k.items:
free_tmp(
do_set_ctx(
tt,
Token(tmp.pos, 'get', None,
[tmp, Token(tmp.pos, 'number', str(n))]))) #REG
n += 1
free_reg(r)
return
r = do(k.items[0])
rr = do(v)
tmp = do(k.items[1])
#code('SET',r,tmp,rr)
emitset(r, tmp, rr)
free_tmp(r) #REG
free_tmp(tmp) #REG
return rr
def do_set_ctx(k, v):
if k.type == 'name':
if (D._globals and k.val not in D.vars) or (k.val in D.globals):
c = do_string(k)
b = do(v)
code(GSET, c, b)
free_tmp(c)
free_tmp(b)
return
a = do_local(k)
b = do(v)
code(MOVE, a, b)
free_tmp(b)
return a
elif k.type in ('tuple', 'list'):
if v.type in ('tuple', 'list'):
n, tmps = 0, []
for kk in k.items:
vv = v.items[n]
tmp = get_tmp()
tmps.append(tmp)
r = do(vv)
code(MOVE, tmp, r)
free_tmp(r) #REG
n += 1
n = 0
for kk in k.items:
vv = v.items[n]
tmp = tmps[n]
free_tmp(do_set_ctx(kk, Token(vv.pos, 'reg', tmp))) #REG
n += 1
return
r = do(v)
un_tmp(r)
n, tmp = 0, Token(v.pos, 'reg', r)
for tt in k.items:
free_tmp(
do_set_ctx(
tt,
Token(tmp.pos, 'get', None,
[tmp, Token(tmp.pos, 'number', str(n))]))) #REG
n += 1
free_reg(r)
return
r = do(k.items[0])
rr = do(v)
tmp = do(k.items[1])
code(SET, r, tmp, rr)
free_tmp(r) #REG
free_tmp(tmp) #REG
return rr
def do_def(tok,kls=None):
items = tok.items
t = get_tag()
rf = fnc(t,'end')
D.begin()
setpos(tok.pos)
r = do_local(Token(tok.pos,'name','__params'))
do_info(items[0].val)
a,b,c,d = p_filter(items[1].items)
for p in a:
v = do_local(p)
tmp = _do_none()
#code('GET','toreg='+str(v),r,tmp)
emitget(v,r,tmp)
free_tmp(tmp) #REG
for p in b:
v = do_local(p.items[0])
do(p.items[1],v)
tmp = _do_none()
code('IGET',v,r,tmp)
free_tmp(tmp) #REG
if c != None:
v = do_local(c.items[0])
tmp = _do_string('*')
#code('GET','toreg='+str(v),r,tmp)
emitget(v,r,tmp)
free_tmp(tmp) #REG
if d != None:
e = do_local(d.items[0])
#code('DICT',e,0,0)
xml('DICT',"reg",e)
tmp = _do_none()
code('IGET',e,r,tmp)
free_tmp(tmp) #REG
free_tmp(do(items[2])) #REG
D.end()
tag(t,'end')
if kls == None:
if D._globals: do_globals(Token(tok.pos,0,0,[items[0]]))
r = do_set_ctx(items[0],Token(tok.pos,'reg',rf))
else:
rn = do_string(items[0])
#code('SET',kls,rn,rf)
emitset(kls,rn,rf)
free_tmp(rn)
free_tmp(rf)
def prefix_neg(t):
r = expression(50)
if r.type == 'number':
r.val = str(-float(r.val))
return r
t.items = [Token(t.pos,'number','0'),r]
return t
def do_module():
tok = P.token
items = []
while not check(P.token,'eof'):
items.append(block())
if len(items) > 1:
return Token(tok.pos,'statements',';',items)
return items.pop()
def do_module():
tok = P.token
items = []
iblock(items)
advance('eof')
if len(items) > 1:
return Token(tok.pos,'statements',';',items)
return items.pop()
def do_symbol(t, r=None):
sets = ['=']
isets = ['+=', '-=', '*=', '/=']
cmps = ['<', '>', '<=', '>=', '==', '!=']
metas = {
'+': 'ADD',
'*': 'MUL',
'/': 'DIV',
'**': 'POW',
'-': 'SUB',
'and': 'AND',
'or': 'OR',
'%': 'MOD',
'>>': 'RSH',
'<<': 'LSH',
'&': 'AND',
'|': 'OR',
}
if t.val == 'None': return _do_none(r)
if t.val == 'True':
return _do_number('1', r)
if t.val == 'False':
return _do_number('0', r)
items = t.items
if t.val in ['and', 'or']:
ss = int(t.val == 'or')
return ss_infix(ss, metas[t.val], items[0], items[1], r)
if t.val in isets:
return imanage(t, do_symbol)
if t.val == 'is':
return infix('EQ', items[0], items[1], r)
if t.val == 'isnot':
return infix('CMP', items[0], items[1], r)
if t.val == 'not':
return infix('EQ', Token(t.pos, 'number', 0), items[0], r)
if t.val == 'in':
return infix('HAS', items[1], items[0], r)
if t.val == 'notin':
r = infix('HAS', items[1], items[0], r)
zero = _do_number('0')
#code('EQ',r,r,free_tmp(zero))
emiteq(r, r, free_tmp(zero))
return r
if t.val in sets:
return do_set_ctx(items[0], items[1])
elif t.val in cmps:
b, c = items[0], items[1]
v = t.val
if v[0] in ('>', '>='):
b, c, v = c, b, '<' + v[1:]
cd = 'EQ'
if v == '<': cd = LT
if v == '<=': cd = LE
if v == '!=': cd = NE
return infix(cd, b, c, r)
else:
return infix(metas[t.val], items[0], items[1], r)
def do_symbol(t, r=None):
sets = ['=']
isets = ['+=', '-=', '*=', '/=', '|=', '&=', '^=']
cmps = ['<', '>', '<=', '>=', '==', '!=']
metas = {
'+': ADD,
'*': MUL,
'/': DIV,
'**': POW,
'-': SUB,
'and': AND,
'or': OR,
'%': MOD,
'>>': RSH,
'<<': LSH,
'&': AND,
'|': OR,
'^': XOR
}
if t.val == 'None': return _do_none(r)
if t.val == 'True':
return _do_number('1', r)
if t.val == 'False':
return _do_number('0', r)
items = t.items
if t.val in ['and', 'or']:
ss = int(t.val == 'or')
return ss_infix(ss, metas[t.val], items[0], items[1], r)
if t.val in isets:
return imanage(t, do_symbol)
if t.val == 'is':
return infix(EQ, items[0], items[1], r)
if t.val == 'isnot':
return infix(CMP, items[0], items[1], r)
if t.val == 'not':
return infix(EQ, Token(t.pos, 'number', 0), items[0], r)
if t.val == 'in':
return infix(HAS, items[1], items[0], r)
if t.val == 'notin':
r = infix(HAS, items[1], items[0], r)
zero = _do_number('0')
code(EQ, r, r, free_tmp(zero))
return r
if t.val in sets:
return do_set_ctx(items[0], items[1])
elif t.val in cmps:
b, c = items[0], items[1]
v = t.val
if v[0] in ('>', '>='):
b, c, v = c, b, '<' + v[1:]
cd = EQ
if v == '<': cd = LT
if v == '<=': cd = LE
if v == '!=': cd = NE
return infix(cd, b, c, r)
else:
return infix(metas[t.val], items[0], items[1], r)
def call_led(t,left):
r = Token(t.pos,'call','$',[left])
while not check(P.token,')'):
tweak(',',0)
r.items.append(expression(0))
if P.token.val == ',': advance(',')
restore()
advance(")")
return r
def do_from(t):
mod = t.items[0]
mod.type = 'string'
v = do(Token(t.pos,'call',None,[
Token(t.pos,'name','import'),
mod]))
item = t.items[1]
if item.val == '*':
free_tmp(do(Token(t.pos,'call',None,[
Token(t.pos,'name','merge'),
Token(t.pos,'name','__dict__'),
Token(t.pos,'reg',v)]))) #REG
else:
item.type = 'string'
free_tmp(do_set_ctx(
Token(t.pos,'get',None,[ Token(t.pos,'name','__dict__'),item]),
Token(t.pos,'get',None,[ Token(t.pos,'reg',v),item])
)) #REG
def advance(self,val=None):
if not check(self.token,val):
error('expected '+val,self.token)
if self.pos < len(self.tokens):
t = self.tokens[self.pos]
self.pos += 1
else:
t = Token((0,0),'eof','eof')
self.token = do(t)
return t
def encode(fname,s,t):
t = Token((1,1),'module','module',[t])
global D
s = tokenize.clean(s)
D = DState(s,fname)
D.begin(True)
do(t)
D.end()
map_tags()
out = D.out; D = None
return ''.join(out)
def if_nud(t):
items = t.items = []
a = expression(0)
advance(':')
b = block()
items.append(Token(t.pos,'elif','elif',[a,b]))
while check(P.token,'elif'):
tok = P.token
advance('elif')
a = expression(0)
advance(':')
b = block()
items.append(Token(tok.pos,'elif','elif',[a,b]))
if check(P.token,'else'):
tok = P.token
advance('else')
advance(':')
b = block()
items.append(Token(tok.pos,'else','else',[b]))
return t
def try_nud(t):
items = t.items = []
advance(':')
b = block()
items.append(b)
while check(P.token,'except'):
tok = P.token
advance('except')
if not check(P.token,':'): a = expression(0)
else: a = Token(tok.pos,'symbol','None')
advance(':')
b = block()
items.append(Token(tok.pos,'except','except',[a,b]))
if check(P.token,'else'):
tok = P.token
advance('else')
advance(':')
b = block()
items.append(Token(tok.pos,'else','else',[b]))
return t
def try_nud(t):
items = t.items = []
advance(':')
b = block()
items.append(b)
while check(P.token,'except'):
tok = P.token
advance('except')
if not check(P.token,':'): a = expression(0)
else: a = Token(tok.pos,'symbol','None')
advance(':')
b = block()
items.append(Token(tok.pos,'except','except',[a,b]))
#commenting this out, i don't think this next bit is valid syntax??
#if check(P.token,'else'):
#tok = P.token
#advance('else')
#advance(':')
#b = block()
#items.append(Token(tok.pos,'else','else',[b]))
return t
def advance(self,val=None):
if not check(self.token,val):
error('expected '+val,self.token)
if self.pos < len(self.tokens):
t = self.tokens[self.pos]
self.pos += 1
else:
t = Token((0,0),'eof','eof')
self.token = do(t)
self._terminal += 1
if check(self.token,'nl','eof',';','dedent'):
self._terminal = 0
return t
def def_nud(t):
items = t.items = []
items.append(P.token); advance()
advance('(')
r = Token(t.pos,'symbol','():',[])
items.append(r)
while not check(P.token,')'):
tweak(',',0)
r.items.append(expression(0))
if check(P.token,','): advance(',')
restore()
advance(')')
advance(':')
items.append(block())
return t
def do_comp(t,r=None):
name = 'comp:'+get_tag()
r = do_local(Token(t.pos,'name',name))
code('LIST',r,0,0)
key = Token(t.pos,'get',None,[
Token(t.pos,'reg',r),
Token(t.pos,'symbol','None')])
ap = Token(t.pos,'symbol','=',[key,t.items[0]])
do(Token(t.pos,'for',None,[t.items[1],t.items[2],ap]))
return r
def block():
items = []
tok = P.token
if check(P.token,'nl'):
while check(P.token,'nl'): advance()
advance('indent')
iblock(items)
advance('dedent')
else:
items.append(expression(0))
while check(P.token,';'):
advance(';')
items.append(expression(0))
P.terminal()
while check(P.token,'nl'): advance()
if len(items) > 1:
return Token(tok.pos,'statements',';',items)
return items.pop()
def get_led(t,left):
r = Token(t.pos,'get','.',[left])
items = [left]
more = False
while not check(P.token,']'):
more = False
if check(P.token,':'):
items.append(Token(P.token.pos,'symbol','None'))
else:
items.append(expression(0))
if check(P.token,':'):
advance(':')
more = True
if more:
items.append(Token(P.token.pos,'symbol','None'))
if len(items) > 2:
items = [left,Token(t.pos,'slice',':',items[1:])]
r.items = items
advance("]")
return r
def main(serialized_tokens):
tokens = [Token(*token) for token in json.loads(serialized_tokens)]
tree = Parser(tokens).parse()
indent = 4 if '-h' in sys.argv else None
print(json.dumps(list(tree), indent=indent))
def do_class(t):
tok = t
items = t.items
parent = None
if items[0].type == 'name':
name = items[0].val
else:
name = items[0].items[0].val
parent = items[0].items[1].val
kls = do(Token(t.pos, 'dict', 0, []))
ts = _do_string(name)
code(GSET, ts, kls)
free_tmp(ts) #REG
init, _new = False, []
if parent:
_new.append(
Token(t.pos, 'call', None, [
Token(t.pos, 'get', None, [
Token(t.pos, 'name', parent),
Token(t.pos, 'string', '__new__'),
]),
Token(t.pos, 'name', 'self'),
]))
for fc in items[1].items:
if fc.type != 'def': continue
fn = fc.items[0].val
if fn == '__init__': init = True
do_def(fc, kls)
_new.append(
Token(fc.pos, 'symbol', '=', [
Token(fc.pos, 'get', None, [
Token(fc.pos, 'name', 'self'),
Token(fc.pos, 'string', fn)
]),
Token(fc.pos, 'call', None, [
Token(fc.pos, 'name', 'bind'),
Token(fc.pos, 'get', None, [
Token(fc.pos, 'name', name),
Token(fc.pos, 'string', fn)
]),
Token(fc.pos, 'name', 'self')
])
]))
do_def(
Token(t.pos, 'def', None, [
Token(t.pos, 'name', '__new__'),
Token(t.pos, 'list', None, [Token(t.pos, 'name', 'self')]),
Token(t.pos, 'statements', None, _new)
]), kls)
t = get_tag()
rf = fnc(t, 'end')
D.begin()
params = do_local(Token(tok.pos, 'name', '__params'))
slf = do_local(Token(tok.pos, 'name', 'self'))
code(DICT, slf, 0, 0)
free_tmp(
do(
Token(tok.pos, 'call', None, [
Token(tok.pos, 'get', None, [
Token(tok.pos, 'name', name),
Token(tok.pos, 'string', '__new__')
]),
Token(tok.pos, 'name', 'self')
]))) #REG
if init:
tmp = get_tmp()
t3 = _do_string('__init__')
code(GET, tmp, slf, t3)
t4 = get_tmp()
code(CALL, t4, tmp, params)
free_tmp(tmp) #REG
free_tmp(t3) #REG
free_tmp(t4) #REG
code(RETURN, slf)
D.end()
tag(t, 'end')
ts = _do_string('__call__')
code(SET, kls, ts, rf)
free_tmp(kls) #REG
free_tmp(ts) #REG
def ilst(typ,t):
return Token(t.pos,typ,typ,lst(t))
def do_class(t):
tok = t
items = t.items
parent = None
if items[0].type == 'name':
name = items[0].val
else:
name = items[0].items[0].val
parent = items[0].items[1].val
kls = do(Token(t.pos,'dict',0,[]))
#ts = _do_string(name)
#code('GSET',ts,kls)
#free_tmp(ts) #REG
#code('GSET','gVarName='+name,'reg='+str(kls))
#code('GSET','gVarName',name,'reg',kls)
emitgset(name,kls)
init,_new = False,[]
if parent:
_new.append(Token(t.pos,'call',None,[
Token(t.pos,'get',None,[
Token(t.pos,'name',parent),
Token(t.pos,'string','__new__'),
]),
Token(t.pos,'name','self'),
]))
for fc in items[1].items:
if fc.type != 'def': continue
fn = fc.items[0].val
if fn == '__init__': init = True
do_def(fc,kls)
_new.append(Token(fc.pos,'symbol','=',[
Token(fc.pos,'get',None,[
Token(fc.pos,'name','self'),
Token(fc.pos,'string',fn)]),
Token(fc.pos,'call',None,[
Token(fc.pos,'name','bind'),
Token(fc.pos,'get',None,[
Token(fc.pos,'name',name),
Token(fc.pos,'string',fn)]),
Token(fc.pos,'name','self')])
]))
do_def(Token(t.pos,'def',None,[
Token(t.pos,'name','__new__'),
Token(t.pos,'list',None,[Token(t.pos,'name','self')]),
Token(t.pos,'statements',None,_new)]),kls)
t = get_tag()
rf = fnc(t,'end')
D.begin()
params = do_local(Token(tok.pos,'name','__params'))
slf = do_local(Token(tok.pos,'name','self'))
comment('self');
#code('DICT',slf,0,0)
xml('DICT',"reg",slf)
free_tmp(do(Token(tok.pos,'call',None,[
Token(tok.pos,'get',None,[
Token(tok.pos,'name',name),
Token(tok.pos,'string','__new__')]),
Token(tok.pos,'name','self')]))) #REG
if init:
tmp = get_tmp()
#t3 = _do_string('__init__')
#code('GET',tmp,slf,t3)
#code('GET','toreg='+str(tmp),'fromClass='+str(slf),"varName='__init__'")
#xml('GET','toreg',tmp,'fromClass',slf,"varName",'__init__')
emitget(tmp,slf,'__init__')
t4 = get_tmp()
#code('CALL',t4,'func='+str(tmp),'params='+str(params))
xml('CALL','ret',t4,'func',tmp,'params',params)
free_tmp(tmp) #REG
#free_tmp(t3) #REG
free_tmp(t4) #REG
comment('return self');
code('<RETURN reg="'+str(slf)+'" />',)
D.end()
tag(t,'end')
#ts = _do_string('__call__')
#code('SET',kls,ts,rf)
#code('SET',"class="+str(kls),"fieldName='__call__'",rf)
#xml('SET',"class",kls,"fieldName", '__call__',"rf",rf)
emitset(kls,'__call__',rf)
free_tmp(kls) #REG
def imanage(orig,fnc):
items = orig.items
orig.val = orig.val[:-1]
t = Token(orig.pos,'symbol','=',[items[0],orig])
return fnc(t)
