def do_call(t,r=None): r = get_tmp(r) items = t.items fnc = do(items[0]) a,b,c,d = p_filter(t.items[1:]) e = None if len(b) != 0 or d != None: e = do(Token(t.pos,'dict',None,[])); un_tmp(e); for p in b: p.items[0].type = 'string' t1,t2 = do(p.items[0]),do(p.items[1]) #code('SET',e,t1,t2) emitset(e,t1,t2) free_tmp(t1); free_tmp(t2) #REG if d: free_tmp(do(Token(t.pos,'call',None,[Token(t.pos,'name','merge'),Token(t.pos,'reg',e),d.items[0]]))) #REG manage_seq('PARAMS',r,a) if c != None: t1,t2 = _do_string('*'),do(c.items[0]) #code('SET',r,t1,t2) emitset(r,t1,t2) free_tmp(t1); free_tmp(t2) #REG if e != None: t1 = _do_none() #code('SET',r,t1,e) emitset(r,t1,e) free_tmp(t1) #REG xml('CALL','ret',r,'func',fnc,'params',r) free_tmp(fnc) #REG return r
def do_class(t): tok = t items = t.items parent = None if items[0].type == 'name': name = items[0].val parent = Token(tok.pos, 'name', 'object') else: name = items[0].items[0].val parent = items[0].items[1] kls = do(Token(t.pos, 'dict', 0, [])) un_tmp(kls) ts = _do_string(name) code(GSET, ts, kls) free_tmp(ts) #REG free_tmp( do( Token(tok.pos, 'call', None, [ Token(tok.pos, 'name', 'setmeta'), Token(tok.pos, 'reg', kls), parent ]))) for fc in items[1].items: if fc.type != 'def': continue do_def(fc, kls) free_reg(kls) #REG
def do_call(t, r=None): r = get_tmp(r) items = t.items fnc = do(items[0]) a, b, c, d = p_filter(t.items[1:]) e = None if len(b) != 0 or d != None: e = do(Token(t.pos, 'dict', None, [])) un_tmp(e) for p in b: p.items[0].type = 'string' t1, t2 = do(p.items[0]), do(p.items[1]) code(SET, e, t1, t2) free_tmp(t1) free_tmp(t2) #REG if d: free_tmp( do( Token(t.pos, 'call', None, [ Token(t.pos, 'name', 'merge'), Token(t.pos, 'reg', e), d.items[0] ]))) #REG manage_seq(PARAMS, r, a) if c != None: t1, t2 = _do_string('*'), do(c.items[0]) code(SET, r, t1, t2) free_tmp(t1) free_tmp(t2) #REG if e != None: t1 = _do_none() code(SET, r, t1, e) free_tmp(t1) #REG code(CALL, r, fnc, r) free_tmp(fnc) #REG return r
def do_import(t): for mod in t.items: mod.type = 'string' v = do_call( Token(t.pos, 'call', None, [Token(t.pos, 'name', 'import'), mod])) mod.type = 'name' do_set_ctx(mod, Token(t.pos, 'reg', v))
def do_class(t): tok = t items = t.items parent = None if items[0].type == 'name': name = items[0].val parent = Token(tok.pos, 'name', 'object') else: name = items[0].items[0].val parent = items[0].items[1] kls = do(Token(t.pos, 'dict', 0, [])) un_tmp(kls) ts = _do_string(name) code(GSET, ts, kls) free_tmp(ts) #REG free_tmp( do( Token(tok.pos, 'call', None, [ Token(tok.pos, 'name', 'setmeta'), Token(tok.pos, 'reg', kls), parent ]))) for member in items[1].items: if member.type == 'def': do_def(member, kls) elif member.type == 'symbol' and member.val == '=': do_classvar(member, kls) else: continue free_reg(kls) #REG
def do_set_ctx(k, v): if k.type == 'name': if (D._globals and k.val not in D.vars) or (k.val in D.globals): #c = do_string(k) b = do(v) #code('<GSET gVarName="'+k.val+'" reg="'+str(b)+'" />','','','') emitgset(k.val, b) #free_tmp(c) free_tmp(b) return a = do_local(k) b = do(v) #code('<MOVE to="'+str(a)+'" from="'+str(b)+'" />', '', '', '') emitmove(a, b) free_tmp(b) return a elif k.type in ('tuple', 'list'): if v.type in ('tuple', 'list'): n, tmps = 0, [] for kk in k.items: vv = v.items[n] tmp = get_tmp() tmps.append(tmp) r = do(vv) #code('< MOVE to="'+str(tmp)+'" from="'+str(r)+'" />', '', '', '') emitmove(tmp, r) free_tmp(r) #REG n += 1 n = 0 for kk in k.items: vv = v.items[n] tmp = tmps[n] free_tmp(do_set_ctx(kk, Token(vv.pos, 'reg', tmp))) #REG n += 1 return r = do(v) un_tmp(r) n, tmp = 0, Token(v.pos, 'reg', r) for tt in k.items: free_tmp( do_set_ctx( tt, Token(tmp.pos, 'get', None, [tmp, Token(tmp.pos, 'number', str(n))]))) #REG n += 1 free_reg(r) return r = do(k.items[0]) rr = do(v) tmp = do(k.items[1]) #code('SET',r,tmp,rr) emitset(r, tmp, rr) free_tmp(r) #REG free_tmp(tmp) #REG return rr
def do_set_ctx(k, v): if k.type == 'name': if (D._globals and k.val not in D.vars) or (k.val in D.globals): c = do_string(k) b = do(v) code(GSET, c, b) free_tmp(c) free_tmp(b) return a = do_local(k) b = do(v) code(MOVE, a, b) free_tmp(b) return a elif k.type in ('tuple', 'list'): if v.type in ('tuple', 'list'): n, tmps = 0, [] for kk in k.items: vv = v.items[n] tmp = get_tmp() tmps.append(tmp) r = do(vv) code(MOVE, tmp, r) free_tmp(r) #REG n += 1 n = 0 for kk in k.items: vv = v.items[n] tmp = tmps[n] free_tmp(do_set_ctx(kk, Token(vv.pos, 'reg', tmp))) #REG n += 1 return r = do(v) un_tmp(r) n, tmp = 0, Token(v.pos, 'reg', r) for tt in k.items: free_tmp( do_set_ctx( tt, Token(tmp.pos, 'get', None, [tmp, Token(tmp.pos, 'number', str(n))]))) #REG n += 1 free_reg(r) return r = do(k.items[0]) rr = do(v) tmp = do(k.items[1]) code(SET, r, tmp, rr) free_tmp(r) #REG free_tmp(tmp) #REG return rr
def do_def(tok,kls=None): items = tok.items t = get_tag() rf = fnc(t,'end') D.begin() setpos(tok.pos) r = do_local(Token(tok.pos,'name','__params')) do_info(items[0].val) a,b,c,d = p_filter(items[1].items) for p in a: v = do_local(p) tmp = _do_none() #code('GET','toreg='+str(v),r,tmp) emitget(v,r,tmp) free_tmp(tmp) #REG for p in b: v = do_local(p.items[0]) do(p.items[1],v) tmp = _do_none() code('IGET',v,r,tmp) free_tmp(tmp) #REG if c != None: v = do_local(c.items[0]) tmp = _do_string('*') #code('GET','toreg='+str(v),r,tmp) emitget(v,r,tmp) free_tmp(tmp) #REG if d != None: e = do_local(d.items[0]) #code('DICT',e,0,0) xml('DICT',"reg",e) tmp = _do_none() code('IGET',e,r,tmp) free_tmp(tmp) #REG free_tmp(do(items[2])) #REG D.end() tag(t,'end') if kls == None: if D._globals: do_globals(Token(tok.pos,0,0,[items[0]])) r = do_set_ctx(items[0],Token(tok.pos,'reg',rf)) else: rn = do_string(items[0]) #code('SET',kls,rn,rf) emitset(kls,rn,rf) free_tmp(rn) free_tmp(rf)
def prefix_neg(t): r = expression(50) if r.type == 'number': r.val = str(-float(r.val)) return r t.items = [Token(t.pos,'number','0'),r] return t
def do_module(): tok = P.token items = [] while not check(P.token,'eof'): items.append(block()) if len(items) > 1: return Token(tok.pos,'statements',';',items) return items.pop()
def do_module(): tok = P.token items = [] iblock(items) advance('eof') if len(items) > 1: return Token(tok.pos,'statements',';',items) return items.pop()
def do_symbol(t, r=None): sets = ['='] isets = ['+=', '-=', '*=', '/='] cmps = ['<', '>', '<=', '>=', '==', '!='] metas = { '+': 'ADD', '*': 'MUL', '/': 'DIV', '**': 'POW', '-': 'SUB', 'and': 'AND', 'or': 'OR', '%': 'MOD', '>>': 'RSH', '<<': 'LSH', '&': 'AND', '|': 'OR', } if t.val == 'None': return _do_none(r) if t.val == 'True': return _do_number('1', r) if t.val == 'False': return _do_number('0', r) items = t.items if t.val in ['and', 'or']: ss = int(t.val == 'or') return ss_infix(ss, metas[t.val], items[0], items[1], r) if t.val in isets: return imanage(t, do_symbol) if t.val == 'is': return infix('EQ', items[0], items[1], r) if t.val == 'isnot': return infix('CMP', items[0], items[1], r) if t.val == 'not': return infix('EQ', Token(t.pos, 'number', 0), items[0], r) if t.val == 'in': return infix('HAS', items[1], items[0], r) if t.val == 'notin': r = infix('HAS', items[1], items[0], r) zero = _do_number('0') #code('EQ',r,r,free_tmp(zero)) emiteq(r, r, free_tmp(zero)) return r if t.val in sets: return do_set_ctx(items[0], items[1]) elif t.val in cmps: b, c = items[0], items[1] v = t.val if v[0] in ('>', '>='): b, c, v = c, b, '<' + v[1:] cd = 'EQ' if v == '<': cd = LT if v == '<=': cd = LE if v == '!=': cd = NE return infix(cd, b, c, r) else: return infix(metas[t.val], items[0], items[1], r)
def do_symbol(t, r=None): sets = ['='] isets = ['+=', '-=', '*=', '/=', '|=', '&=', '^='] cmps = ['<', '>', '<=', '>=', '==', '!='] metas = { '+': ADD, '*': MUL, '/': DIV, '**': POW, '-': SUB, 'and': AND, 'or': OR, '%': MOD, '>>': RSH, '<<': LSH, '&': AND, '|': OR, '^': XOR } if t.val == 'None': return _do_none(r) if t.val == 'True': return _do_number('1', r) if t.val == 'False': return _do_number('0', r) items = t.items if t.val in ['and', 'or']: ss = int(t.val == 'or') return ss_infix(ss, metas[t.val], items[0], items[1], r) if t.val in isets: return imanage(t, do_symbol) if t.val == 'is': return infix(EQ, items[0], items[1], r) if t.val == 'isnot': return infix(CMP, items[0], items[1], r) if t.val == 'not': return infix(EQ, Token(t.pos, 'number', 0), items[0], r) if t.val == 'in': return infix(HAS, items[1], items[0], r) if t.val == 'notin': r = infix(HAS, items[1], items[0], r) zero = _do_number('0') code(EQ, r, r, free_tmp(zero)) return r if t.val in sets: return do_set_ctx(items[0], items[1]) elif t.val in cmps: b, c = items[0], items[1] v = t.val if v[0] in ('>', '>='): b, c, v = c, b, '<' + v[1:] cd = EQ if v == '<': cd = LT if v == '<=': cd = LE if v == '!=': cd = NE return infix(cd, b, c, r) else: return infix(metas[t.val], items[0], items[1], r)
def call_led(t,left): r = Token(t.pos,'call','$',[left]) while not check(P.token,')'): tweak(',',0) r.items.append(expression(0)) if P.token.val == ',': advance(',') restore() advance(")") return r
def do_from(t): mod = t.items[0] mod.type = 'string' v = do(Token(t.pos,'call',None,[ Token(t.pos,'name','import'), mod])) item = t.items[1] if item.val == '*': free_tmp(do(Token(t.pos,'call',None,[ Token(t.pos,'name','merge'), Token(t.pos,'name','__dict__'), Token(t.pos,'reg',v)]))) #REG else: item.type = 'string' free_tmp(do_set_ctx( Token(t.pos,'get',None,[ Token(t.pos,'name','__dict__'),item]), Token(t.pos,'get',None,[ Token(t.pos,'reg',v),item]) )) #REG
def advance(self,val=None): if not check(self.token,val): error('expected '+val,self.token) if self.pos < len(self.tokens): t = self.tokens[self.pos] self.pos += 1 else: t = Token((0,0),'eof','eof') self.token = do(t) return t
def encode(fname,s,t): t = Token((1,1),'module','module',[t]) global D s = tokenize.clean(s) D = DState(s,fname) D.begin(True) do(t) D.end() map_tags() out = D.out; D = None return ''.join(out)
def if_nud(t): items = t.items = [] a = expression(0) advance(':') b = block() items.append(Token(t.pos,'elif','elif',[a,b])) while check(P.token,'elif'): tok = P.token advance('elif') a = expression(0) advance(':') b = block() items.append(Token(tok.pos,'elif','elif',[a,b])) if check(P.token,'else'): tok = P.token advance('else') advance(':') b = block() items.append(Token(tok.pos,'else','else',[b])) return t
def try_nud(t): items = t.items = [] advance(':') b = block() items.append(b) while check(P.token,'except'): tok = P.token advance('except') if not check(P.token,':'): a = expression(0) else: a = Token(tok.pos,'symbol','None') advance(':') b = block() items.append(Token(tok.pos,'except','except',[a,b])) if check(P.token,'else'): tok = P.token advance('else') advance(':') b = block() items.append(Token(tok.pos,'else','else',[b])) return t
def try_nud(t): items = t.items = [] advance(':') b = block() items.append(b) while check(P.token,'except'): tok = P.token advance('except') if not check(P.token,':'): a = expression(0) else: a = Token(tok.pos,'symbol','None') advance(':') b = block() items.append(Token(tok.pos,'except','except',[a,b])) #commenting this out, i don't think this next bit is valid syntax?? #if check(P.token,'else'): #tok = P.token #advance('else') #advance(':') #b = block() #items.append(Token(tok.pos,'else','else',[b])) return t
def advance(self,val=None): if not check(self.token,val): error('expected '+val,self.token) if self.pos < len(self.tokens): t = self.tokens[self.pos] self.pos += 1 else: t = Token((0,0),'eof','eof') self.token = do(t) self._terminal += 1 if check(self.token,'nl','eof',';','dedent'): self._terminal = 0 return t
def def_nud(t): items = t.items = [] items.append(P.token); advance() advance('(') r = Token(t.pos,'symbol','():',[]) items.append(r) while not check(P.token,')'): tweak(',',0) r.items.append(expression(0)) if check(P.token,','): advance(',') restore() advance(')') advance(':') items.append(block()) return t
def do_comp(t,r=None): name = 'comp:'+get_tag() r = do_local(Token(t.pos,'name',name)) code('LIST',r,0,0) key = Token(t.pos,'get',None,[ Token(t.pos,'reg',r), Token(t.pos,'symbol','None')]) ap = Token(t.pos,'symbol','=',[key,t.items[0]]) do(Token(t.pos,'for',None,[t.items[1],t.items[2],ap])) return r
def block(): items = [] tok = P.token if check(P.token,'nl'): while check(P.token,'nl'): advance() advance('indent') iblock(items) advance('dedent') else: items.append(expression(0)) while check(P.token,';'): advance(';') items.append(expression(0)) P.terminal() while check(P.token,'nl'): advance() if len(items) > 1: return Token(tok.pos,'statements',';',items) return items.pop()
def get_led(t,left): r = Token(t.pos,'get','.',[left]) items = [left] more = False while not check(P.token,']'): more = False if check(P.token,':'): items.append(Token(P.token.pos,'symbol','None')) else: items.append(expression(0)) if check(P.token,':'): advance(':') more = True if more: items.append(Token(P.token.pos,'symbol','None')) if len(items) > 2: items = [left,Token(t.pos,'slice',':',items[1:])] r.items = items advance("]") return r
def main(serialized_tokens): tokens = [Token(*token) for token in json.loads(serialized_tokens)] tree = Parser(tokens).parse() indent = 4 if '-h' in sys.argv else None print(json.dumps(list(tree), indent=indent))
def do_class(t): tok = t items = t.items parent = None if items[0].type == 'name': name = items[0].val else: name = items[0].items[0].val parent = items[0].items[1].val kls = do(Token(t.pos, 'dict', 0, [])) ts = _do_string(name) code(GSET, ts, kls) free_tmp(ts) #REG init, _new = False, [] if parent: _new.append( Token(t.pos, 'call', None, [ Token(t.pos, 'get', None, [ Token(t.pos, 'name', parent), Token(t.pos, 'string', '__new__'), ]), Token(t.pos, 'name', 'self'), ])) for fc in items[1].items: if fc.type != 'def': continue fn = fc.items[0].val if fn == '__init__': init = True do_def(fc, kls) _new.append( Token(fc.pos, 'symbol', '=', [ Token(fc.pos, 'get', None, [ Token(fc.pos, 'name', 'self'), Token(fc.pos, 'string', fn) ]), Token(fc.pos, 'call', None, [ Token(fc.pos, 'name', 'bind'), Token(fc.pos, 'get', None, [ Token(fc.pos, 'name', name), Token(fc.pos, 'string', fn) ]), Token(fc.pos, 'name', 'self') ]) ])) do_def( Token(t.pos, 'def', None, [ Token(t.pos, 'name', '__new__'), Token(t.pos, 'list', None, [Token(t.pos, 'name', 'self')]), Token(t.pos, 'statements', None, _new) ]), kls) t = get_tag() rf = fnc(t, 'end') D.begin() params = do_local(Token(tok.pos, 'name', '__params')) slf = do_local(Token(tok.pos, 'name', 'self')) code(DICT, slf, 0, 0) free_tmp( do( Token(tok.pos, 'call', None, [ Token(tok.pos, 'get', None, [ Token(tok.pos, 'name', name), Token(tok.pos, 'string', '__new__') ]), Token(tok.pos, 'name', 'self') ]))) #REG if init: tmp = get_tmp() t3 = _do_string('__init__') code(GET, tmp, slf, t3) t4 = get_tmp() code(CALL, t4, tmp, params) free_tmp(tmp) #REG free_tmp(t3) #REG free_tmp(t4) #REG code(RETURN, slf) D.end() tag(t, 'end') ts = _do_string('__call__') code(SET, kls, ts, rf) free_tmp(kls) #REG free_tmp(ts) #REG
def ilst(typ,t): return Token(t.pos,typ,typ,lst(t))
def do_class(t): tok = t items = t.items parent = None if items[0].type == 'name': name = items[0].val else: name = items[0].items[0].val parent = items[0].items[1].val kls = do(Token(t.pos,'dict',0,[])) #ts = _do_string(name) #code('GSET',ts,kls) #free_tmp(ts) #REG #code('GSET','gVarName='+name,'reg='+str(kls)) #code('GSET','gVarName',name,'reg',kls) emitgset(name,kls) init,_new = False,[] if parent: _new.append(Token(t.pos,'call',None,[ Token(t.pos,'get',None,[ Token(t.pos,'name',parent), Token(t.pos,'string','__new__'), ]), Token(t.pos,'name','self'), ])) for fc in items[1].items: if fc.type != 'def': continue fn = fc.items[0].val if fn == '__init__': init = True do_def(fc,kls) _new.append(Token(fc.pos,'symbol','=',[ Token(fc.pos,'get',None,[ Token(fc.pos,'name','self'), Token(fc.pos,'string',fn)]), Token(fc.pos,'call',None,[ Token(fc.pos,'name','bind'), Token(fc.pos,'get',None,[ Token(fc.pos,'name',name), Token(fc.pos,'string',fn)]), Token(fc.pos,'name','self')]) ])) do_def(Token(t.pos,'def',None,[ Token(t.pos,'name','__new__'), Token(t.pos,'list',None,[Token(t.pos,'name','self')]), Token(t.pos,'statements',None,_new)]),kls) t = get_tag() rf = fnc(t,'end') D.begin() params = do_local(Token(tok.pos,'name','__params')) slf = do_local(Token(tok.pos,'name','self')) comment('self'); #code('DICT',slf,0,0) xml('DICT',"reg",slf) free_tmp(do(Token(tok.pos,'call',None,[ Token(tok.pos,'get',None,[ Token(tok.pos,'name',name), Token(tok.pos,'string','__new__')]), Token(tok.pos,'name','self')]))) #REG if init: tmp = get_tmp() #t3 = _do_string('__init__') #code('GET',tmp,slf,t3) #code('GET','toreg='+str(tmp),'fromClass='+str(slf),"varName='__init__'") #xml('GET','toreg',tmp,'fromClass',slf,"varName",'__init__') emitget(tmp,slf,'__init__') t4 = get_tmp() #code('CALL',t4,'func='+str(tmp),'params='+str(params)) xml('CALL','ret',t4,'func',tmp,'params',params) free_tmp(tmp) #REG #free_tmp(t3) #REG free_tmp(t4) #REG comment('return self'); code('<RETURN reg="'+str(slf)+'" />',) D.end() tag(t,'end') #ts = _do_string('__call__') #code('SET',kls,ts,rf) #code('SET',"class="+str(kls),"fieldName='__call__'",rf) #xml('SET',"class",kls,"fieldName", '__call__',"rf",rf) emitset(kls,'__call__',rf) free_tmp(kls) #REG
def imanage(orig,fnc): items = orig.items orig.val = orig.val[:-1] t = Token(orig.pos,'symbol','=',[items[0],orig]) return fnc(t)