def deffnc(tok, items, end): p, n, l, d = p_filter(items) args = Token(tok.pos, 'list', 0, [i.as_string() for i in p] + [i.items[0].as_string() for i in n]) defaults = Token(tok.pos, 'list', 0, [i.items[1] for i in n]) # RA = tp_def( RA+1, ....) rf, rargs, rdefaults, rvarargs, rvarkw = get_tmps(5) un_tmp(rf) do_list(args, rargs) do_list(defaults, rdefaults) if l != None: do_string(l.items[0].as_string(), rvarargs) else: _do_none(rvarargs) if d != None: do_string(d.items[0].as_string(), rvarkw) else: _do_none(rvarkw) t = [get_tag(), end] n = D.snum + ':' + ':'.join([str(v) for v in t]) insert(('fnc', rf, n)) free_tmps([rargs, rdefaults, rvarargs, rvarkw]) return rf, t[0]
def do_set_ctx(k, v): if k.type == 'name': if (D._globals and k.val not in D.vars) or (k.val in D.globals): c = do_string(k.as_string()) b = do(v) code(GSET, c, b) free_tmp(c) free_tmp(b) return a = do_local(k) b = do(v) code(MOVE, a, b) free_tmp(b) return a elif k.type in ('tuple', 'list'): if v.type in ('tuple', 'list'): n, tmps = 0, [] for kk in k.items: vv = v.items[n] tmp = get_tmp() tmps.append(tmp) r = do(vv) code(MOVE, tmp, r) free_tmp(r) #REG n += 1 n = 0 for kk in k.items: vv = v.items[n] tmp = tmps[n] free_tmp(do_set_ctx(kk, Token(vv.pos, 'reg', tmp))) #REG n += 1 return r = do(v) un_tmp(r) n, tmp = 0, Token(v.pos, 'reg', r) for tt in k.items: free_tmp( do_set_ctx( tt, Token(tmp.pos, 'get', None, [tmp, Token(tmp.pos, 'integer', str(n))]))) #REG n += 1 free_reg(r) return r = do(k.items[0]) rr = do(v) tmp = do(k.items[1]) code(SET, r, tmp, rr) free_tmp(r) #REG free_tmp(tmp) #REG return rr
def do_from(t): mod = t.items[0] names = t.items[1] mod = mod.as_string() if names.type == 'largs': # it really shouldn't be largs -- need to fix the parser. if names.val == '*': names = names.as_string() elif names.type == 'name': names = Token(names.pos, 'tuple', None, [names.as_string()]) elif names.type == 'tuple': names = Token(names.pos, 'tuple', None, [i.as_string() for i in names.items]) else: tokenize.u_error('SyntaxError', D.code, t.pos) v = do( Token(t.pos, 'call', None, [ Token(t.pos, 'name', '__import__'), mod, names, ])) g = do(Token(t.pos, 'name', '__dict__')) code(UPDATE, g, v) free_tmp(g) free_tmp(v)
def do_import(t): if len(t.items) == 1: mod = t.items[0] name = mod else: mod, name = t.items v = do_call( Token(t.pos, 'call', None, [ Token(t.pos, 'name', '__import__'), mod.as_string(), Token(t.pos, 'symbol', 'None') ])) assert name.type == 'name' do_set_ctx(name, Token(t.pos, 'reg', v))
def block(): items = [] tok = P.token if check(P.token, 'nl'): while check(P.token, 'nl'): advance() advance('indent') iblock(items) advance('dedent') else: items.append(expression(0)) while check(P.token, ';'): advance(';') items.append(expression(0)) P.terminal() while check(P.token, 'nl'): advance() # FIXME: this is not normal. Consider always return # statements and squeeze on the caller side with a helper. # also see class_nud where we use ilst to revert the squeeze. if len(items) > 1: return Token(tok.pos, 'statements', ';', items) return items.pop()
def do_module(): tok = P.token items = [] iblock(items) advance('eof') if len(items) > 1: return Token(tok.pos, 'statements', ';', items) return items.pop()
def call_led(t, left): r = Token(t.pos, 'call', '$', [left]) while not check(P.token, ')'): tweak(',', 0) r.items.append(expression(0)) if P.token.val == ',': advance(',') restore() advance(")") return r
def do_class(tok): items = tok.items parent = None if items[0].type == 'name': name = items[0] parent = Token(tok.pos, 'name', 'object') else: name = items[0].items[0] parent = items[0].items[1] kls = get_tmp() code(CLASS, kls) un_tmp(kls) ts = do_string(name.as_string()) code(GSET, ts, kls) free_tmp(ts) #REG free_tmp( do( Token(tok.pos, 'call', None, [ Token(tok.pos, 'name', 'setmeta'), Token(tok.pos, 'reg', kls), parent ]))) free_reg(kls) #REG # define a function for the class body, and call it. rf, t = deffnc(tok, [], 'end') D.begin() setpos(tok.pos) # Run the class body. free_tmp(do(items[1])) #REG # merge the local variables to the class: # we must refetch the kls object, because after begin() the kls reg is # invalidated. ts = do_string(name.as_string()) kls = get_tmp() code(GGET, kls, ts) un_tmp(kls) free_tmp(ts) #REG for val in D.vars: val_name = Token(tok.pos, 'name', val) ts = do_string(val_name.as_string()) code(SET, kls, ts, get_reg(val)) free_tmp(ts) #REG free_reg(kls) D.end() tag(t, 'end') tmp, lparams, dparams = get_tmps(3) _do_none(lparams) _do_none(dparams) code(CALL, tmp, rf, lparams) free_tmp(rf) free_tmp(tmp) free_tmp(lparams) free_tmp(dparams)
def prefix_neg(t): r = expression(50) if r.type == 'float': r.val = str(-float(r.val)) return r if r.type == 'integer': r.val = str(-int(r.val)) return r t.items = [Token(t.pos, 'integer', '0'), r] return t
def if_nud(t): items = t.items = [] a = expression(0) advance(':') b = block() items.append(Token(t.pos, 'elif', 'elif', [a, b])) while check(P.token, 'elif'): tok = P.token advance('elif') a = expression(0) advance(':') b = block() items.append(Token(tok.pos, 'elif', 'elif', [a, b])) if check(P.token, 'else'): tok = P.token advance('else') advance(':') b = block() items.append(Token(tok.pos, 'else', 'else', [b])) return t
def try_nud(t): items = t.items = [] advance(':') b = block() items.append(b) while check(P.token, 'except'): tok = P.token advance('except') if not check(P.token, ':'): a = expression(0) else: a = Token(tok.pos, 'symbol', 'None') advance(':') b = block() items.append(Token(tok.pos, 'except', 'except', [a, b])) #commenting this out, i don't think this next bit is valid syntax?? #if check(P.token,'else'): #tok = P.token #advance('else') #advance(':') #b = block() #items.append(Token(tok.pos,'else','else',[b])) return t
def advance(self, val=None): if not check(self.token, val): error('expected ' + val, self.token) if self.pos < len(self.tokens): t = self.tokens[self.pos] self.pos += 1 else: t = Token((0, 0), 'eof', 'eof') self.token = do(t) self._terminal += 1 if check(self.token, 'nl', 'eof', ';', 'dedent'): self._terminal = 0 return t
def encode(fname, s, t): t = Token((1, 1), 'module', 'module', [t]) global D s = tokenize.clean(s) D = DState(s, fname) D.begin(True) do(t) D.end() map_tags() out = D.out D = None # Use a function instead of ''.join() so that bytes and # strings during bootstrap return join(out)
def do_def(tok): items = tok.items rf, t = deffnc(tok, items[1].items, 'end') D.begin() setpos(tok.pos) p, n, l, d = p_filter(items[1].items) # declare regs from 0 for i in p: # positional args do_local(i) for i in n: # positional args with defaults do_local(i.items[0]) if l != None: # varargs do_local(l.items[0]) else: do_local(Token(tok.pos, 'name', '__varargs__')) if d != None: # varkw do_local(d.items[0]) else: do_local(Token(tok.pos, 'name', '__kwargs__')) do_info(items[0].val) free_tmp(do(items[2])) #REG D.end() tag(t, 'end') if D._globals: do_globals(Token(tok.pos, 0, 0, [items[0]])) free_tmp(do_set_ctx(items[0], Token(tok.pos, 'reg', rf))) free_tmp(rf)
def do_comp(t, r=None): name = 'comp:' + get_tag() r = do_local(Token(t.pos, 'name', name)) code(LIST, r, 0, 0) key = Token(t.pos, 'get', None, [Token(t.pos, 'reg', r), Token(t.pos, 'symbol', 'None')]) ap = Token(t.pos, 'symbol', '=', [key, t.items[0]]) do(Token(t.pos, 'for', None, [t.items[1], t.items[2], ap])) return r
def def_nud(t): items = t.items = [] items.append(P.token) advance() advance('(') r = Token(t.pos, 'symbol', '():', []) items.append(r) while not check(P.token, ')'): tweak(',', 0) r.items.append(expression(0)) if check(P.token, ','): advance(',') restore() advance(')') advance(':') items.append(block()) return t
def import_nud(t): items = t.items = [] s = '' while True: if check(P.token, 'mget'): s = s + '.' advance() elif check(P.token, 'name'): s = s + P.token.val advance() else: break expr = Token(t.pos, 'name', s) items.append(expr) if check(P.token, 'as'): advance('as') items.append(expression(0)) return t
def from_nud(t): items = t.items = [] # relative import s = '' while True: if check(P.token, 'mget'): s = s + '.' advance() elif check(P.token, 'name'): s = s + P.token.val advance() else: break expr = Token(t.pos, 'name', s) items.append(expr) advance('import') items.append(expression(0)) return t
def do_for(tok): items = tok.items reg = get_tmp() itr = do(items[1]) i = _do_integer(0) t = stack_tag() tag(t, 'loop') tag(t, 'continue') code(ITER, reg, itr, i) jump(t, 'end') free_tmp(do_set_ctx(items[0], Token(tok.pos, 'reg', reg))) free_tmp(do(items[2])) #REG jump(t, 'loop') tag(t, 'break') tag(t, 'end') pop_tag() free_tmp(itr) #REG free_tmp(i)
def do_call(t, r=None): items = t.items fnc = do(items[0]) p, n, l, d = p_filter(t.items[1:]) e = None if len(n) != 0 or d != None: e = do(Token(t.pos, 'dict', None, [])) un_tmp(e) for i in n: i1, i2 = i.items t1, t2 = do(i1.as_string()), do(i2) code(SET, e, t1, t2) free_tmp(t1) free_tmp(t2) #REG if d: f = do(d.items[0]) un_tmp(f) code(UPDATE, e, f) lparams, dparams = get_tmps(2) # FIXME: for CPython compat we shall change the convention # to always set the two last PARAMS to * and **, # then on the caller side unpack the args. manage_seq(LIST, lparams, p) if l != None: t2 = do(l.items[0]) code(ADD, lparams, lparams, t2) free_tmp(t2) #REG if e != None: code(MOVE, dparams, e) else: _do_none(dparams) r = get_tmp(r) # CALL consumes two registers starting from lparams code(CALL, r, fnc, lparams) free_tmp(fnc) #REG free_tmps([lparams, dparams]) #REG return r
def get_led(t, left): r = Token(t.pos, 'get', '.', [left]) items = [left] more = False while not check(P.token, ']'): more = False if check(P.token, ':'): items.append(Token(P.token.pos, 'symbol', 'None')) else: items.append(expression(0)) if check(P.token, ':'): advance(':') more = True if more: items.append(Token(P.token.pos, 'symbol', 'None')) if len(items) > 2: items = [left, Token(t.pos, 'slice', ':', items[1:])] r.items = items advance("]") return r
def ilst(typ, t): return Token(t.pos, typ, typ, lst(t))
def imanage(orig, fnc): items = orig.items orig.val = orig.val[:-1] t = Token(orig.pos, 'symbol', '=', [items[0], orig]) return fnc(t)