def parse(self): """ Convert the parsed tokens into a list of expressions then join them """ self.stream = tokenise(self.source) steps = [] for token in self.stream: code = self._token_to_code(token) if code: steps.append(code) # result = [str(x) for x in steps] return ast.Expression( body=ast.ListComp( elt=build_call( ast.Name(id='str', ctx=ast.Load()), args=[ ast.Name(id='x', ctx=ast.Load()), ], ), generators=[ ast.comprehension( target=ast.Name(id='x', ctx=ast.Store()), iter=ast.List(elts=steps, ctx=ast.Load()), ifs=[] ) ] ) )
def _init_non_primitive_nested_class(node_assign, object_, prop): """ If the nested list is non-primitive, initialise sub-classes in a list comp If the nest is primitive, we can simply get it Marshmallow will do the type marshalling """ return ast.ListComp( elt=ast.Call( func=ast.Name(id=ObjectGenerator._nesting_class(node_assign)), args=[ast.Name(id="el")], keywords=[], ), generators=[ ast.comprehension( target=ast.Name(id="el"), iter=ast.Call( func=ast.Attribute(value=ast.Name(id=object_), attr="get"), args=[ast.Str(s=prop), ast.Dict(keys=[], values=[])], keywords=[], ), ifs=[], is_async=0, ) ], )
def visit_ListComp(self, node: ListComp, *args, **kwargs) -> C.ListComp: elt = self.visit(node.elt, *args, **kwargs) generators = self.visit(node.generators, *args, **kwargs) return C.ListComp( elt=elt, generators=generators, )
def visitListComp(self, n, varchecks, *args): # In comprehensions, we can't generate protectors to guard # arguments since the body is just an expression. Instead we # add variables to varchecks to indicate in visitName that the # variable should be checked directly. This can lead to # duplicated checks but I suspect that's relatively rare. generators, varchecks = self.handleComprehensions( n.generators, n.lineno, n.col_offset, varchecks, *args) elt = self.dispatch(n.elt, varchecks, *args) return ast.ListComp(elt=elt, generators=generators)
def generate(self, element:Element, GC:GenerationContext): acode = element.code if len(acode) is 2 and is_form(acode[1], "for"): for_form = acode[1].code # list comprehension # («[]» (for (in i lst) (f i))) # list compr in_el = for_form[1] in_el_code = in_el.code #with GC.let(domain=ExDom): assert is_identifier(in_el, "in") target_element = in_el_code[1] iter_element = in_el_code[2] with GC.let(domain=LVDom): target_code = GC.generate(target_element) with GC.let(domain=ExDom): iter_code = GC.generate(iter_element) generators = [ ast.comprehension(target=target_code, iter=iter_code, ifs=[]) ] to_evaluate_element = for_form[2] with GC.let(domain=ExDom): to_evaluate_code = GC.generate(to_evaluate_element) return ast.ListComp(to_evaluate_code, generators) else: els = self.generate_as_expressions(GC, *acode[1:]) if GC.domain == LVDom: return ast.List(els, ast.Store()) return expr_wrap(ast.List(els, ast.Load()), GC)
def genProcessCollectionToList( funcName: str, parentName: str, iterFuncName: str, processorFunc: ast.Attribute, firstArgName: str = "parsed", iterVarName: str = "f", returnType: None = None) -> typing.Iterator[ast.FunctionDef]: """Generates a function transforming a collection (`min`) or a macro around it into a python `list` of nodes""" o = ast.Name(id=firstArgName, ctx=ast.Load()) yield ast.FunctionDef( name=funcName, args=ast.arguments( posonlyargs=[], args=[ astSelfArg, ast.arg(arg=firstArgName, annotation=None, type_comment=None), ], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[], ), body=[ ast.Return(value=ast.ListComp( elt=ast.Call( func=processorFunc, args=[ast.Name(id=iterVarName)], keywords=[], ), generators=[ ast.comprehension( target=ast.Name(id=iterVarName), iter=ast.Call(func=ast.Attribute( value=ASTSelf, attr=iterFuncName, ctx=ast.Load()), args=[o], keywords=[]), ifs=[], is_async=0) ], )) ], decorator_list=[], returns=(genTypingIterable(returnType) if returnType else None), type_comment=None, )
def handle_str(node): """Replaces an ast.Constant containing a string, with an ascii representation joined together Args: **node (:obj: `ast.Constant`)**: The node to replace with the ast.Call node Returns: A new obscured node with updated locations """ if type(node.parent) != ast.FormattedValue and type( node.parent.parent) != ast.FormattedValue and type( node.parent) != ast.JoinedStr and type( node.parent.parent) != ast.JoinedStr: new_node = ast.Call( func=ast.Attribute( value=ast.Constant(value='', kind=None), attr='join', ctx=ast.Load(), ), args=[ ast.ListComp( elt=ast.Call(func=ast.Name(id='chr', ctx=ast.Load()), args=[ast.Name(id='x', ctx=ast.Load())], keywords=[]), generators=[ ast.comprehension( target=ast.Name(id='x', ctx=ast.Store()), iter=ast.List( elts=[ ast.Constant(value=ord(x), kind=None) for x in node.value ], ctx=ast.Load(), ), ifs=[], is_async=0, ), ], ), ], keywords=[]) return ast.copy_location(new_node, node) else: return node
def makeGroup(name, base_score, useNumPy=None): groupTemp = ast.Subscript(value=treesAccumulator, slice=ast.Index(value=ast.Name(name))) code = [] res = ast.Name(id="res") baseScore = astNum(n=base_score) treeCall = ast.Call(func=treeTemp, args=[vector], keywords=[]) iterArgDict = { "target": treeTemp, "iter": groupTemp, } if not useNumPy: code.append(ast.Assign(targets=[res], value=baseScore)) code.append( ast.For( body=[ast.AugAssign(target=res, op=ast.Add(), value=treeCall)], orelse=[], **iterArgDict, )) else: res = ast.BinOp( left=baseScore, op=ast.Add(), right=ast.Call( func=numpySum, args=[ ast.ListComp( elt=treeCall, generators=[ ast.comprehension( ifs=[], **iterArgDict, ) ], ) ], keywords=[], ), ) code.append(ast.Return(res)) return wrapWithFunc(code, name)
def compile_list_comprehension(self, expr): # (list-comp expr (target iter) cond?) expr.pop(0) expression = expr.pop(0) tar_it = iter(expr.pop(0)) targets = zip(tar_it, tar_it) cond = self.compile(expr.pop(0)) if expr != [] else None ret = ast.ListComp(lineno=expr.start_line, col_offset=expr.start_column, elt=self.compile(expression), generators=[]) for target, iterable in targets: ret.generators.append( ast.comprehension(target=self._storeize(self.compile(target)), iter=self.compile(iterable), ifs=[])) if cond: ret.generators[-1].ifs.append(cond) return ret
def visit_SetComp(self, node): node = self.generic_visit(node) listexpr = ast.ListComp(elt=node.elt, generators=node.generators) return mk_call('set', [listexpr])
def sequence_helper(self, root, sentence): for i in range(len(root.children)): self.sequence_helper(root.children[i], sentence) for child in root.children[i].children: assert (isinstance(child, LeafNode)) if isinstance(root.children[i], LoopNode): loop_node = root.children[i] loop_var = None iter_obj = None if not loop_node.body_ctx.computed: loop_node.body_ctx.computed = True for key, val in zip(loop_node.children_keys, loop_node.children): target = ast.Name(id=key, ctx=ast.Store()) value = self.choose_child_leaf_node(val) init_assignement = ast.Assign(targets=[target], value=value) # checking forloop case if is_forloop_initializer(init_assignement): loop_var = get_forloop_var(init_assignement) iter_obj = get_forloop_iter_obj(init_assignement) else: sentence.append(init_assignement) iter_stmt = self.translate_ctx(loop_node.body_ctx.body) if loop_var != None and iter_obj != None: loop_node_stmt = loop_node.compute_forloop_stmt( iter_stmt, loop_var, iter_obj) else: loop_node_stmt = loop_node.compute_whileloop_stmt( iter_stmt) sentence += loop_node_stmt param_node = Param(loop_node.id, loop_node.output_var) self.peg.replace_node(loop_node, param_node) if isinstance(root.children[i], BranchNode): branch_node = root.children[i] if not branch_node.body_ctx.computed: for key, val in zip(branch_node.children_keys, branch_node.children): target = ast.Name(id=key, ctx=ast.Store()) value = self.choose_child_leaf_node(val) init_assignement = ast.Assign(targets=[target], value=value) sentence.append(init_assignement) true_stmt = self.translate_ctx( branch_node.body_ctx.true_branch) false_stmt = self.translate_ctx( branch_node.body_ctx.false_branch) branch_node_stmt = branch_node.compute_stmt( true_stmt, false_stmt) branch_node.body_ctx.computed = True for stmt in branch_node_stmt: sentence.append(stmt) param_node = Param(branch_node.id, branch_node.output_var) self.peg.replace_node(branch_node, param_node) if isinstance(root.children[i], BinOpNode): operator = root.children[i] fresh_var = self.create_fresh_var('op') target = ast.Name(id=fresh_var, ctx=ast.Store()) args = [ self.choose_child_leaf_node(arg) for arg in operator.children ] op_exp = ast.BinOp(left=args[0], op=operator.op, right=args[1]) self.process_stmt(sentence, operator, fresh_var, op_exp) param_node = Param(operator.id, fresh_var) self.peg.replace_node(operator, param_node) if isinstance(root.children[i], FunctionCall): func_call = root.children[i] fresh_var = self.create_fresh_var('func_call') target = ast.Name(id=fresh_var, ctx=ast.Store()) name = self.choose_child_leaf_node(func_call.name()) args = [ self.choose_child_leaf_node(arg) for arg in func_call.args() ] func_call_exp = ast.Call(func=name, args=args, keywords=[]) self.process_stmt(sentence, func_call, fresh_var, func_call_exp) param_node = Param(func_call.id, fresh_var) self.peg.replace_node(func_call, param_node) if isinstance(root.children[i], IfExpNode): if_e = root.children[i] fresh_var = self.create_fresh_var('if_exp') target = ast.Name(id=fresh_var, ctx=ast.Store()) cond = self.choose_child_leaf_node(if_e.cond()) t = self.choose_child_leaf_node(if_e.t()) f = self.choose_child_leaf_node(if_e.f()) if_e_exp = ast.IfExp(test=cond, body=t, orelse=f) self.process_stmt(sentence, if_e, fresh_var, if_e_exp) param_node = Param(if_e.id, fresh_var) self.peg.replace_node(if_e, param_node) if isinstance(root.children[i], CompareNode): compare = root.children[i] fresh_var = self.create_fresh_var('comp') target = ast.Name(id=fresh_var, ctx=ast.Store()) head = self.choose_child_leaf_node(compare.head()) tail = [ self.choose_child_leaf_node(arg) for arg in compare.tail() ] cmp_exp = ast.Compare(left=head, ops=compare.ops, comparators=tail) self.process_stmt(sentence, compare, fresh_var, cmp_exp) param_node = Param(compare.id, fresh_var) self.peg.replace_node(compare, param_node) if isinstance(root.children[i], BoolOpNode): bool_op_node = root.children[i] fresh_var = self.create_fresh_var('bool_op') target = ast.Name(id=fresh_var, ctx=ast.Store()) values = [ self.choose_child_leaf_node(arg) for arg in bool_op_node.children ] bool_op_exp = ast.BoolOp(op=bool_op_node.op, values=values) self.process_stmt(sentence, bool_op_node, fresh_var, bool_op_exp) param_node = Param(bool_op_node.id, fresh_var) self.peg.replace_node(bool_op_node, param_node) if isinstance(root.children[i], UnaryOpNode): un_op_node = root.children[i] fresh_var = self.create_fresh_var('unary_op_exp') target = ast.Name(id=fresh_var, ctx=ast.Store()) operand = self.choose_child_leaf_node(un_op_node.operand()) un_op_exp = ast.UnaryOp(op=un_op_node.op, operand=operand) self.process_stmt(sentence, un_op_node, fresh_var, un_op_exp) param_node = Param(un_op_node.id, fresh_var) self.peg.replace_node(un_op_node, param_node) if isinstance(root.children[i], ListNode): list_node = root.children[i] fresh_var = self.create_fresh_var('list_exp') target = ast.Name(id=fresh_var, ctx=ast.Store()) elts = [ self.choose_child_leaf_node(elem) for elem in list_node.children ] list_exp = ast.List(elts=elts, ctx=ast.Load()) self.process_stmt(sentence, list_node, fresh_var, list_exp) param_node = Param(list_node.id, fresh_var) self.peg.replace_node(list_node, param_node) if isinstance(root.children[i], TupleNode): tuple_node = root.children[i] fresh_var = self.create_fresh_var('tuple_exp') target = ast.Name(id=fresh_var, ctx=ast.Store()) elts = [ self.choose_child_leaf_node(elem) for elem in tuple_node.children ] tuple_exp = ast.Tuple(elts=elts, ctx=ast.Load()) self.process_stmt(sentence, tuple_node, fresh_var, tuple_exp) param_node = Param(tuple_node.id, fresh_var) self.peg.replace_node(tuple_node, param_node) if isinstance(root.children[i], SetNode): set_node = root.children[i] fresh_var = self.create_fresh_var('set_exp') target = ast.Name(id=fresh_var, ctx=ast.Store()) elts = [ self.choose_child_leaf_node(elem) for elem in set_node.children ] set_exp = ast.Set(elts=elts, ctx=ast.Load()) self.process_stmt(sentence, set_node, fresh_var, set_exp) param_node = Param(set_node.id, fresh_var) self.peg.replace_node(set_node, param_node) if isinstance(root.children[i], DictNode): dict_node = root.children[i] fresh_var = self.create_fresh_var('dict_exp') target = ast.Name(id=fresh_var, ctx=ast.Store()) keys = [ self.choose_child_leaf_node(k) for k in dict_node.keys() ] values = [ self.choose_child_leaf_node(v) for v in dict_node.values() ] dict_exp = ast.Dict(keys=keys, values=values, ctx=ast.Load()) self.process_stmt(sentence, dict_node, fresh_var, dict_exp) param_node = Param(dict_node.id, fresh_var) self.peg.replace_node(dict_node, param_node) if isinstance(root.children[i], AttributeNode): attr_node = root.children[i] fresh_var = self.create_fresh_var('attr_exp') value = self.choose_child_leaf_node(attr_node.value()) attr = attr_node.attr attr_exp = ast.Attribute(value=value, attr=attr, ctx=ast.Load()) self.rhs_assignement_expressions[attr_node.id] = attr_exp param_node = Param(attr_node.id, fresh_var) self.peg.replace_node(attr_node, param_node) if isinstance(root.children[i], SubscriptNode): subs_node = root.children[i] fresh_var = self.create_fresh_var('subs_exp') target = ast.Name(id=fresh_var, ctx=ast.Store()) value = self.choose_child_leaf_node(subs_node.value()) slice = self.choose_child_leaf_node(subs_node.slice()) subs_exp = ast.Subscript(value=value, slice=slice, ctx=ast.Load()) self.process_stmt(sentence, subs_node, fresh_var, subs_exp) param_node = Param(subs_node.id, fresh_var) self.peg.replace_node(subs_node, param_node) if isinstance(root.children[i], IndexNode): idx_node = root.children[i] fresh_var = self.create_fresh_var('idx_exp') value = self.choose_child_leaf_node(idx_node.value()) idx_exp = ast.Index(value=value) # never append just index to sentence self.rhs_assignement_expressions[idx_node.id] = idx_exp param_node = Param(idx_node.id, fresh_var) self.peg.replace_node(idx_node, param_node) if isinstance(root.children[i], SliceNode): slice_node = root.children[i] fresh_var = self.create_fresh_var('slice_exp') target = ast.Name(id=fresh_var, ctx=ast.Store()) lower = self.choose_child_leaf_node(slice_node.lower()) upper = self.choose_child_leaf_node(slice_node.upper()) step = self.choose_child_leaf_node(slice_node.step()) slice_exp = ast.Slice(lower=lower, upper=upper, step=step) self.rhs_assignement_expressions[slice_node.id] = slice_exp param_node = Param(slice_node.id, fresh_var) self.peg.replace_node(slice_node, param_node) if isinstance(root.children[i], ExtSliceNode): eslice_node = root.children[i] fresh_var = self.create_fresh_var('eslice_exp') dims = [ self.choose_child_leaf_node(d) for d in eslice_node.dims() ] eslice_exp = ast.ExtSlice(dims=dims) self.rhs_assignement_expressions[eslice_node.id] = eslice_exp param_node = Param(eslice_node.id, fresh_var) self.peg.replace_node(eslice_node, param_node) if isinstance(root.children[i], NPArrayNode): ndarr_node = root.children[i] fresh_var = self.create_fresh_var('ndarr_exp') target = ast.Name(id=fresh_var, ctx=ast.Store()) lists = self.choose_child_leaf_node(ndarr_node.children[0]) ndarr_attr = ast.Attribute(value=ast.Name(id='np', ctx=ast.Load()), attr='array') ndarr_exp = ast.Call(func=ndarr_attr, args=[lists], keywords=[]) self.process_stmt(sentence, ndarr_node, fresh_var, ndarr_exp) param_node = Param(ndarr_node.id, fresh_var) self.peg.replace_node(ndarr_node, param_node) if isinstance(root.children[i], ComprehensionNode): comp_node = root.children[i] fresh_var = self.create_fresh_var('compr_exp') t = self.choose_child_leaf_node(comp_node.target()) iter_obj = self.choose_child_leaf_node(comp_node.iter_obj()) ifs = [self.choose_child_leaf_node(i) for i in comp_node.ifs()] comp_exp = ast.comprehension(target=t, iter=iter_obj, ifs=ifs) self.rhs_assignement_expressions[comp_node.id] = comp_exp param_node = Param(comp_node.id, fresh_var) self.peg.replace_node(comp_node, param_node) if isinstance(root.children[i], ListCompNode): lcomp_node = root.children[i] fresh_var = self.create_fresh_var('lcomp_exp') target = ast.Name(id=fresh_var, ctx=ast.Store()) elt = self.choose_child_leaf_node(lcomp_node.element()) generators = [ self.choose_child_leaf_node(gen) for gen in lcomp_node.generators() ] lcomp_exp = ast.ListComp(elt=elt, generators=generators) self.process_stmt(sentence, lcomp_node, fresh_var, lcomp_exp) param_node = Param(lcomp_node.id, fresh_var) self.peg.replace_node(lcomp_node, param_node)
def ListComp(draw, expression) -> ast.ListComp: return ast.ListComp(elt=draw(expression), generators=draw( lists(comprehension(expression), min_size=1, max_size=3)))
def __build_function(self, dom_name, full_name, func_params): assert 'name' in func_params func_name = func_params['name'] docstr = self.__build_desc_string(dom_name, func_name, func_params) args = [ast.arg('self', None)] message_params = [] func_body = [] if docstr: func_body.append(ast.Expr(ast.Str("\n"+docstr+"\n\t\t"))) for param in func_params.get("parameters", []): argname = param['name'] param_optional = param.get("optional", False) if param_optional is False: message_params.append(ast.keyword(argname, ast.Name(id=argname, ctx=ast.Load()))) args.append(ast.arg(argname, None)) if self.do_debug_prints: func_body.append(self.__build_debug_print(argname, argname)) param_type = param.get("type", None) if param_type in CHECKS: if param_optional: check = self.__build_conditional_arg_check(argname, CHECKS[param_type]) else: check = self.__build_unconditional_arg_check(argname, CHECKS[param_type]) if check: func_body.append(check) optional_params = [param.get("name") for param in func_params.get("parameters", []) if param.get("optional", False)] func_kwargs = None if len(optional_params): value = ast.List(elts=[ast.Str(s=param, ctx=ast.Store()) for param in optional_params], ctx=ast.Load()) create_list = ast.Assign(targets=[ast.Name(id='expected', ctx=ast.Store())], value=value) func_body.append(create_list) passed_arg_list = ast.Assign(targets=[ast.Name(id='passed_keys', ctx=ast.Store())], value=ast.Call(func=ast.Name(id='list', ctx=ast.Load()), args=[ast.Call(func=ast.Attribute(value=ast.Name(id='kwargs', ctx=ast.Load()), attr='keys', ctx=ast.Load()), args=[], keywords=[])], keywords=[])) func_body.append(passed_arg_list) comprehension = ast.comprehension(target=ast.Name(id='key', ctx=ast.Store()), iter=ast.Name(id='passed_keys', ctx=ast.Load()), ifs=[], is_async=False) comparator = ast.Name(id='expected', ctx=ast.Load()) listcomp = ast.ListComp(elt=ast.Compare(left=ast.Name(id='key', ctx=ast.Load()), ops=[ast.In()], comparators=[comparator]), generators=[comprehension]) check_message = ast.BinOp( left = ast.Str(s="Allowed kwargs are {}. Passed kwargs: %s".format(optional_params)), op = ast.Mod(), right = ast.Name(id='passed_keys', ctx=ast.Load()), lineno = self.__get_line()) kwarg_check = ast.Assert(test=ast.Call(func=ast.Name(id='all', ctx=ast.Load()), args=[listcomp], keywords=[]), msg=check_message) func_body.append(kwarg_check) func_kwargs = ast.Name(id='kwargs', ctx=ast.Load()) fname = "{}.{}".format(dom_name, func_name) fname = ast.Str(s=fname, ctx=ast.Load()) if (sys.version_info[0], sys.version_info[1]) == (3, 5) or \ (sys.version_info[0], sys.version_info[1]) == (3, 6): # More irritating minor semantic differences in the AST between 3.4 and 3.5 if func_kwargs: message_params.append(ast.keyword(arg=None, value=ast.Name(id='kwargs', ctx=ast.Load()))) communicate_call = ast.Call( func=ast.Attribute(value=ast.Name(id='self', ctx=ast.Load()), ctx=ast.Load(), attr='synchronous_command'), args=[fname], keywords=message_params) elif (sys.version_info[0], sys.version_info[1]) == (3,4): communicate_call = ast.Call( func=ast.Attribute(value=ast.Name(id='self', ctx=ast.Load()), ctx=ast.Load(), attr='synchronous_command'), args=[fname], kwargs=func_kwargs, keywords=message_params) else: print("Version:", sys.version_info) raise RuntimeError("This script only functions on python 3.4, 3.5 or 3.6. Active python version {}.{}".format(*sys.version_info)) do_communicate = ast.Assign(targets=[ast.Name(id='subdom_funcs', ctx=ast.Store())], value=communicate_call) func_ret = ast.Return(value=ast.Name(id='subdom_funcs', ctx=ast.Load())) if len(optional_params) and self.do_debug_prints: func_body.append(self.__build_debug_print('kwargs', 'kwargs')) func_body.append(do_communicate) func_body.append(func_ret) if len(optional_params): kwarg = ast.arg(arg='kwargs', annotation=None) else: kwarg = None sig = ast.arguments( args=args, vararg=None, varargannotation=None, kwonlyargs=[], kwarg=kwarg, kwargannotation=None, defaults=[], kw_defaults=[]) func = ast.FunctionDef( name = "{}_{}".format(full_name, func_name), args = sig, body = func_body, decorator_list = [], lineno = self.__get_line(), col_offset = 0, ) return func
kw_defaults=[1], kwonlyargs=[], ), ast.AnnAssign(simple=True, target=ast.Tuple(ast.Name("a", ast.Load()))), ast.With(items=[]), ast.Raise(exc=None, cause=1), ast.Try(body=[]), ast.Try(body=[ast.Pass()], handlers=[], finalbody=[]), ast.Try( body=[ast.Pass()], handlers=[], finalbody=[ast.Pass()], orelse=[ast.Pass()], ), ast.ImportFrom(level=-1, names=[1]), ast.BoolOp(values=[1]), ast.Dict(keys=[1], values=[]), ast.Compare(comparators=[]), ast.Compare(comparators=[1], ops=[]), ast.Constant(value=int), ast.Constant(value=(1, 2, int)), ast.ListComp(generators=[]), ], ) def test_simple_ast_validator(node): validator = SimpleASTValidator() with pytest.raises(SyntaxError) as cm: validator.validate(ast.fix_missing_locations(node)) assert cm.value.node is node
def test_operators(self): boolop0 = ast.BoolOp() boolop1 = ast.BoolOp(ast.And(), [ ast.Name('True', ast.Load()), ast.Name('False', ast.Load()), ast.Name('a',ast.Load())]) boolop2 = ast.BoolOp(ast.And(), [ ast.Name('True', ast.Load()), ast.Name('False', ast.Load()), ast.Name('a',ast.Load())], 0, 0) binop0 = ast.BinOp() binop1 = ast.BinOp(ast.Str('xy'), ast.Mult(), ast.Num(3)) binop2 = ast.BinOp(ast.Str('xy'), ast.Mult(), ast.Num(3), 0, 0) unaryop0 = ast.UnaryOp() unaryop1 = ast.UnaryOp(ast.Not(), ast.Name('True',ast.Load())) unaryop2 = ast.UnaryOp(ast.Not(), ast.Name('True',ast.Load()), 0, 0) lambda0 = ast.Lambda() lambda1 = ast.Lambda(ast.arguments([ast.Name('x', ast.Param())], None, None, []), ast.Name('x', ast.Load())) ifexp0 = ast.IfExp() ifexp1 = ast.IfExp(ast.Name('True',ast.Load()), ast.Num(1), ast.Num(0)) ifexp2 = ast.IfExp(ast.Name('True',ast.Load()), ast.Num(1), ast.Num(0), 0, 0) dict0 = ast.Dict() dict1 = ast.Dict([ast.Num(1), ast.Num(2)], [ast.Str('a'), ast.Str('b')]) dict2 = ast.Dict([ast.Num(1), ast.Num(2)], [ast.Str('a'), ast.Str('b')], 0, 0) set0 = ast.Set() set1 = ast.Set([ast.Num(1), ast.Num(2)]) set2 = ast.Set([ast.Num(1), ast.Num(2)], 0, 0) lc0 = ast.ListComp() lc1 = ast.ListComp( ast.Name('x',ast.Load()), [ast.comprehension(ast.Name('x', ast.Store()), ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])]) lc2 = ast.ListComp( ast.Name('x',ast.Load()), [ast.comprehension(ast.Name('x', ast.Store()), ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])], 0, 0) setcomp0 = ast.SetComp() setcomp1 = ast.SetComp(ast.Name('x', ast.Load()), [ast.comprehension(ast.Name('x', ast.Store()), ast.Str('abracadabra'), [ast.Compare(ast.Name('x', ast.Load()), [ast.NotIn()], [ast.Str('abc')])])]) comprehension0 = ast.comprehension() comprehension1 = ast.comprehension(ast.Name('x', ast.Store()), ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), []) # "{i : chr(65+i) for i in (1,2)}") dictcomp0 = ast.DictComp() dictcomp1 = ast.DictComp(ast.Name('i', ast.Load()), ast.Call(ast.Name('chr', ast.Load()), [ast.BinOp(ast.Num(65), ast.Add(), ast.Name('i', ast.Load()))], [], None, None), [ast.comprehension(ast.Name('i', ast.Store()), ast.Tuple([ast.Num(1), ast.Num(n=2)], ast.Load()), [])]) dictcomp2 = ast.DictComp(ast.Name('i', ast.Load()), ast.Call(ast.Name('chr', ast.Load()), [ast.BinOp(ast.Num(65), ast.Add(), ast.Name('i', ast.Load()))], [], None, None), [ast.comprehension(ast.Name('i', ast.Store()), ast.Tuple([ast.Num(1), ast.Num(n=2)], ast.Load()), [])],0,0) # (x for x in (1,2)) genexp0 = ast.GeneratorExp() genexp1 = ast.GeneratorExp(ast.Name('x', ast.Load()), [ast.comprehension(ast.Name('x', ast.Store()), ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])]) genexp2 = ast.GeneratorExp(ast.Name('x', ast.Load()), [ast.comprehension(ast.Name('x', ast.Store()), ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])],0,0) # yield 2 yield0 = ast.Yield() yield1 = ast.Yield(ast.Num(2)) yield2 = ast.Yield(ast.Num(2),0,0) yield20 = ast.Yield(lineno=0, col_offset=0) # a>0 compare0 = ast.Compare() compare1 = ast.Compare(ast.Name('a', ast.Load()), [ast.Gt()], [ast.Num(0)]) compare2 = ast.Compare(ast.Name('a', ast.Load()), [ast.Gt()], [ast.Num(0)],0,0) # chr(65) call0 = ast.Call() call1 = ast.Call(ast.Name('chr', ast.Load()), [ast.Num(65)], [], None, None) call2 = ast.Call(ast.Name('chr', ast.Load()), [ast.Num(65)], [], None, None, 0, 0) call20 = ast.Call(ast.Name('f', ast.Load()), [ast.Num(0)], []) call21 = ast.Call(ast.Name('f', ast.Load()), [ast.Num(0)], [], lineno=0, col_offset=0) # 0 num0 = ast.Num() num1 = ast.Num(0) num2 = ast.Num(0,0,0) # "foo" str0 = ast.Str() str1 = ast.Str("foo") str2 = ast.Str("foo",0,0) # TODO: come back repr0 = ast.Repr() repr1 = ast.Repr(ast.Num(0)) repr2 = ast.Repr(ast.Num(0),0,0) # foo.bar attr0 = ast.Attribute() attr1 = ast.Attribute(ast.Name('foo', ast.Load()), 'bar', ast.Load()) attr2 = ast.Attribute(ast.Name('foo', ast.Load()), 'bar', ast.Load(), 0,0) # a[1:2] subscript0 = ast.Subscript() subscript1 = ast.Subscript(ast.Name('a', ast.Load()), ast.Slice(ast.Num(1), ast.Num(2)), ast.Load()) subscript2 = ast.Subscript(ast.Name('a', ast.Load()), ast.ExtSlice([ast.Num(1), ast.Num(2)]), ast.Load(), 0, 0) # name name0 = ast.Name() name1 = ast.Name("name", ast.Load()) name2 = ast.Name("name", ast.Load(),0,0) # [1,2] list0 = ast.List() list1 = ast.List([ast.Num(1), ast.Num(2)], ast.Load()) list2 = ast.List([ast.Num(1), ast.Num(2)], ast.Load(),0,0) # (1,2) tuple0 = ast.Tuple() tuple1 = ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()) tuple2 = ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load(), 0, 0)
def test_empty_init(self): # Jython 2.5.0 did not allow empty constructors for many ast node types # but CPython ast nodes do allow this. For the moment, I don't see a # reason to allow construction of the super types (like ast.AST and # ast.stmt) as well as the op types that are implemented as enums in # Jython (like boolop), but I've left them in but commented out for # now. We may need them in the future since CPython allows this, but # it may fall under implementation detail. #ast.AST() ast.Add() ast.And() ast.Assert() ast.Assign() ast.Attribute() ast.AugAssign() ast.AugLoad() ast.AugStore() ast.BinOp() ast.BitAnd() ast.BitOr() ast.BitXor() ast.BoolOp() ast.Break() ast.Call() ast.ClassDef() ast.Compare() ast.Continue() ast.Del() ast.Delete() ast.Dict() ast.Div() ast.Ellipsis() ast.Eq() ast.Exec() ast.Expr() ast.Expression() ast.ExtSlice() ast.FloorDiv() ast.For() ast.FunctionDef() ast.GeneratorExp() ast.Global() ast.Gt() ast.GtE() ast.If() ast.IfExp() ast.Import() ast.ImportFrom() ast.In() ast.Index() ast.Interactive() ast.Invert() ast.Is() ast.IsNot() ast.LShift() ast.Lambda() ast.List() ast.ListComp() ast.Load() ast.Lt() ast.LtE() ast.Mod() ast.Module() ast.Mult() ast.Name() ast.Not() ast.NotEq() ast.NotIn() ast.Num() ast.Or() ast.Param() ast.Pass() ast.Pow() ast.Print() ast.RShift() ast.Raise() ast.Repr() ast.Return() ast.Slice() ast.Store() ast.Str() ast.Sub() ast.Subscript() ast.Suite() ast.TryExcept() ast.TryFinally() ast.Tuple() ast.UAdd() ast.USub() ast.UnaryOp() ast.While() ast.With() ast.Yield() ast.alias() ast.arguments() #ast.boolop() #ast.cmpop() ast.comprehension() #ast.excepthandler() #ast.expr() #ast.expr_context() ast.keyword()
def makeRegressor(model: Xgboost, useSigmoids: bool = False, useNumPy: bool = False) -> ast.Module: """Converts a regression XGBoost decision tree model into a python module of if-else trees""" #code:typing.List[ast.stmt]=[] code = [] numpyModuleName = "numpy" if useSigmoids: code.extend( generateFermiSplitFunction( sigmoidSplitFuncName, tanhModuleName=(numpyModuleName if useNumPy else "math"))) if useNumPy: code.append( ast.ImportFrom(module=numpyModuleName, names=[ast.alias(name=numpySumName, asname=None)], level=0)) features = int(model.param.num_feature) trees = int(model.gbm_.param.num_trees) groups = int(model.gbm_.param.num_output_group) if groups == 0: groups = 1 treesInAGroup = trees // groups groupFunctions = [] groupFunctionsNames = [] elts = [] for j in range(groups): group = [] for i in range(treesInAGroup): t = model.gbm_.trees[j * treesInAGroup + i] name = "g" + str(j) + "_t" + str(i) code.extend(treesToFuncs(t, name, useSigmoids)) group.append(ast.Name(id=name)) elts.append(ast.Tuple(elts=group)) groupFuncName = "g" + str(j) groupFunctionsNames.append(ast.Name(groupFuncName)) groupFunctions.append( makeGroup(groupFuncName, model.param.base_score, useNumPy=useNumPy)) code.extend(groupFunctions) code.append( ast.Assign(targets=[treesAccumulator], value=ast.Dict(keys=groupFunctionsNames, values=elts))) code.append( wrapWithFunc([ ast.Expr(value=astStr(s=genModelSummary(model))), ast.Assert(test=ast.Compare(left=ast.Call( func=lenFunc, args=[vector], keywords=[]), ops=[ast.Eq()], comparators=[astNum(n=features)]), msg=ast.BinOp(left=ast.BinOp( left=astStr(s="This model requires exactly " + str(features) + " feature, but "), op=ast.Add(), right=ast.Call(func=strFunc, args=[ ast.Call(func=lenFunc, args=[vector], keywords=[]) ], keywords=[])), op=ast.Add(), right=astStr(s=" were given"))), ast.Assign( targets=[res], value=ast.ListComp( elt=ast.Call(func=groupTemp, args=[vector], keywords=[]), generators=[ ast.comprehension( target=groupTemp, iter=treesAccumulator, ifs=[]) ])), ast.Return(value=res) ], regressorFunctionName)) return ast.Module(body=code)
def visitListComp(self, n, *args): generators = self.reduce(n.generators, *args) elt = self.dispatch(n.elt, *args) return ast.ListComp(elt=elt, generators=generators)
def as_ast(dct): """See https://docs.python.org/2/library/ast.html""" if dct['ast_type'] == "Module": return ast.Module(dct["body"]) elif dct['ast_type'] == "Interactive": return ast.Interactive(dct["body"]) elif dct['ast_type'] == "Expression": return ast.Expression(dct["body"]) elif dct['ast_type'] == "Suite": return ast.Suite(dct["body"]) elif dct['ast_type'] == "FunctionDef": return ast.FunctionDef(dct["name"], dct["args"], dct["body"], dct["decorator_list"]) elif dct['ast_type'] == "ClassDef": return ast.ClassDef(dct["name"], dct["bases"], dct["body"], dct["decorator_list"]) elif dct['ast_type'] == "Return": return ast.Return(dct["value"]) elif dct['ast_type'] == "Delete": return ast.Delete(dct["targets"]) elif dct['ast_type'] == "Assign": return ast.Assign(dct["targets"], dct["value"]) elif dct['ast_type'] == "AugAssign": return ast.AugAssign(dct["target"], dct["op"], dct["value"]) elif dct['ast_type'] == "Print": return ast.Print(dct["dest"], dct["values"], dct["nl"]) elif dct['ast_type'] == "For": return ast.For(dct["target"], dct["iter"], dct["body"], dct["orelse"]) elif dct['ast_type'] == "While": return ast.While(dct["test"], dct["body"], dct["orelse"]) elif dct['ast_type'] == "If": return ast.If(dct["test"], dct["body"], dct["orelse"]) elif dct['ast_type'] == "With": return ast.With(dct["context_expr"], dct["optional_vars"], dct["body"]) elif dct['ast_type'] == "Raise": return ast.Raise(dct["type"], dct["inst"], dct["tback"]) elif dct['ast_type'] == "TryExcept": return ast.TryExcept(dct["body"], dct["handlers"], dct["orelse"]) elif dct['ast_type'] == "TryFinally": return ast.TryFinally(dct["body"], dct["finalbody"]) elif dct['ast_type'] == "Assert": return ast.Assert(dct["test"], dct["msg"]) elif dct['ast_type'] == "Import": return ast.Import(dct["names"]) elif dct['ast_type'] == "ImportFrom": return ast.ImportFrom(dct["module"], dct["names"], dct["level"]) elif dct['ast_type'] == "Exec": return ast.Exec(dct["body"], dct["globals"], dct["locals"]) elif dct['ast_type'] == "Global": return ast.Global(dct["names"]) elif dct['ast_type'] == "Expr": return ast.Expr(dct["value"]) elif dct['ast_type'] == "Pass": return ast.Pass() elif dct['ast_type'] == "Break": return ast.Break() elif dct['ast_type'] == "Continue": return ast.Continue() elif dct['ast_type'] == "BoolOp": return ast.BoolOp(dct["op"], dct["values"]) elif dct['ast_type'] == "BinOp": return ast.BinOp(dct["left"], dct["op"], dct["right"]) elif dct['ast_type'] == "UnaryOp": return ast.UnaryOp(dct["op"], dct["operand"]) elif dct['ast_type'] == "Lambda": return ast.Lambda(dct["args"], dct["body"]) elif dct['ast_type'] == "IfExp": return ast.IfExp(dct["test"], dct["body"], dct["orelse"]) elif dct['ast_type'] == "Dict": return ast.Dict(dct["keys"], dct["values"]) elif dct['ast_type'] == "Set": return ast.Set(dct["elts"]) elif dct['ast_type'] == "ListComp": return ast.ListComp(dct["elt"], dct["generators"]) elif dct['ast_type'] == "SetComp": return ast.SetComp(dct["elt"], dct["generators"]) elif dct['ast_type'] == "DictComp": return ast.DictComp(dct["key"], dct["value"], dct["generators"]) elif dct['ast_type'] == "GeneratorExp": return ast.GeneratorExp(dct["elt"], dct["generators"]) elif dct['ast_type'] == "Yield": return ast.Yield(dct["value"]) elif dct['ast_type'] == "Compare": return ast.Compare(dct["left"], dct["ops"], dct["comparators"]) elif dct['ast_type'] == "Call": return ast.Call(dct["func"], dct["args"], dct["keywords"], dct["starargs"], dct["kwargs"]) elif dct['ast_type'] == "Repr": return ast.Repr(dct["value"]) elif dct['ast_type'] == "Num": return ast.Num(dct["n"]) elif dct['ast_type'] == "Str": # Converting to ASCII return ast.Str(dct["s"].encode('ascii', 'ignore')) elif dct['ast_type'] == "Attribute": return ast.Attribute(dct["value"], dct["attr"], dct["ctx"]) elif dct['ast_type'] == "Subscript": return ast.Subscript(dct["value"], dct["slice"], dct["ctx"]) elif dct['ast_type'] == "Name": return ast.Name(dct["id"], dct["ctx"]) elif dct['ast_type'] == "List": return ast.List(dct["elts"], dct["ctx"]) elif dct['ast_type'] == "Tuple": return ast.Tuple(dct["elts"], dct["ctx"]) elif dct['ast_type'] == "Load": return ast.Load() elif dct['ast_type'] == "Store": return ast.Store() elif dct['ast_type'] == "Del": return ast.Del() elif dct['ast_type'] == "AugLoad": return ast.AugLoad() elif dct['ast_type'] == "AugStore": return ast.AugStore() elif dct['ast_type'] == "Param": return ast.Param() elif dct['ast_type'] == "Ellipsis": return ast.Ellipsis() elif dct['ast_type'] == "Slice": return ast.Slice(dct["lower"], dct["upper"], dct["step"]) elif dct['ast_type'] == "ExtSlice": return ast.ExtSlice(dct["dims"]) elif dct['ast_type'] == "Index": return ast.Index(dct["value"]) elif dct['ast_type'] == "And": return ast.And() elif dct['ast_type'] == "Or": return ast.Or() elif dct['ast_type'] == "Add": return ast.Add() elif dct['ast_type'] == "Sub": return ast.Sub() elif dct['ast_type'] == "Mult": return ast.Mult() elif dct['ast_type'] == "Div": return ast.Div() elif dct['ast_type'] == "Mod": return ast.Mod() elif dct['ast_type'] == "Pow": return ast.Pow() elif dct['ast_type'] == "LShift": return ast.LShift() elif dct['ast_type'] == "RShift": return ast.RShift() elif dct['ast_type'] == "BitOr": return ast.BitOr() elif dct['ast_type'] == "BitXor": return ast.BitXor() elif dct['ast_type'] == "BitAnd": return ast.BitAnd() elif dct['ast_type'] == "FloorDiv": return ast.FloorDiv() elif dct['ast_type'] == "Invert": return ast.Invert() elif dct['ast_type'] == "Not": return ast.Not() elif dct['ast_type'] == "UAdd": return ast.UAdd() elif dct['ast_type'] == "USub": return ast.USub() elif dct['ast_type'] == "Eq": return ast.Eq() elif dct['ast_type'] == "NotEq": return ast.NotEq() elif dct['ast_type'] == "Lt": return ast.Lt() elif dct['ast_type'] == "LtE": return ast.LtE() elif dct['ast_type'] == "Gt": return ast.Gt() elif dct['ast_type'] == "GtE": return ast.GtE() elif dct['ast_type'] == "Is": return ast.Is() elif dct['ast_type'] == "IsNot": return ast.IsNot() elif dct['ast_type'] == "In": return ast.In() elif dct['ast_type'] == "NotIn": return ast.NotIn() elif dct['ast_type'] == "comprehension": return ast.comprehension(dct["target"], dct["iter"], dct["ifs"]) elif dct['ast_type'] == "ExceptHandler": return ast.ExceptHandler(dct["type"], dct["name"], dct["body"]) elif dct['ast_type'] == "arguments": return ast.arguments(dct["args"], dct["vararg"], dct["kwarg"], dct["defaults"]) elif dct['ast_type'] == "keyword": return ast.keyword(dct["arg"], dct["value"]) elif dct['ast_type'] == "alias": return ast.alias(dct["name"], dct["asname"]) else: return dct
def visit_DictComp(self, node): node = self.generic_visit(node) elt = mk_tuple([node.key, node.value]) listexpr = ast.ListComp(elt=elt, generators=node.generators) return mk_call('dict', [listexpr])