def visit_Call(self, node): existing_node = self.generic_visit(node) func_node = existing_node.func if self._is_untraceable_attribute(func_node): return existing_node comparisons = [] # [(name, node)] names = self._get_attribute_names(func_node) if names is not None: comparisons.append( ('.'.join(names[:-1]), existing_node.func.value)) for arg_node in existing_node.args: if isinstance(arg_node, Name): comparisons.append((arg_node.id, arg_node)) if not comparisons: return existing_node args = [ List(elts=[], ctx=Load()), List(elts=[], ctx=Load()), existing_node, List(elts=[], ctx=Load()), Num(n=existing_node.lineno) ] for name, node in comparisons: args[0].elts.append(Str(s=name)) # name args[1].elts.append( # repr() before self._create_bare_context_call('get_repr', [node])) args[3].elts.append( # repr() after self._create_bare_context_call('get_repr', [node])) new_node = self._create_bare_context_call('record_call', args) return new_node
def handle_q(self, q): if not q.children: expr = Name(id="True", **self.file) elif len(q.children) == 1: expr = self._attr_lookup(*q.children[0]) elif q.connector == "AND": expr = Call( func=Name(id="all", **self.file), args=[ List(elts=[self._attr_lookup(k, v) for k, v in q.children], **self.file) ], keywords=[], kwonlyargs=[], **self.file, ) else: # q.connector == 'OR' expr = Call( func=Name(id="any", **self.file), args=[ List(elts=[self._attr_lookup(k, v) for k, v in q.children], **self.file) ], keywords=[], kwonlyargs=[], **self.file, ) if q.negated: return UnaryOp(op=Not(), operand=expr, **self.file) return expr
def visit_While(self, node): return self.make_node( node, "while", values=[ self.visit(node.test), self.make_node( node, "block", values=[ copy_loc( node, List(elts=list(map(self.visit, node.body)), ctx=Load()), ) ], ), self.make_node( node, "block", values=[ copy_loc( node, List(elts=list(map(self.visit, node.orelse)), ctx=Load()), ) ], ), ], )
def visit_For(self, node): return self.make_node( node, "for", values=[ self.visit(node.target), self.visit(node.iter), self.make_node( node, "block", values=[ copy_loc( node, List(elts=list(map(self.visit, node.body)), ctx=Load()), ) ], ), self.make_node( node, "block", values=[ copy_loc( node, List(elts=list(map(self.visit, node.orelse)), ctx=Load()), ) ], ), ], )
def __init__(self, elts): NesType.__init__(self) List.__init__(self, elts=elts) lst = [l.n if isinstance(l, Num) else l for l in elts] list.__init__(self, lst) self.is_used = True self.locked = False
def __init__(self, elts): NesType.__init__(self) List.__init__(self, elts=elts) lst = [l.n if isinstance(l, Num) else l for l in elts] list.__init__(self, lst) self.is_used = True self.locked = False
def __fromkeys(keys, arg=None): if arg: return Call(func=Attribute(value=Name(id='dict', ctx=Load()), attr='fromkeys', ctx=Load()), args=[List(elts=keys, ctx=Load()), arg]) else: return Call(func=Attribute(value=Name(id='dict', ctx=Load()), attr='fromkeys', ctx=Load()), args=[List(elts=keys, ctx=Load())])
def visit_Assign(self, node): self.generic_visit(node) if not all(isinstance(tgt, Name) for tgt in node.targets): return node names = List(elts=[Constant(value=tgt.id) for tgt in node.targets], ctx=Load()) values = List( elts=[Name(id=tgt.id, ctx=Load()) for tgt in node.targets], ctx=Load()) return node, Expr( Call(func=Name(id="__assign__", ctx=Load()), args=[names, values], keywords=[]))
def visit_FunctionDef(self, node, drop_decorator=False): if node.args.posonlyargs: raise WormSyntaxError( "Worm does not support positional only arguments.", at=get_loc(self.filename, node) ) if node.args.kwonlyargs: raise WormSyntaxError( "Worm does not support keyword only arguments.", at=get_loc(self.filename, node) ) args = copy_loc( node, List(elts=list(map(self.visit_arg, node.args.args)), ctx=Load()) ) defaults = copy_loc( node, List(elts=list(map(self.visit, node.args.defaults)), ctx=Load()) ) if len(node.body) > 0 and is_docstring(node.body[0]): d, *body = node.body docstring = d.value else: docstring = Constant(None) body = node.body func = self.make_node( node, "funcDef", values=[ copy_loc(node, Constant(node.name)), args, defaults, self.make_node( node, "block", values=[ copy_loc( node, List(elts=list(map(self.visit, body)), ctx=Load()), ) ], ), node.returns or copy_loc(node, Constant(None)), docstring, ], ) if drop_decorator: return func else: return reduce(compose_dec, reversed(node.decorator_list), func)
def test_param2argparse_param_default_ast_expr_with_list(self) -> None: """ Tests that param2argparse_param works to change the type based on the default whence said default is an ast.List inside an ast.Expr """ run_ast_test( gen_ast=param2argparse_param( ( "byo", { "default": Expr( List( elts=[], ctx=Load(), expr=None, ), expr_value=None, ), "typ": "str", }, ), ), gold=argparse_add_argument_expr, test_case_instance=self, )
def test_parse_to_scalar(self) -> None: """Test various inputs and outputs for `parse_to_scalar`""" for fst, snd in ( (5, 5), ("5", "5"), (set_value(5), 5), (ast.Expr(None), NoneStr), ): self.assertEqual(parse_to_scalar(fst), snd) self.assertEqual( get_value(parse_to_scalar(ast.parse("[5]").body[0]).elts[0]), 5 ) self.assertTrue( cmp_ast( parse_to_scalar(ast.parse("[5]").body[0]), List([set_value(5)], Load()), ) ) self.assertEqual(parse_to_scalar(ast.parse("[5]")), "[5]") parse_to_scalar(ast.parse("[5]").body[0]) self.assertRaises(NotImplementedError, parse_to_scalar, memoryview(b"")) self.assertRaises(NotImplementedError, parse_to_scalar, memoryview(b""))
def collect_output(self, parent): """ Context manager that collects any yield expressions added to ``parent`` and turns them into calls to ``__piglet_acc_list.append``. The name of the accumulator object is returned by the function """ acc = self.unique_id('acc') append = self.unique_id('append') pos = len(parent.body) parent.body.append(Assign(targets=[StoreName(acc)], value=List(elts=[], ctx=Load()))) parent.body.append(Assign(targets=[StoreName(append)], value=Attribute(value=LoadName(acc), attr='append', ctx=Load()))) yield acc for n in parent.body[pos:]: for node, ancestors in astwalk(n): if isinstance(node, Expr) and isinstance(node.value, Yield): node.value = Call(func=LoadName(append), args=[node.value.value], starargs=None, kwargs=None, keywords=[])
def compile(self, im_root): """ Compile a :class:`piglet.intermediate.RootNode` to an :class:`ast.Module` object """ assert self.src_root is None, \ "{!r}.compile called more than once".format(self) try: self.src_root = im_root fn = self._compile_function(im_root, self.module, '__piglet_root__') add_arg_default(fn, make_arg('__piglet_bases'), List(elts=[], ctx=Load())) fn.args.kwarg = make_kwarg('__piglet_extra_blocks') module = self.module module = _hoist_variables_to_piglet_context(module) module = _ensure_all_functions_yield(module) module = _deduplicate_exception_annotations(module) module = add_locations(module) except PigletParseError as e: e.filename = self.filename raise return module
class ImportBombast(RenameBombast): # import sys -> sys = __import__('sys', globals(), locals(), [], 0) one = Transformation(lambda n: Assign( targets=[Name(id=n.names[0].name, ctx=Store())], value=Call(func=Name(id='__import__', ctx=Load()), args=[ Str(s=n.names[0].name), Call(func=Name(id='globals', ctx=Load()), args=[], keywords=[], starargs=None, kwargs=None), Call(func=Name(id='locals', ctx=Load()), args=[], keywords=[], starargs=None, kwargs=None), List(elts=[], ctx=Load()), Num(n=0) ], keywords=[], starargs=None, kwargs=None))) def transform(self): num_imports = len(self.node.names) if num_imports == 1: return self.one.transform(self.node) else: return self.node
def insert_with_block_check(self, node): """Modifies a with statement node in-place to add an initial check for whether or not the block should be executed. If the block is not executed it will raise a XonshBlockError containing the required information. """ nwith = self._nwith # the nesting level of the current with-statement lineno = get_lineno(node) col = get_col(node, 0) # Add or discover target names targets = set() i = 0 # index of unassigned items def make_next_target(): nonlocal i targ = '__xonsh_with_target_{}_{}__'.format(nwith, i) n = Name(id=targ, ctx=Store(), lineno=lineno, col_offset=col) targets.add(targ) i += 1 return n for item in node.items: if item.optional_vars is None: if has_elts(item.context_expr): targs = [make_next_target() for _ in item.context_expr.elts] optvars = Tuple(elts=targs, ctx=Store(), lineno=lineno, col_offset=col) else: optvars = make_next_target() item.optional_vars = optvars else: targets.update(gather_names(item.optional_vars)) # Ok, now that targets have been found / created, make the actual check # to see if we are in a non-executing block. This is equivalent to # writing the following condition: # # if getattr(targ0, '__xonsh_block__', False) or \ # getattr(targ1, '__xonsh_block__', False) or ...: # raise XonshBlockError(lines, globals(), locals()) tests = [_getblockattr(t, lineno, col) for t in sorted(targets)] if len(tests) == 1: test = tests[0] else: test = BoolOp(op=Or(), values=tests, lineno=lineno, col_offset=col) ldx, udx = self._find_with_block_line_idx(node) lines = [Str(s=s, lineno=lineno, col_offset=col) for s in self.lines[ldx:udx]] check = If(test=test, body=[ Raise(exc=xonsh_call('XonshBlockError', args=[List(elts=lines, ctx=Load(), lineno=lineno, col_offset=col), xonsh_call('globals', args=[], lineno=lineno, col=col), xonsh_call('locals', args=[], lineno=lineno, col=col)], lineno=lineno, col=col), cause=None, lineno=lineno, col_offset=col)], orelse=[], lineno=lineno, col_offset=col) node.body.insert(0, check)
def load(self, output_path, variables): self.settings = { OPERATIONS_OUTFILE: Str(output_path), OPERATIONS_ANNOTATEDVARIABLES: List(elts=[Str(variable.name) for variable in variables]) } self.visit(self.operations) return self.operations
def make_slides_assign(slides_count): return Assign( targets=[Name(id='SLIDES', ctx=Store())], value=List( elts=[Name(id=f'Slide{idx}', ctx=Load()) for idx in range(1, slides_count + 1)], ctx=Load() ), type_comment=None )
def visit_AnnAssign(self, node): return self.make_node( node, "assign", values=[ copy_loc(node.target, List(elts=[self.visit(node.target)], ctx=Load())), self.visit(node.value), node.annotation, ], )
def visit_Assign(self, node): return self.make_node( node, "assign", values=[ copy_loc( node, List(elts=list(map(self.visit, node.targets)), ctx=Load()) ), self.visit(node.value), ], )
def _interpolated_str_to_ast_value(source): items = List( [(Str(item) if isinstance(item, (str, ustr)) else parse_and_strip(u'x = ({})'.format(item.value))[0].value) for item in interpolate.parse_interpolations(source)], Load() ) return Call(func=Attribute(Str(''), 'join', Load()), args=[items], starargs=None, kwargs=None, keywords=[])
def test_gen_with_imports_from_file_and_prepended_import(self) -> None: """ Tests `gen` with `imports_from_file` and `prepend` """ output_filename = os.path.join( self.tempdir, "test_gen_with_imports_from_file_and_prepended_import_output.py", ) with patch("sys.stdout", new_callable=StringIO), patch( "sys.stderr", new_callable=StringIO ): self.assertIsNone( gen( name_tpl="{name}Config", input_mapping="gen_test_module.input_map", imports_from_file="gen_test_module", type_="class", prepend=_import_gen_test_module_str, output_filename=output_filename, emit_call=True, emit_default_doc=False, ) ) with open(output_filename, "rt") as f: gen_ast = ast.parse(f.read()) gold = Module( body=[ _import_gen_test_module_ast, _import_star_from_input_ast, self.expected_class_ast, # self.input_module_ast.body[1], Assign( targets=[Name("__all__", Store())], value=List( ctx=Load(), elts=[set_value("FooConfig")], expr=None, ), expr=None, lineno=None, **maybe_type_comment ), ], type_ignores=[], stmt=None, ) run_ast_test( self, gen_ast=gen_ast, gold=gold, )
def visit_ClassDef(self, node): if node.keywords: raise WormSyntaxError( "Worm does not support keywords in class definition.", at=get_loc(self.filename, node) ) if len(node.body) > 0 and is_docstring(node.body[0]): d, *body = node.body docstring = d.value else: docstring = Constant(None) body = node.body class_ = self.make_node( node, "class", values=[ copy_loc(node, Constant(node.name)), copy_loc( node, List(elts=list(map(self.visit, node.bases)), ctx=Load()) ), self.make_node( node, "block", values=[ copy_loc( node, List(elts=list(map(self.visit, body)), ctx=Load()), ) ], ), docstring, ], ) return reduce(compose_dec, reversed(node.decorator_list), class_)
def visit_comp(self, node): """ Find all functions that are called multiple times with the same arguments as we will replace them with one variable """ call_visitor = DuplicateCallFinder() call_visitor.visit(node) # Keep track of what calls we need to replace using a stack so we # support nested comprehensions self.calls_to_replace_stack.append(call_visitor.duplicate_calls) # Visit children of this list comprehension and replace calls self.generic_visit(node) # Gather the existing if statements as we need to move them to the # last comprehension generator (or there will be issues looking up # identifiers) existing_ifs = [] for generator in node.generators: existing_ifs += generator.ifs generator.ifs = [] # Create a new for loop for each function call result that we want # to alias and add it to the list comprehension for call in call_visitor.duplicate_calls: new_comprehension = comprehension( # Notice that we're storing (Store) the result of the call # instead of loading it (Load) target=Name( id=OptimizeComprehensions._identifier_from_Call(call), ctx=Store()), iter=List(elts=[call], ctx=Load()), ifs=[], is_async=0, ) # Add linenos and other things the compile needs to node fix_missing_locations(new_comprehension) node.generators.append(new_comprehension) node.generators[-1].ifs = existing_ifs # Make sure we clear the calls to replace so we don't replace other # calls outside of the scope of this current list comprehension self.calls_to_replace_stack.pop() return node
def parseList(parser): parser.check("List opening '[' ", lexeme="[") listExpression = List(lineo=parser.currentToken[2]) parser.next() while parser.hasnext(): if parser.matchLexeme(']'): parser.next() return listExpression listExpression.addExpression(parseExpression(parser)) if parser.matchLexeme(']'): parser.next() return listExpression parser.check(expected="comma ", lexeme=",") parser.next()
def visit_AugAssign(self, node): target = self.visit(node.target) return self.make_node( node, "assign", values=[ copy_loc(target, List(elts=[target], ctx=Load())), self.make_node( node, "binary", values=[ op_table(node.op), target, self.visit(node.value), ], ), ], )
def test_param2argparse_param_default_ast_expr_with_list(self) -> None: """ Tests that param2argparse_param works to change the type based on the default whence said default is an ast.List inside an ast.Expr """ run_ast_test( gen_ast=param2argparse_param(( "byo", { "default": Expr( List( elts=[], ctx=Load(), expr=None, ), expr_value=None, ), "typ": "str", }, ), ), gold=Expr( Call( args=[set_value("--byo")], func=Attribute( Name("argument_parser", Load()), "add_argument", Load(), ), keywords=[ keyword(arg="action", value=set_value("append"), identifier=None), keyword(arg="required", value=set_value(True), identifier=None), ], expr=None, expr_func=None, )), test_case_instance=self, )
def merge_assignment_lists(node, name, unique_sort=True): """ Merge multiple same-name lists within the body of a node into one, e.g., if you have multiple ```__all__``` :param node: AST node with a '.body' :type node: ```Union[Module, ClassDef, FunctionDef, If, Try, While, With, AsyncFor, AsyncFunctionDef, AsyncWith, ExceptHandler, Expression, For, IfExp, Interactive, Lambda ]``` :param name: Name to match (matches against `id` field of `Name`) :type name: ```str``` :param unique_sort: Whether to ensure its unique + sorted :type unique_sort: ```bool``` """ asses = tuple(get_ass_where_name(node, name)) # if len(asses) < 2: return # Could extract the `AnnAssign` stuff I suppose… del_ass_where_name(node, name) elts = chain.from_iterable( map( attrgetter("elts"), asses, ) ) node.body.append( Assign( targets=[Name("__all__", Store())], value=List( ctx=Load(), elts=sorted(frozenset(elts), key=get_value) if unique_sort else list(elts), expr=None, ), expr=None, lineno=None, **maybe_type_comment ) )
def visit_Call(self, node): if is_unquoting(node.func): if isinstance(node.func, Name) and node.func.id == "_x": n = node.func node.func = copy_loc( n, Attribute( # FIXME that shit will explode value=copy_loc(n, Name(id="worm", ctx=Load())), attr="expand", ctx=Load(), ), ) tr = RewriteTopLevel() node.args = list(map(tr.visit, node.args)) for kw in node.keywords: kw.value = self.visit(kw.value) else: raise NotImplementedError() return node else: return self.make_node( node, "call", values=[ self.visit(node.func), copy_loc( node, List(elts=list(map(self.visit, node.args)), ctx=Load()) ), copy_loc( node, Dict( keys=[Constant(k.arg) for k in node.keywords], values=[self.visit(k.value) for k in node.keywords], ), ), ], )
def build_matcher(tree, modified): if isinstance(tree, Num): return q(LiteralMatcher(u(tree.n))) if isinstance(tree, Str): return q(LiteralMatcher(u(tree.s))) if isinstance(tree, Name): if tree.id in ['True', 'False', 'None']: return q(LiteralMatcher(ast(tree))) elif tree.id in ['_']: return q(WildcardMatcher()) modified.add(tree.id) return q(NameMatcher(u(tree.id))) if isinstance(tree, List): sub_matchers = [] for child in tree.elts: sub_matchers.append(build_matcher(child, modified)) return Call(Name('ListMatcher', Load()), sub_matchers, [], None, None) if isinstance(tree, Tuple): sub_matchers = [] for child in tree.elts: sub_matchers.append(build_matcher(child, modified)) return Call(Name('TupleMatcher', Load()), sub_matchers, [], None, None) if isinstance(tree, Call): sub_matchers = [] for child in tree.args: sub_matchers.append(build_matcher(child, modified)) positional_matchers = List(sub_matchers, Load()) kw_matchers = [] for kw in tree.keywords: kw_matchers.append( keyword(kw.arg, build_matcher(kw.value, modified))) return Call(Name('ClassMatcher', Load()), [tree.func, positional_matchers], kw_matchers, None, None) if (isinstance(tree, BinOp) and isinstance(tree.op, BitAnd)): sub1 = build_matcher(tree.left, modified) sub2 = build_matcher(tree.right, modified) return Call(Name('ParallelMatcher', Load()), [sub1, sub2], [], None, None) raise Exception("Unrecognized tree " + repr(tree))
def exmod( module, emit_name, blacklist, whitelist, output_directory, dry_run, filesystem_layout="as_input", ): """ Expose module as `emit` types into `output_directory` :param module: Module name or path :type module: ```str``` :param emit_name: What type(s) to generate. :type emit_name: ```List[Literal["argparse", "class", "function", "sqlalchemy", "sqlalchemy_table"]]``` :param blacklist: Modules/FQN to omit. If unspecified will emit all (unless whitelist). :type blacklist: ```List[str]``` :param whitelist: Modules/FQN to emit. If unspecified will emit all (minus blacklist). :type whitelist: ```List[str]``` :param output_directory: Where to place the generated exposed interfaces to the given `--module`. :type output_directory: ```str``` :param dry_run: Show what would be created; don't actually write to the filesystem :type dry_run: ```bool``` :param filesystem_layout: Hierarchy of folder and file names generated. "java" is file per package per name. :type filesystem_layout: ```Literal["java", "as_input"]``` """ if dry_run: print("mkdir\t{output_directory!r}".format( output_directory=output_directory)) elif not path.isdir(output_directory): makedirs(output_directory) if blacklist: raise NotImplementedError("blacklist") elif whitelist: raise NotImplementedError("whitelist") module_name, new_module_name = map(path.basename, (module, output_directory)) module = (partial(module_from_file, module_name=module_name) if path.isdir(module) else import_module)(module) module_root_dir = path.dirname(module.__file__) + path.sep _mkdir_and_emit_file = partial( mkdir_and_emit_file, emit_name=emit_name, module_name=module_name, new_module_name=new_module_name, filesystem_layout=filesystem_layout, output_directory=output_directory, dry_run=dry_run, ) # Might need some `groupby` in case multiple files are in the one project; same for `get_module_contents` imports = list( map( _mkdir_and_emit_file, map( lambda name_source: ( name_source[0], (lambda filename: filename[len(module_name) + 1:] if filename.startswith(module_name) else filename) (relative_filename(getfile(name_source[1]))), { "params": OrderedDict(), "returns": OrderedDict() } if dry_run else parse.class_(name_source[1]), ), # sorted( map( lambda name_source: ( name_source[0][len(module_name) + 1:], name_source[1], ), get_module_contents( module, module_root_dir=module_root_dir).items(), ), # key=itemgetter(0), # ), ), ), ) assert len(imports), "Module contents are empty" modules_names = tuple( map( lambda name_module: ( name_module[0], tuple(map(itemgetter(1), name_module[1])), ), groupby( map( lambda node_mod: ( node_mod[0], node_mod[2].module, ), imports, ), itemgetter(0), ), )) init_filepath = path.join(output_directory, new_module_name, "__init__{extsep}py".format(extsep=extsep)) if dry_run: print("write\t{init_filepath!r}".format(init_filepath=init_filepath)) else: emit.file( Module( body=list( chain.from_iterable(( (Expr(set_value("\nExport internal imports\n")), ), map( lambda module_names: ImportFrom( module=module_names[0], names=list( map( lambda names: alias( names, None, identifier=None, identifier_name=None, ), module_names[1], )), level=1, identifier=None, ), modules_names, ), (Assign(targets=[Name("__all__", Store())], value=List( ctx=Load(), elts=list( map( set_value, sorted( frozenset( chain.from_iterable( map( itemgetter(1), modules_names, )), )), )), expr=None, ), expr=None, lineno=None, **maybe_type_comment), ), ))), stmt=None, type_ignores=[], ), init_filepath, mode="wt", )
def populate_files(tempdir, input_module_str=None): """ Populate files in the tempdir :param tempdir: Temporary directory :type tempdir: ```str``` :param input_module_str: Input string to write to the input_filename. If None, uses preset mock module. :type input_module_str: ```Optional[str]``` :returns: input filename, input str, expected_output :rtype: ```Tuple[str, str, str, Module]``` """ input_filename = os.path.join(tempdir, "input{extsep}py".format(extsep=extsep)) input_class_name = "Foo" input_class_ast = emit.class_( parse.function(deepcopy(method_adder_ast)), emit_call=False, class_name=input_class_name, ) input_module_ast = Module( body=[ input_class_ast, Assign(targets=[Name("input_map", Store())], value=Dict( keys=[set_value(input_class_name)], values=[Name(input_class_name, Load())], expr=None, ), expr=None, lineno=None, **maybe_type_comment), Assign( targets=[Name("__all__", Store())], value=List( ctx=Load(), elts=[set_value(input_class_name), set_value("input_map")], expr=None, ), expr=None, lineno=None, **maybe_type_comment), ], type_ignores=[], stmt=None, ) input_module_str = input_module_str or to_code(input_module_ast) # expected_output_class_str = ( # "class FooConfig(object):\n" # ' """\n' # " The amazing Foo\n\n" # " :cvar a: An a. Defaults to 5\n" # ' :cvar b: A b. Defaults to 16"""\n' # " a = 5\n" # " b = 16\n\n" # " def __call__(self):\n" # " self.a = 5\n" # " self.b = 16\n" # ) expected_class_ast = emit.class_( parse.function(deepcopy(method_adder_ast)), emit_call=True, class_name="{input_class_name}Config".format( input_class_name=input_class_name), ) with open(input_filename, "wt") as f: f.write(input_module_str) return input_filename, input_module_ast, input_class_ast, expected_class_ast