def SynErr_idx(string): """Return the index of a SyntaxError, or None.""" try: ast.parse(string) except SyntaxError as e: return e.offset - 1 # we want zero-indexed
def transpile(input, prefix='.', outdir=None, namespace='python', verbosity=0): transpiler = Transpiler(namespace=namespace, verbosity=verbosity) for file_or_dir in input: if os.path.isfile(file_or_dir): if verbosity: print("Compiling %s ..." % file_or_dir) with open(file_or_dir) as source: ast_module = ast.parse(source.read(), mode='exec') transpiler.transpile(file_or_dir, ast_module, prefix) elif os.path.isdir(file_or_dir): for root, dirs, files in os.walk(file_or_dir, followlinks=True): for filename in files: if os.path.splitext(filename)[1] == '.py': source_file = os.path.join(root, filename) if verbosity: print("Compiling %s ..." % source_file) with open(source_file) as source: ast_module = ast.parse(source.read(), mode='exec') transpiler.transpile(source_file, ast_module, prefix) else: print("Unknown source file: %s" % file_or_dir, file=sys.stderr) transpiler.write(outdir)
def is_valid_python(source): try: ast.parse(source, mode='eval') except SyntaxError: return False else: return True
def parse_config(files, commands): inner_globals = dict(SAFE_FUNCTIONS) inner_globals["__builtins__"] = {} # Disable any builtin function inner_globals["named_items"] = {} # Store the provided named items for file in files: text = file.read() root = ast.parse(text, filename=file.name) execute_code(root, file.name, inner_globals) for text in commands: root = ast.parse(text, filename="<execute>") execute_code(root, '--execute', inner_globals) # Make sure we have the necessary variables in the inner_globals dictionary for varname in REQUIRED: if varname not in inner_globals: raise ParseException("Missing the {!r} variable".format(varname)) comparer = inner_globals["comparer"] groups = inner_globals["groups"] items = inner_globals["named_items"] if "total" in inner_globals: total = inner_globals["total"] else: total = len(groups) return Config(comparer, items, groups, total)
def ParseCodeToTree(code): """Parse the given code to a lib2to3 pytree. Arguments: code: a string with the code to parse. Raises: SyntaxError if the code is invalid syntax. parse.ParseError if some other parsing failure. Returns: The root node of the parsed tree. """ # This function is tiny, but the incantation for invoking the parser correctly # is sufficiently magical to be worth abstracting away. try: # Try to parse using a Python 3 grammar, which is more permissive (print and # exec are not keywords). parser_driver = driver.Driver(_GRAMMAR_FOR_PY3, convert=pytree.convert) tree = parser_driver.parse_string(code, debug=False) except parse.ParseError: # Now try to parse using a Python 2 grammar; If this fails, then # there's something else wrong with the code. try: parser_driver = driver.Driver(_GRAMMAR_FOR_PY2, convert=pytree.convert) tree = parser_driver.parse_string(code, debug=False) except parse.ParseError: # Raise a syntax error if the code is invalid python syntax. try: ast.parse(code) except SyntaxError as e: raise e else: raise return _WrapEndMarker(tree)
def post(self, filename): cserver = self.get_server() isFolder = self.get_argument('isFolder', default=None) if isFolder: self.write(cserver.ensure_dir(filename)) else: contents = self.get_argument('contents', default='') force = int(self.get_argument('force', default=0)) if filename.endswith('.py') or cserver.is_macro(filename): if not contents.endswith('\n'): text = contents + '\n' # to make ast.parse happy else: text = contents try: ast.parse(text, filename=filename, mode='exec') # parse it looking for syntax errors except Exception as err: cserver.send_pub_msg(str(err), 'file_errors') self.send_error(400) return if not force: ret = cserver.file_forces_reload(filename) if ret: self.send_error(409) # user will be prompted to overwrite file and reload project return self.write(str(cserver.write_file(filename, contents)))
def walk(filename, visitor): """Generate an AST for the given filename and walk over it using the given visitor instance. """ filename = os.path.abspath(filename) try: tree = ast.parse(open(filename, 'r').read()) except SyntaxError: if sys.version_info[0] < 3: e = sys.exc_info()[1] log.warn('SyntaxError while parsing file %s: %s' % (filename, str(e))) return # We're probably in Python 3 and looking at a file intended for # Python 2. Otherwise there's an unintended SyntaxError in the # file, so there are bigger problems anyways try: import lib2to3.refactor tool = StringRefactoringTool( lib2to3.refactor.get_fixers_from_package('lib2to3.fixes')) tool.refactor_file(filename, write=True) tree = ast.parse(tool.refactored[filename]) except ImportError: # Without 2to3 we can't do much more. # TODO: Issue a warning? return visitor.visit(tree)
def run(data): if not hasattr(data, 'can_reproduce_example_from_repr'): data.can_reproduce_example_from_repr = True with local_settings(self.settings): with BuildContext(data, is_final=is_final): with deterministic_PRNG(): args, kwargs = data.draw(self.search_strategy) if expected_failure is not None: text_repr[0] = arg_string(test, args, kwargs) if print_example: example = '%s(%s)' % ( test.__name__, arg_string(test, args, kwargs)) try: ast.parse(example) except SyntaxError: data.can_reproduce_example_from_repr = False report('Falsifying example: %s' % (example,)) elif current_verbosity() >= Verbosity.verbose: report( lambda: 'Trying example: %s(%s)' % ( test.__name__, arg_string(test, args, kwargs))) with deterministic_PRNG(): return test(*args, **kwargs)
def infer_ast(src): """Attempts to infer an abstract syntax tree from the provided value. - Python ast.AST instances are passed through. - Strings are parsed. A SyntaxError is raised if invalid. - Functions are sent through cypy.fn_get_source to get a source string, then parsed. If the source can't be a found, an exception is raised by fn_get_source. .. WARNING:: Functions defined on the iPython command line do not have their source saved. A bug has been filed: http://github.com/ipython/ipython/issues/issue/120 """ if isinstance(src, _ast.AST): return src elif isinstance(src, basestring): return _ast.parse(src) else: # if a function instance is passed in, it's source is found # and parsed. note that finding source can be precarious for # functions defined on the command line. If you get an error # you'll have to use strings instead of regular function # definitions src = cypy.fn_get_source(src) return _ast.parse(src)
def test_function(self): """ Function - must exist in the module defined by the owner. Numeric functions have a blank owner. """ # First we want to check the functions that have owners, attempt to # import the owners and check if the function is a member of the owner. idx = (self.data.owner != '') | (self.data.functiontype == 'NumexprFunction') for data in self.data[idx][['owner', 'function']].itertuples(): # TODO - Validate the numeric functions. if data.owner == '__builtin__': try: ast.parse(data.function) idx[data.Index - 1] = True except SyntaxError: idx[data.Index - 1] = False elif data.owner != '': try: module = importlib.import_module(data.owner) idx[data.Index - 1] = hasattr(module, data.function) except ImportError: idx[data.Index - 1] = False idx = numpy.logical_not(idx) self.assertEqual(len(self.data[idx]), 0, msg='Invalid function:\n%s' % self.data[idx][['id', 'function', 'owner']])
def scripts2rst(path, f): """ creates rst summary documentation for files in scripts folder which is not a package so cannot be imported """ # get list of script files try: files = [name for name in os.listdir(path) if not name.startswith("_") and name.endswith(".py")] except: return f.write("**Scripts**\n\n") # loop over script files for name in files: sfile = os.path.join(path, name) try: try: source = ast.parse(open(sfile, "r", encoding="utf8").read()) except: #py2 fails if encoding in string source = ast.parse(open(sfile, "r").read()) except Exception as e: log.warning("Problem parsing %s\n%s"%(name, e)) f.write(name+"\n") doc = i.cleandoc(ast.get_docstring(source)) or "." doc = py2decode(doc) #py2 f.write(" "+doc.splitlines()[0]+"\n") f.write("\n")
def __init__(self, ast_node=None, source=None, path=None, filename=None): if filename: with open(filename, 'r') as file_obj: source = file_obj.read() ast_node = ast.parse(source) self._filename = filename self._path = path elif path: source = inspect.getmodule() ast_node = ast.parse(self._source) self._filename = inspect.getsourcefile(Import(path=path)._import) self._path = path else: self._path = None self._filename = None if ast_node and not isinstance(ast_node, ast.Module): raise TypeError('Expected an ast.Module object') super(Module, self).__init__(ast_node=ast_node, source=source) map( # set metadata properties on the object lambda x: setattr(self, x, self.get_var('__' + x + '__')), METADATA ) self._type = 'Module'
def test_free_vars(self): stmt = ast.parse("foo", "", "single") with self.assertRaises(ValueError): freevars(stmt, []) suite = ast.parse("foo; bar();", "exec") with self.assertRaises(ValueError): freevars(suite, []) def freevars_(source, env=[]): return freevars(ast.parse(source, "", "eval"), env) self.assertEqual(freevars_("1"), []) self.assertEqual(freevars_("..."), []) self.assertEqual(freevars_("a"), ["a"]) self.assertEqual(freevars_("a", ["a"]), []) self.assertEqual(freevars_("f(1)"), ["f"]) self.assertEqual(freevars_("f(x)"), ["f", "x"]) self.assertEqual(freevars_("f(x)", ["f"]), ["x"]) self.assertEqual(freevars_("a + 1"), ["a"]) self.assertEqual(freevars_("a + b"), ["a", "b"]) self.assertEqual(freevars_("a + b", ["a", "b"]), []) self.assertEqual(freevars_("a[b]"), ["a", "b"]) self.assertEqual(freevars_("a[b]", ["a", "b"]), []) self.assertEqual(freevars_("f(x, *a)", ["f"]), ["x", "a"]) self.assertEqual(freevars_("f(x, *a, y=1)", ["f"]), ["x", "a"]) self.assertEqual(freevars_("f(x, *a, y=1, **k)", ["f"]), ["x", "a", "k"]) if sys.version_info >= (3, 5): self.assertEqual(freevars_("f(*a, *b, k=c, **d, **e)", ["f"]), ["a", "b", "c", "d", "e"]) self.assertEqual(freevars_("True"), []) self.assertEqual(freevars_("'True'"), []) self.assertEqual(freevars_("None"), []) self.assertEqual(freevars_("b'None'"), []) self.assertEqual(freevars_("a < b"), ["a", "b"]) self.assertEqual(freevars_("a < b <= c"), ["a", "b", "c"]) self.assertEqual(freevars_("1 < a <= 3"), ["a"]) self.assertEqual(freevars_("{}"), []) self.assertEqual(freevars_("[]"), []) self.assertEqual(freevars_("()"), []) self.assertEqual(freevars_("[a, 1]"), ["a"]) self.assertEqual(freevars_("{a: b}"), ["a", "b"]) self.assertEqual(freevars_("{a, b}"), ["a", "b"]) self.assertEqual(freevars_("0 if abs(a) < 0.1 else b", ["abs"]), ["a", "b"]) self.assertEqual(freevars_("lambda a: b + 1"), ["b"]) self.assertEqual(freevars_("lambda a: b + 1", ["b"]), []) self.assertEqual(freevars_("lambda a: a + 1"), []) self.assertEqual(freevars_("(lambda a: a + 1)(a)"), ["a"]) self.assertEqual(freevars_("lambda a, *arg: arg + (a,)"), []) self.assertEqual(freevars_("lambda a, *arg, **kwargs: arg + (a,)"), []) self.assertEqual(freevars_("[a for a in b]"), ["b"]) self.assertEqual(freevars_("[1 + a for c in b if c]"), ["a", "b"]) self.assertEqual(freevars_("{a for _ in [] if b}"), ["a", "b"]) self.assertEqual(freevars_("{a for _ in [] if b}", ["a", "b"]), [])
def veloce_generate_js(filepath, requirejs=False, root_path=None, output=None, deep=None): dirname = os.path.abspath(os.path.dirname(filepath)) if not root_path: root_path = dirname basename = os.path.basename(filepath) output_name = os.path.join(dirname, basename + '.js') if not output: print('Generating {}'.format(output_name)) # generate js with open(os.path.join(dirname, basename)) as f: input = parse(f.read()) tree = parse(input) python_core = Veloce() python_core.visit(tree) script = python_core.writer.value() if requirejs: out = 'define(function(require) {\n' out += script if isinstance(python_core.__all__, str): out += '\nreturn {};\n'.format(python_core.__all__) elif python_core.__all__: public = '{{{}}}'.format(', '.join(map(lambda x: '{}: {}'.format(x[0], x[1]), zip(python_core.__all__, python_core.__all__)))) out += '\nreturn {};\n'.format(public) else: raise Exception('__all__ is not defined!') out += '\n})\n' script = out if deep: for dependency in python_core.dependencies: if dependency.startswith('.'): generate_js(os.path.join(dirname, dependency + '.py'), requirejs, root_path, output, deep) else: generate_js(os.path.join(root_path, dependency[1:] + '.py'), requirejs, root_path, output, deep) output.write(script)
def update(self, name, script, template): self.name = name try: ast.parse(script) self.script = script except SyntaxError, e: raise UserException(e)
def correct_parentheses(s): try: t = re.sub('([)\]])', r'\1,', s) ast.parse(t) return True except Exception: return False
def _analyzeGens(top, absnames): genlist = [] for g in top: if isinstance(g, _UserCode): tree = g elif isinstance(g, (_AlwaysComb, _AlwaysSeq, _Always)): f = g.func s = inspect.getsource(f) s = _dedent(s) tree = ast.parse(s) #print ast.dump(tree) tree.sourcefile = inspect.getsourcefile(f) tree.lineoffset = inspect.getsourcelines(f)[1]-1 tree.symdict = f.func_globals.copy() tree.callstack = [] # handle free variables tree.nonlocaldict = {} if f.func_code.co_freevars: for n, c in zip(f.func_code.co_freevars, f.func_closure): obj = _cell_deref(c) if isinstance(g, _AlwaysComb): if not ( isinstance(obj, (int, long, EnumType,_Signal)) or \ _isMem(obj) or _isTupleOfInts(obj) ): info = "File %s, line %s: " % (tree.sourcefile, tree.lineoffset) print type(obj) raise ConversionError(_error.UnsupportedType, n, info) tree.symdict[n] = obj # currently, only intbv as automatic nonlocals (until Python 3.0) if isinstance(obj, intbv): tree.nonlocaldict[n] = obj tree.name = absnames.get(id(g), str(_Label("BLOCK"))).upper() v = _FirstPassVisitor(tree) v.visit(tree) if isinstance(g, _AlwaysComb): v = _AnalyzeAlwaysCombVisitor(tree, g.senslist) elif isinstance(g, _AlwaysSeq): v = _AnalyzeAlwaysSeqVisitor(tree, g.senslist, g.reset, g.sigregs, g.varregs) else: v = _AnalyzeAlwaysDecoVisitor(tree, g.senslist) v.visit(tree) else: # @instance f = g.gen.gi_frame s = inspect.getsource(f) s = _dedent(s) tree = ast.parse(s) # print ast.dump(tree) tree.sourcefile = inspect.getsourcefile(f) tree.lineoffset = inspect.getsourcelines(f)[1]-1 tree.symdict = f.f_globals.copy() tree.symdict.update(f.f_locals) tree.nonlocaldict = {} tree.callstack = [] tree.name = absnames.get(id(g), str(_Label("BLOCK"))).upper() v = _FirstPassVisitor(tree) v.visit(tree) v = _AnalyzeBlockVisitor(tree) v.visit(tree) genlist.append(tree) return genlist
def prettyPrintNode(self, p): '''The driver for beautification: beautify a single node.''' # c = self.c if not should_beautify(p): # @nobeautify is in effect. return if not p.b: # Pretty printing might add text! return if not p.b.strip(): # Do this *after* we are sure @beautify is in effect. self.replace_body(p, '') return t1 = time.time() # Replace Leonine syntax with special comments. comment_string, s0 = comment_leo_lines(p) try: s1 = g.toEncodedString(s0) node1 = ast.parse(s1, filename='before', mode='exec') except IndentationError: self.skip_message('IndentationError', p) return except SyntaxError: self.skip_message('SyntaxError', p) return except Exception: g.es_exception() self.skip_message('Exception', p) return t2 = time.time() readlines = g.ReadLinesClass(s0).next tokens = list(tokenize.generate_tokens(readlines)) t3 = time.time() s2 = self.run(tokens) t4 = time.time() try: s2_e = g.toEncodedString(s2) node2 = ast.parse(s2_e, filename='before', mode='exec') ok = compare_ast(node1, node2) except Exception: g.es_exception() g.trace('Error in %s...\n%s' % (p.h, s2_e)) self.skip_message('BeautifierError', p) return if not ok: self.skip_message('BeautifierError', p) return t5 = time.time() # Restore the tags after the compare s3 = uncomment_leo_lines(comment_string, p, s2) self.replace_body(p, s3) # Update the stats self.n_input_tokens += len(tokens) self.n_output_tokens += len(self.code_list) self.n_strings += len(s3) self.parse_time += (t2 - t1) self.tokenize_time += (t3 - t2) self.beautify_time += (t4 - t3) self.check_time += (t5 - t4) self.total_time += (t5 - t1)
def visit_Call(self, node): name = node.func.id args = node.args[0] if name in self.variables: if isinstance(args, UnaryOp): # we have s(+1) if (isinstance(args.op, UAdd)): args = args.operand date = args.n elif (isinstance(args.op, USub)): args = args.operand date = -args.n else: raise Exception("Unrecognized subscript.") else: date = args.n if self.shift =='S': return ast.parse('{}'.format(name)).body[0].value else: new_date = date+self.shift if new_date != 0: return ast.parse('{}({})'.format(name,new_date)).body[0].value else: return ast.parse('{}'.format(name)).body[0].value else: # , keywords=node.keywords, kwargs=node.kwargs) return Call(func=node.func, args=[self.visit(e) for e in node.args], keywords=[])
def __init__(self, network, name, source, target, parameters): Connector.__init__(self, network, name, source, target, parameters) # lets load up the weight ModularConnectorFunction's self.weight_functions = {} self.delay_functions = {} self.simulator_time_step = self.sim.get_time_step() # lets determine the list of variables in weight expressions v = ExpVisitor() v.visit(ast.parse(self.parameters.weight_expression)) self.weight_function_names = v.names # lets determine the list of variables in delay expressions v = ExpVisitor() v.visit(ast.parse(self.parameters.delay_expression)) self.delay_function_names = v.names for k in self.weight_function_names: self.weight_functions[k] = load_component(self.parameters.weight_functions[k].component)( self.source, self.target, self.parameters.weight_functions[k].params ) assert isinstance(self.weight_functions[k], ModularConnectorFunction) for k in self.delay_function_names: self.delay_functions[k] = load_component(self.parameters.delay_functions[k].component)( self.source, self.target, self.parameters.delay_functions[k].params )
def test_forbidden_names(self): tree = ast.parse("__abc__") with self.assertRaises(exceptions.SyntaxError) as cm: self.validator(tree) self.assertEquals(1, cm.exception.lineno) self.assertEquals(1, cm.exception.offset) self.assertEquals("double_underscore_name", cm.exception.reason) tree = ast.parse("__") with self.assertRaises(exceptions.SyntaxError) as cm: self.validator(tree) self.assertEquals(1, cm.exception.lineno) self.assertEquals(1, cm.exception.offset) self.assertEquals("double_underscore_name", cm.exception.reason) tree = ast.parse("___xtra_underscores___") with self.assertRaises(exceptions.SyntaxError) as cm: self.validator(tree) self.assertEquals(1, cm.exception.lineno) self.assertEquals(1, cm.exception.offset) self.assertEquals("double_underscore_name", cm.exception.reason) tree = ast.parse("__abc__.__attr__") with self.assertRaises(exceptions.SyntaxError) as cm: self.validator(tree) self.assertEquals(1, cm.exception.lineno) self.assertEquals(1, cm.exception.offset) self.assertEquals("double_underscore_name", cm.exception.reason)
def test_generate_files(self): """Test generate_files returns a tuple.""" self.prototype.set_jinja_env(self.api_version) details = self.prototype.generate_files() self.assertIsInstance(details, list) # namedtuples in tuple for file_details in details: self.assertIsInstance(file_details, tuple) self.assertIsInstance(file_details.filename, basestring) self.assertIsInstance(file_details.filecontent, basestring) name, contents = file_details if name.endswith(".py"): # We have a "coding utf-8" line in there, we need to encode contents = contents.encode("utf-8") ast.parse(contents) if pep8: checker = pep8.Checker( name, contents.splitlines(True)) res = checker.check_all() self.assertFalse( res, "Python file {0} has pep8 errors:\n" "{1}\n{2}".format(name, checker.report.messages, repr(contents)) ) elif name.endswith(".xml"): # TODO validate valid odoo xml lxml.etree.fromstring(contents)
def runcode(self, the_code, source, filename='<input>'): # code taken from InteractiveInterpreter.runsource in code.py try: tree = ast.parse(source) try: expr = ast.parse(source, mode='eval') except: expr = None #todo get this to work for multiple expr's, not just 1: if expr and len(tree.body) == 1: # _ = expr_value tree.body[0] = ast_wrap_in_assn('_', tree.body[0]) # print _ underscore = _ast.Name(id="_", ctx=_ast.Load()) print_node = ast_print_node([_ast.Str(s=' '*50), underscore]) tree.body.append(print_node) # play_whatever #todo doesn't work for generators yet play_whatever_node = ast_call_node('music.play_whatever', '_', show_notes=SHOW_NOTES) tree.body.append(play_whatever_node) #print ast.dump(tree) code_obj = compile(tree, '<input>', 'exec') exec code_obj in self.locals except SystemExit: raise except: self.showtraceback() else: if code.softspace(sys.stdout, 0): print
def warn_about_none_ast(self, node, module_path, lineno): """ Returns an AST issuing a warning if the value of node is `None`. This is used to warn the user when asserting a function that asserts internally already. See issue #3191 for more details. """ # Using parse because it is different between py2 and py3. AST_NONE = ast.parse("None").body[0].value val_is_none = ast.Compare(node, [ast.Is()], [AST_NONE]) send_warning = ast.parse( """ from _pytest.warning_types import PytestWarning from warnings import warn_explicit warn_explicit( PytestWarning('asserting the value None, please use "assert is None"'), category=None, filename={filename!r}, lineno={lineno}, ) """.format( filename=module_path.strpath, lineno=lineno ) ).body return ast.If(val_is_none, send_warning, [])
def test_get_docstring(self): node = ast.parse('def foo():\n """line one\n line two"""') self.assertEqual(ast.get_docstring(node.body[0]), 'line one\nline two') node = ast.parse('async def foo():\n """spam\n ham"""') self.assertEqual(ast.get_docstring(node.body[0]), 'spam\nham')
def OnApply(self, event): """Event handler for Apply button""" # See if we have valid python try: ast.parse(self.macros) except: # Grab the traceback and print it for the user s = StringIO() e = exc_info() # usr_tb will more than likely be none because ast throws # SytnaxErrorsas occurring outside of the current # execution frame usr_tb = get_user_codeframe(e[2]) or None print_exception(e[0], e[1], usr_tb, None, s) post_command_event(self.parent, self.MacroErrorMsg, err=s.getvalue()) success = False else: self.result_ctrl.SetValue('') post_command_event(self.parent, self.MacroReplaceMsg, macros=self.macros) post_command_event(self.parent, self.MacroExecuteMsg) success = True event.Skip() return success
def get_changes(self): if not self.text_before: return tree_before = ast.parse(self.text_before) tree = ast.parse(self.text) if ast.dump(tree)==ast.dump(tree_before): print('status','no changes to the script') else: print('status','executing changes to %s'%self.file) # identify changed nodes in the tree and execute # note that this feature reruns any changed child of the script parent #! track line numbers are report to the user? tree_before,tree = [[self.CodeChunk(i,index=ii) for ii,i in enumerate(ast.iter_child_nodes(ast.parse(t)))] for t in [self.text_before,self.text]] intersect = set.intersection(set(tree),set(tree_before)) novel = list(set.difference(set(tree),intersect)) novel_linenos = set([i.this.lineno for i in novel]) class CodeSurgery(ast.NodeTransformer): def visit(self, node): if hasattr(node,'lineno') and node.lineno not in novel_linenos: return ast.parse('last_lineno = %d'%node.lineno).body[0] else: return ast.NodeTransformer.generic_visit(self,node) code_ready = ast.fix_missing_locations(CodeSurgery().visit(ast.parse(self.text))) # run the remainder out = self.namespace #! exec to eval for python <2.7.15 eval(compile(code_ready,filename='<ast>',mode='exec'),out,out)
def python_code(tokenIterator, envName, endToken): """ Given an iterator of tokens, and the name of current environment, iterates through to the end of the environment, and returns the text in the environment as python code. This function only works for environments that are supposed to contain Python code. """ count = 0 pythonLine = '' pythonCode = [] #parseNodes = [] startLine = 0 for token, lineNum in tokenIterator: if not startLine: #The first token is the endline at the end of \begin{envName}, so the environment starts on the same line as the first token in tokenIterator startLine = lineNum if endToken == token: pythonCode = [line for line in pythonCode if line.strip()] #We use this to figure out what the top level indent is, and strip that away so that Python can parse the code properly. topLevelIndent = 1 if pythonCode[0][0] == '\t' else compute_indent(pythonCode[0]) try: ast.parse(''.join(line[topLevelIndent:] for line in pythonCode)) except SyntaxError, e: raise transExceptions.TranslationError(' '.join([parseTree.color("Error:", parseTree.bcolors.RED), 'Error in Python code found in', parseTree.color(envName,parseTree.bcolors.YELLOW), 'environment. Environment start line:', parseTree.color(str(startLine), parseTree.bcolors.GREEN), 'Python error:\n\n', str(e)])) else: return (startLine, pythonCode) else: if token.strip(' \t') == '\n': pythonCode.append(pythonLine + token) pythonLine = '' else: pythonLine += token
def compare_strings(a,b): t1 = ast.parse(a) t2 = ast.parse(b) comp = Compare() val = comp.compare(t1,t2) d = comp.d return val
def pythonium_generate_js(filepath, requirejs=False, root_path=None, output=None, deep=None): dirname = os.path.abspath(os.path.dirname(filepath)) if not root_path: root_path = dirname basename = os.path.basename(filepath) output_name = os.path.join(dirname, basename + '.js') if not output: print('Generating {}'.format(output_name)) # generate js with open(os.path.join(dirname, basename)) as f: input = parse(f.read()) tree = parse(input) pythonium = Pythonium() pythonium.visit(tree) script = pythonium.writer.value() if requirejs: out = 'define(function(require) {\n' out += script all = map(lambda x: "'{}': {}".format(x, x), pythonium.__all__) all = '{{{}}}'.format(', '.join(all)) out += 'return {}'.format(all) out += '\n})\n' script = out if deep: for dependency in python_core.dependencies: if dependency.startswith('.'): generate_js(os.path.join(dirname, dependency + '.py'), requirejs, root_path, output, deep) else: generate_js(os.path.join(root_path, dependency[1:] + '.py'), requirejs, root_path, output, deep) output.write(script)
def find_version(*parts): finder = VersionFinder() finder.visit(ast.parse(read(*parts))) return finder.version
def parse_code(student_code): """ parses code and returns a new CaitNode that is the root of the student """ return CaitNode(ast.parse(student_code))
def temp_code(input_file): f = open(input_file,"r") x = v() t = ast.parse(f.read()) x.visit(t) return x.listL
def check_django_versions(self, git_repo): """ A repo is either a library or an application. If it's a library, then it should have a setup.py or a setup.cfg, and it should have a tox.ini file that runs its tests with at least django 1.8 and django 1.11, if its setup.py lists Django as a dependency. If it's a repository, then it should have a requirements/base.txt (or at least a requirements.txt) and that should pin either Django==1.8 or Django==1.11, if it specifies Django as a dependency. """ working_dir = Path(git_repo.working_tree_dir) setup_py = working_dir / 'setup.py' setup_cfg = working_dir / 'setup.cfg' requirements_base_txt = working_dir / 'requirements/base.txt' requirements_txt = working_dir / 'requirements.txt' manage_py = working_dir / 'manage.py' tox_ini = working_dir / 'tox.ini' is_django_application = manage_py.exists() print(is_django_application) print(setup_py.exists()) if requirements_base_txt.exists(): requirements_file = requirements_base_txt else: requirements_file = requirements_txt print(requirements_file.exists()) if not is_django_application and setup_py.exists(): parsed_setup_py = ast.parse(setup_py.bytes(), 'setup.py') if uses_pbr(parsed_setup_py): has_django = requirements_txt_has_django(requirements_file) else: has_django = setup_py_has_django(parsed_setup_py) if not has_django: return tested_versions = tox_tested_django_versions(tox_ini) assert LIBRARY_REQUIRED_DJANGO_VERSIONS in tested_versions elif requirements_file.exists(): django_specifier = None for req in parsed_requirements_txt(requirements_file): if requirement_is_django(req): if django_specifier is None: django_specifier = req.specifier else: django_specifier &= req.specifier if django_specifier is None: return largest_accepted_version = max(version for version in DJANGO_VERSIONS if version in django_specifier) for supported_version in APPLICATION_ALLOWED_DJANGO_VERSIONS: if largest_accepted_version in supported_version: return msg = ( "No allowed version range contained the largest allowed django " + f"version {largest_accepted_version} in the version specifier {django_specifier}" ) assert False, msg else: assert False, "Couldn't determine if repo is an application or a library"
def _log_problem(self, error_code, message): self.problems.append(Problem(self._cursor.node, error_code, message)) class Checker: name = "flake8-bionic-dnode-match" version = "0.1" def __init__(self, tree): self._tree = tree def run(self): visitor = MatchVisitor() visitor.visit(self._tree) for problem in visitor.problems: yield problem.to_flake8_tuple() if __name__ == "__main__": import sys from pathlib import Path filenames = sys.argv[1:] for filename in filenames: code = Path(filename).read_text() tree = ast.parse(code) for message in Checker(tree).run(): print(message) # noqa: T001
def second(file_name): print("Second star: {}".format(sum(eval(compile((translate_back if (translate_back := ast.NodeTransformer()) and not setattr(translate_back, 'visit_Add', lambda _: ast.Mult()) and not setattr(translate_back, 'visit_Mult', lambda _: ast.Add()) else None).visit(ast.parse(io.open(file_name).read().translate({43: 42, 42: 43, 10: 44}), mode='eval')), '', 'eval')))))
def run( script_path: str, method_name: str = "main", args: Optional[Tuple] = None, kwargs: Optional[Dict] = None, project: Any = None, _include_frame: bool = False, ) -> Any: """Loads a project script and runs a method in it. script_path: path of script to load method_name: name of method to run args: method args kwargs: method kwargs project: (deprecated) Returns: return value from called method """ if args is None: args = tuple() if kwargs is None: kwargs = {} script, project = _get_path(script_path) # temporarily add project objects to the main namespace, so the script can import them if project is not None: project._add_to_main_namespace() # modify sys.path to ensure script can be imported root_path = Path(".").resolve().root sys.path.insert(0, root_path) try: module = _import_from_path(script) name = module.__name__ if not hasattr(module, method_name): raise AttributeError(f"Module '{name}' has no method '{method_name}'") try: module_path = Path(module.__file__).relative_to(Path(".").absolute()) except ValueError: module_path = Path(module.__file__) print( f"\nRunning '{color('bright blue')}{module_path}{color}::" f"{color('bright cyan')}{method_name}{color}'..." ) func = getattr(module, method_name) if not _include_frame: return func(*args, **kwargs) # this voodoo preserves the call frame of the function after it has finished executing. # we do this so that `brownie run -i` is able to drop into the console with the same # namespace as the function that it just ran. # first, we extract the source code from the function and parse it to an AST source = inspect.getsource(func) func_ast = ast.parse(source) # next, we insert some new logic into the beginning of the function. this imports # the sys module and assigns the current frame to a global var `__brownie_frame` injected_source = "import sys\nglobal __brownie_frame\n__brownie_frame = sys._getframe()" injected_ast = ast.parse(injected_source) func_ast.body[0].body = injected_ast.body + func_ast.body[0].body # type: ignore # now we compile the AST into a code object, using the module's `__dict__` as our globals # so that we have access to all the required imports and other objects f_locals: Dict = module.__dict__.copy() del f_locals[method_name] func_code = compile(func_ast, "", "exec") exec(func_code, f_locals) # finally, we execute our new function from inside the copied globals dict. the frame # is added to the dict as `__global_frame` per our injected code, and we return it for # use within the console. so simple! return_value = f_locals[method_name](*args, **kwargs) return return_value, f_locals["__brownie_frame"] finally: # cleanup namespace and sys.path sys.path.remove(root_path) if project is not None: project._remove_from_main_namespace()
def _uses_loop(function): loop_statements = ast.For, ast.While, ast.AsyncFor, ast.ListComp nodes = ast.walk(ast.parse(inspect.getsource(function))) return any(isinstance(node, loop_statements) for node in nodes)
def check(code, filename="<unknown>", options=None): if options is None: options = Flake8Options checker.memoize.mem = {} Flake8Checker.parse_options(options) return list(Flake8Checker(ast.parse(code), filename).run())
def from_code(code: str, field: str = "", inner=True, simplify=True, conds_align=False): """ Get a Flowchart instance from a str of Python code. Args: code: str, Python code to draw flowchart field: str, path to field (function) you want to draw flowchart inner: bool, True: parse the body of field; Field: parse the body as an object simplify: bool, for If & Loop statements: simplify the one-line-body or not. conds_align: bool, for consecutive If statements: conditionNode alignment support (Issue#14) or not Returns: A Flowchart instance parsed from given code. `inner=True` means parse `field.body`, otherwise parse [field]. E.g. ``` def a(): print('a') ``` inner=True => `st (function a) -> subroutine (print) -> end` inner=False => `op=>operation: def a(): print('a')` The field is the path to the target of flowchartilizing. It should be the *path* to a `def` code block in code. E.g. ``` def foo(): pass class Bar(): def fuzz(self): pass def buzz(self, f): def g(self): f(self) return g(self) Bar().buzz(foo) ``` Available path: - "" (means the whole code) - "foo" - "Bar.fuzz" - "Bar.buzz" - "Bar.buzz.g" """ code_ast = ast.parse(code) field_ast = Flowchart.find_field_from_ast(code_ast, field) assert hasattr(field_ast, "body") assert field_ast.body, f"{field}: nothing to parse. Check given code and field please." f = field_ast.body if inner else [field_ast] p = parse(f, simplify=simplify, conds_align=conds_align) return Flowchart(p.head)
def process(filename, exclude_dirs=['test','site-packages']): """Process a VFS filename for Brython.""" print("Generating {}".format(filename)) nb = 0 nb_err = 0 main_root = os.path.dirname(filename) VFS = {} for stdlib_dir in ("libs", "Lib"): lib_path = os.path.join(main_root, stdlib_dir) for root, _dir, files in os.walk(lib_path): flag = False root_elts = root.split(os.sep) for exclude in exclude_dirs: if exclude in root_elts: flag = True continue if flag: continue # skip these modules if '__pycache__' in _dir: _dir.remove("__pycache__") nb += 1 if stdlib_dir == "Lib": if root == lib_path: package = [] else: package = root[len(lib_path) + 1:].split(os.sep) for _file in files: ext = os.path.splitext(_file)[1] if ext not in ('.js', '.py'): continue if re.match(r'^module\d+\..*$', _file): continue if not git.in_index(_file): print(_file, "not in git index") continue nb += 1 file_name = os.path.join(root, _file) with open(file_name, encoding='utf-8') as f: data = f.read() if ext == '.py': data = python_minifier.minify(data, preserve_lines=True) path_elts = package[:] if os.path.basename(filename) != "__init__.py": path_elts.append(os.path.basename(file_name)[:-3]) fqname = ".".join(path_elts) with open(os.path.join(root, file_name), encoding="utf-8") as f: tree = ast.parse(f.read()) visitor = Visitor(lib_path, package) visitor.visit(tree) imports = sorted(list(visitor.imports)) vfs_path = os.path.join(root, _file).replace(main_root, '') vfs_path = vfs_path.replace("\\", "/") if vfs_path.startswith('/libs/crypto_js/rollups/'): if _file not in ('md5.js', 'sha1.js', 'sha3.js', 'sha224.js', 'sha256.js', 'sha384.js', 'sha512.js'): continue mod_name = vfs_path[len(stdlib_dir) + 2:].replace('/', '.') mod_name, ext = os.path.splitext(mod_name) is_package = mod_name.endswith('__init__') if ext == ".py": if is_package: mod_name = mod_name[:-9] VFS[mod_name] = [ext, data, imports, 1] else: VFS[mod_name] = [ext, data, imports] else: VFS[mod_name] = [ext, data] print("adding {}".format(mod_name)) print('{} files, {} errors'.format(nb, nb_err)) with open(filename, "w") as out: out.write('__BRYTHON__.use_VFS = true;\n') out.write('__BRYTHON__.VFS={}\n\n'.format(json.dumps(VFS)))
def safe_eval(expr, locals={}, include_exceptions=False): ''' this is intended for allowing things like: with_items: a_list_variable where Jinja2 would return a string but we do not want to allow it to call functions (outside of Jinja2, where the env is constrained) Based on: http://stackoverflow.com/questions/12523516/using-ast-and-whitelists-to-make-pythons-eval-safe ''' # this is the whitelist of AST nodes we are going to # allow in the evaluation. Any node type other than # those listed here will raise an exception in our custom # visitor class defined below. SAFE_NODES = set(( ast.Expression, ast.Compare, ast.Str, ast.List, ast.Tuple, ast.Dict, ast.Call, ast.Load, ast.BinOp, ast.UnaryOp, ast.Num, ast.Name, ast.Add, ast.Sub, ast.Mult, ast.Div, )) # AST node types were expanded after 2.6 if not sys.version.startswith('2.6'): SAFE_NODES.union(set((ast.Set, ))) # builtin functions that are not safe to call INVALID_CALLS = ( 'classmethod', 'compile', 'delattr', 'eval', 'execfile', 'file', 'filter', 'help', 'input', 'object', 'open', 'raw_input', 'reduce', 'reload', 'repr', 'setattr', 'staticmethod', 'super', 'type', ) class CleansingNodeVisitor(ast.NodeVisitor): def generic_visit(self, node): if type(node) not in SAFE_NODES: #raise Exception("invalid expression (%s) type=%s" % (expr, type(node))) raise Exception("invalid expression (%s)" % expr) super(CleansingNodeVisitor, self).generic_visit(node) def visit_Call(self, call): if call.func.id in INVALID_CALLS: raise Exception("invalid function: %s" % call.func.id) if not isinstance(expr, basestring): # already templated to a datastructure, perhaps? if include_exceptions: return (expr, None) return expr try: parsed_tree = ast.parse(expr, mode='eval') cnv = CleansingNodeVisitor() cnv.visit(parsed_tree) compiled = compile(parsed_tree, expr, 'eval') result = eval(compiled, {}, locals) if include_exceptions: return (result, None) else: return result except SyntaxError, e: # special handling for syntax errors, we just return # the expression string back as-is if include_exceptions: return (expr, None) return expr
def setup(loader): module = ast.parse(SRC) module.__preinit__('threading', '', None, internal=True) loader.setupStandrad('threading', module)
def _replace_magics( source: Sequence[str], magic_substitutions: List[MagicHandler], command: str, *, skip_bad_cells: bool, ) -> str: """ Replace IPython line magics with valid python code. Parameters ---------- source Source from notebook cell. magic_substitutions List to store all the ipython magics substitutions Returns ------- str Line from cell, with line magics replaced with python code """ try: ast.parse("".join(source)) except SyntaxError: pass else: # Source has no IPython magic, return it directly return "".join(source) cell_magic_finder = CellMagicFinder() body = TransformerManager().transform_cell("".join(source)) try: tree = ast.parse(body) except SyntaxError: if skip_bad_cells: handler = MagicHandler("".join(source), command, magic_type=None) magic_substitutions.append(handler) return handler.replacement return "".join(source) cell_magic_finder.visit(tree) # if first line is cell magic, process it separately if cell_magic_finder.header is not None: assert cell_magic_finder.body is not None header = _process_source( cell_magic_finder.header, command, magic_substitutions, skip_bad_cells=skip_bad_cells, ) cell = _process_source( cell_magic_finder.body, command, magic_substitutions, skip_bad_cells=skip_bad_cells, ) return "\n".join([header, cell]) return _process_source( "".join(source), command, magic_substitutions, skip_bad_cells=skip_bad_cells )
def h_visit(code): '''Compile the code into an AST tree and then pass it to :func:`~radon.metrics.h_visit_ast`. ''' return h_visit_ast(ast.parse(code))
def recursive_finder(name, data, py_module_names, py_module_cache, zf): """ Using ModuleDepFinder, make sure we have all of the module_utils files that the module its module_utils files needs. """ # Parse the module and find the imports of ansible.module_utils tree = ast.parse(data) finder = ModuleDepFinder() finder.visit(tree) # # Determine what imports that we've found are modules (vs class, function. # variable names) for packages # normalized_modules = set() # Loop through the imports that we've found to normalize them # Exclude paths that match with paths we've already processed # (Have to exclude them a second time once the paths are processed) module_utils_paths = [ p for p in module_utils_loader._get_paths(subdirs=False) if os.path.isdir(p) ] module_utils_paths.append(_MODULE_UTILS_PATH) for py_module_name in finder.submodules.difference(py_module_names): module_info = None if py_module_name[0] == 'six': # Special case the python six library because it messes up the # import process in an incompatible way module_info = imp.find_module('six', module_utils_paths) py_module_name = ('six', ) idx = 0 elif py_module_name[0] == '_six': # Special case the python six library because it messes up the # import process in an incompatible way module_info = imp.find_module( '_six', [os.path.join(p, 'six') for p in module_utils_paths]) py_module_name = ('six', '_six') idx = 0 else: # Check whether either the last or the second to last identifier is # a module name for idx in (1, 2): if len(py_module_name) < idx: break try: module_info = imp.find_module(py_module_name[-idx], [ os.path.join(p, *py_module_name[:-idx]) for p in module_utils_paths ]) break except ImportError: continue # Could not find the module. Construct a helpful error message. if module_info is None: msg = [ 'Could not find imported module support code for %s. Looked for' % (name, ) ] if idx == 2: msg.append('either %s.py or %s.py' % (py_module_name[-1], py_module_name[-2])) else: msg.append(py_module_name[-1]) raise AnsibleError(' '.join(msg)) # Found a byte compiled file rather than source. We cannot send byte # compiled over the wire as the python version might be different. # imp.find_module seems to prefer to return source packages so we just # error out if imp.find_module returns byte compiled files (This is # fragile as it depends on undocumented imp.find_module behaviour) if module_info[2][2] not in (imp.PY_SOURCE, imp.PKG_DIRECTORY): msg = [ 'Could not find python source for imported module support code for %s. Looked for' % name ] if idx == 2: msg.append('either %s.py or %s.py' % (py_module_name[-1], py_module_name[-2])) else: msg.append(py_module_name[-1]) raise AnsibleError(' '.join(msg)) if idx == 2: # We've determined that the last portion was an identifier and # thus, not part of the module name py_module_name = py_module_name[:-1] # If not already processed then we've got work to do # If not in the cache, then read the file into the cache # We already have a file handle for the module open so it makes # sense to read it now if py_module_name not in py_module_cache: if module_info[2][2] == imp.PKG_DIRECTORY: # Read the __init__.py instead of the module file as this is # a python package normalized_name = py_module_name + ('__init__', ) if normalized_name not in py_module_names: normalized_path = os.path.join( os.path.join(module_info[1], '__init__.py')) normalized_data = _slurp(normalized_path) py_module_cache[normalized_name] = (normalized_data, normalized_path) normalized_modules.add(normalized_name) else: normalized_name = py_module_name if normalized_name not in py_module_names: normalized_path = module_info[1] normalized_data = module_info[0].read() module_info[0].close() py_module_cache[normalized_name] = (normalized_data, normalized_path) normalized_modules.add(normalized_name) # Make sure that all the packages that this module is a part of # are also added for i in range(1, len(py_module_name)): py_pkg_name = py_module_name[:-i] + ('__init__', ) if py_pkg_name not in py_module_names: pkg_dir_info = imp.find_module(py_pkg_name[-1], [ os.path.join(p, *py_pkg_name[:-1]) for p in module_utils_paths ]) normalized_modules.add(py_pkg_name) py_module_cache[py_pkg_name] = (_slurp(pkg_dir_info[1]), pkg_dir_info[1]) # FIXME: Currently the AnsiBallZ wrapper monkeypatches module args into a global # variable in basic.py. If a module doesn't import basic.py, then the AnsiBallZ wrapper will # traceback when it tries to monkypatch. So, for now, we have to unconditionally include # basic.py. # # In the future we need to change the wrapper to monkeypatch the args into a global variable in # their own, separate python module. That way we won't require basic.py. Modules which don't # want basic.py can import that instead. AnsibleModule will need to change to import the vars # from the separate python module and mirror the args into its global variable for backwards # compatibility. if ('basic', ) not in py_module_names: pkg_dir_info = imp.find_module('basic', module_utils_paths) normalized_modules.add(('basic', )) py_module_cache[('basic', )] = (_slurp(pkg_dir_info[1]), pkg_dir_info[1]) # End of AnsiballZ hack # # iterate through all of the ansible.module_utils* imports that we haven't # already checked for new imports # # set of modules that we haven't added to the zipfile unprocessed_py_module_names = normalized_modules.difference( py_module_names) for py_module_name in unprocessed_py_module_names: py_module_path = os.path.join(*py_module_name) py_module_file_name = '%s.py' % py_module_path zf.writestr(os.path.join("ansible/module_utils", py_module_file_name), py_module_cache[py_module_name][0]) display.vvvvv("Using module_utils file %s" % py_module_cache[py_module_name][1]) # Add the names of the files we're scheduling to examine in the loop to # py_module_names so that we don't re-examine them in the next pass # through recursive_finder() py_module_names.update(unprocessed_py_module_names) for py_module_file in unprocessed_py_module_names: recursive_finder(py_module_file, py_module_cache[py_module_file][0], py_module_names, py_module_cache, zf) # Save memory; the file won't have to be read again for this ansible module. del py_module_cache[py_module_file]
def parse_stmt(stmt): return ast.parse(stmt).body[0]
def read_version(): """Read version from httpretty/version.py without loading any files""" finder = VersionFinder() finder.visit(ast.parse(local_file('httpretty', '__init__.py'))) return finder.version
def core_module_docstrings(include_core=True, include_user=False, config=None, format="md"): """ Get docstrings for all core modules and user ones if requested returns a dict of {<module_name>: <docstring>} """ paths = {} docstrings = {} if include_core: for file in os.listdir(modules_directory()): if file.endswith(".py"): name = file[:-3] if name != "__init__": paths[name] = (os.path.join(modules_directory(), file), "core") if include_user: # include user modules for include_path in sorted(config["include_paths"]): include_path = os.path.abspath(include_path) + "/" if not os.path.isdir(include_path): continue for file in sorted(os.listdir(include_path)): if not file.endswith(".py"): continue name = file[:-3] paths[name] = (os.path.join(include_path, file), "user") for name in paths: path, module_type = paths[name] with open(path) as f: try: module = ast.parse(f.read()) except SyntaxError: # there is a syntax error so ignore module continue raw_docstring = ast.get_docstring(module) # prevent issue when no docstring exists if raw_docstring is None: continue # remove any sample outputs parts = re.split("^SAMPLE OUTPUT$", raw_docstring, flags=re.M) docstring = parts[0] if format == "md": docstring = [ d for d in _from_docstring_md( str(docstring).strip().split("\n")) ] elif format == "rst": docstring = [ d for d in _from_docstring_rst( str(docstring).strip().split("\n")) ] else: raise Exception("`md` and `rst` format supported only") docstrings[name] = docstring + ["\n"] return docstrings
def parse_expr(expr): return ast.parse(expr).body[0].value
def parse(self, string, **kwargs): ex = ast.parse(string, mode='eval') return self.handle(ex.body, **kwargs)
def make_single_statement(stmts): template = 'if 1: pass' t = ast.parse(template).body[0] t.body = stmts return t
def freevars_(source, env=None): return freevars(ast.parse(source, "", "eval"), env or [])
def _create_assign_lambda(s, o, lamb): assert isinstance( o, Signal ), "You can only assign(//=) a lambda function to a Wire/InPort/OutPort." srcs, line = inspect.getsourcelines(lamb) assert len( srcs ) == 1, "We can only handle single-line lambda connect right now." src = compiled_re.sub(r'\2', srcs[0]).lstrip(' ') root = ast.parse(src) assert isinstance(root, ast.Module) and len( root.body) == 1, "Invalid lambda (contact pymtl3 developer)" root = root.body[0] assert isinstance(root, ast.AugAssign) and isinstance( root.op, ast.FloorDiv) lhs, rhs = root.target, root.value # We expect the lambda to have no argument: # {'args': [], 'vararg': None, 'kwonlyargs': [], 'kw_defaults': [], 'kwarg': None, 'defaults': []} assert isinstance( rhs, ast.Lambda ) and not rhs.args.args and rhs.args.vararg is None, \ "The lambda shouldn't contain any argument." rhs = rhs.body # Compose a new and valid function based on the lambda's lhs and rhs # Note that we don't need to add those source code of closure var # assignment to linecache. To get the matching line number in the # error message, we set the line number of update block blk_name = "_lambda__{}".format(repr(o).replace(".", "_")) lambda_upblk = ast.FunctionDef( name=blk_name, args=ast.arguments(args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[ ast.Assign(targets=[lhs], value=rhs, lineno=2, col_offset=6) ], decorator_list=[], returns=None, lineno=1, col_offset=4, ) lambda_upblk_module = ast.Module(body=[lambda_upblk]) # Manually wrap the lambda upblk with a closure function that adds the # desired variables to the closure of `_lambda__*` # We construct AST for the following function to add free variables in the # closure of the lambda function to the closure of the generated lambda # update block. # # def closure( lambda_closure ): # <FreeVarName1> = lambda_closure[<Idx1>].cell_contents # <FreeVarName2> = lambda_closure[<Idx2>].cell_contents # ... # <FreeVarNameN> = lambda_closure[<IdxN>].cell_contents # def _lambda__<lambda_blk_name>(): # # the assignment statement appears here # return _lambda__<lambda_blk_name> new_root = ast.Module(body=[ ast.FunctionDef( name="closure", args=ast.arguments(args=[ ast.arg(arg="lambda_closure", annotation=None, lineno=1, col_offset=12) ], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[]), body=[ ast.Assign( targets=[ ast.Name(id=var, ctx=ast.Store(), lineno=1 + idx, col_offset=2) ], value=ast.Attribute( value=ast.Subscript( value=ast.Name( id='lambda_closure', ctx=ast.Load(), lineno=1 + idx, col_offset=5 + len(var), ), slice=ast.Index(value=ast.Num( n=idx, lineno=1 + idx, col_offset=19 + len(var), ), ), ctx=ast.Load(), lineno=1 + idx, col_offset=5 + len(var), ), attr='cell_contents', ctx=ast.Load(), lineno=1 + idx, col_offset=5 + len(var), ), lineno=1 + idx, col_offset=2, ) for idx, var in enumerate(lamb.__code__.co_freevars) ] + [lambda_upblk] + [ ast.Return( value=ast.Name( id=blk_name, ctx=ast.Load(), lineno=4 + len(lamb.__code__.co_freevars), col_offset=9, ), lineno=4 + len(lamb.__code__.co_freevars), col_offset=2, ) ], decorator_list=[], returns=None, lineno=1, col_offset=0, ) ]) # In Python 3 we need to supply a dict as local to get the newly # compiled function from closure. # Then `closure(lamb.__closure__)` returns the lambda update block with # the correct free variables in its closure. dict_local = {} exec(compile(new_root, blk_name, "exec"), lamb.__globals__, dict_local) blk = dict_local['closure'](lamb.__closure__) # Add the source code to linecache for the compiled function new_src = "def {}():\n {}\n".format(blk_name, src.replace("//=", "=")) linecache.cache[blk_name] = (len(new_src), None, new_src.splitlines(), blk_name) ComponentLevel1.update(s, blk) # This caching here does no caching because the block name contains # the signal name intentionally to avoid conflicts. With //= it is # more possible than normal update block to have conflicts: # if param == 1: s.out //= s.in_ + 1 # else: s.out //= s.out + 100 # Here these two blocks will implicity have the same name but they # have different contents based on different param. # So the cache call here is just to reuse the existing interface to # register the AST/src of the generated block for elaborate or passes # to use. s._cache_func_meta(blk, is_update_ff=False, given=("".join(srcs), lambda_upblk_module, line, inspect.getsourcefile(lamb))) return blk
def run(self): assert (isinstance(self.ast_root, ast.Module) and self.ast_root._fields == ("body", ) and len(self.ast_root.body) == 1 and isinstance(self.ast_root.body[0], ast.FunctionDef)) main_func_node = self.ast_root.body[0] assert hasattr(self.definition, "_gtscript_") # self.resolved_externals = self.resolve_external_symbols( # self.definition._gtscript_["nonlocals"], # self.definition._gtscript_["imported"], # self.external_context, # ) self.resolved_externals = self.definition._gtscript_["externals"] api_signature, fields_decls, parameter_decls = self.extract_arg_descriptors( ) # Inline constant values for name, value in self.resolved_externals.items(): if hasattr(value, "_gtscript_"): assert callable(value) func_node = ast.parse(gt_meta.get_ast(value)).body[0] local_context = self.resolve_external_symbols( value._gtscript_["nonlocals"], value._gtscript_["imported"], self.external_context, exhaustive=False, ) ValueInliner.apply(func_node, context=local_context) value._gtscript_["ast"] = func_node value._gtscript_["local_context"] = local_context local_context = self.resolve_external_symbols( self.definition._gtscript_["nonlocals"], self.definition._gtscript_["imported"], self.external_context, exhaustive=False, ) ValueInliner.apply(main_func_node, context=local_context) # Inline function calls CallInliner.apply(main_func_node, context=local_context) # Generate definition IR domain = gt_ir.Domain.LatLonGrid() computations = IRMaker( fields=fields_decls, parameters=parameter_decls, local_symbols={}, # Not used externals=local_context, domain=domain, extra_temp_decls={}, # Not used )(self.ast_root) self.definition_ir = gt_ir.StencilDefinition( name=self.main_name, domain=domain, api_signature=api_signature, api_fields=[ fields_decls[item.name] for item in api_signature if item.name in fields_decls ], parameters=[ parameter_decls[item.name] for item in api_signature if item.name in parameter_decls ], computations=computations, externals=self.resolved_externals, ) return self.definition_ir
def validate_(source): return validate_exp(ast.parse(source, mode="eval"))
def ast_node(expr: str) -> ast.AST: """Helper function to parse a string denoting an expression into an AST node""" # ast.parse returns "Module(body=[Node])" return ast.parse(expr).body[0]
def version(): """Return version string.""" with io.open('autopep8.py') as input_file: for line in input_file: if line.startswith('__version__'): return ast.parse(line).body[0].value.s
def dump(source, mode): """Dump source after parsing with mode""" a = ast.parse(source, mode=mode) return ast.dump(a, annotate_fields=True, include_attributes=False)
def __init__(self, source): self.names = list() self.imports = list() self.visit(ast.parse(source=source))