def AST_FunctionDataExtract(f, fullCode): ''' Extracts function data from AST ''' # Init FunctionData = { "language": "c", "name": '', "params": {}, "desc": "", "imports": [], "code": "", "return": {} } # Extract # Name FunctionData['name'] = f.decl.name # Desc # FunctionData['desc'] = [x.s for x in f.body] # Imports ImportsData = [] FunctionData['imports'] = ImportsData # Params FunctionData['params'] = [] if f.decl.type.args is not None: params = f.decl.type.args.params for i in range(len(params)): param = params[i] d = param.init d_dict = {"value": d} t = AST_GetType(param.type) paramData = { "name": param.name, "desc": "", "default_value": json.dumps(d_dict), "type": t } FunctionData['params'].append(paramData) # Code FunctionData['code'] = ast.get_source_segment(fullCode, f) # Return FunctionData['return'] = {"type": AST_GetType(f.decl)} return FunctionData
def lazyimport(line, cell): modules = {} for node in ast.iter_child_nodes(ast.parse(cell)): if isinstance(node, (ast.ImportFrom, ast.Import)): for name in [n.asname or n.name for n in node.names]: source = ast.get_source_segment(cell, node) setattr(builtins, name, DummyImport(name, source)) modules[name] = source else: raise RuntimeError(f'Unrecognized import: "{source}"') if 'debug' in line.lower(): print("Lazily imported packages are (name -> import statement):") print(json.dumps(modules, indent=2))
def skip_import(self, node: Union[ast.Import, ast.ImportFrom]) -> bool: if C.PY38_PLUS: lines = ast.get_source_segment(self.source, node).splitlines() for lineno, comment in self._comments.items(): with contextlib.suppress(IndexError): lines[lineno - node.lineno] += " " + comment source_segment = "".join(lines) # NOTE "\n".join(lines) else: source_segment = self.source.splitlines()[node.lineno - 1] return (bool( re.search( self.skip_comments_regex, source_segment, re.IGNORECASE, )) or self.any_import_error)
def safe_visit(self, node, *args, **kwargs): try: method(self, node, *args, **kwargs) except Exception as e: node_repr = ast.dump(node) if sys.version_info >= (3, 8) and self.source_code: node_repr = ast.get_source_segment(self.source_code, node) node_repr = "\n# ".join(node_repr.split("\n")) warning = f"# could not {action} on node:\n" \ f"# {node_repr}\n" \ f"# {str(e)}" logger.warning(warning, exc_info=True) self.warnings.append(warning) if call_generic_visit: self.generic_visit(node)
def get_raw_metrics(self, node): # astunparse.unparse() parses triple quote strings # a single quote strings. A single quote string is # interpreted as a sloc instead of a multi. # source_segement = unparse(node) source_segment = get_source_segment(self.code, node) raw_metrics = analyze(source_segment) raw_metrics_dict = raw_to_dict(raw_metrics) self.loc = raw_metrics_dict["loc"] self.lloc = raw_metrics_dict["lloc"] self.sloc = raw_metrics_dict["sloc"] self.comments = raw_metrics_dict["comments"] self.multi = raw_metrics_dict["multi"] self.blank = raw_metrics_dict["blank"] self.single_comments = raw_metrics_dict["single_comments"]
def get_value(v): if isinstance(v, ast.BinOp): a = get_value(v.left) b = get_value(v.right) return a elif isinstance(v, ast.Name): return loc.get(v.id) elif isinstance(v, ast.Call): args = [get_value(a) for a in v.args] func = loc.get(v.func.id, None) or glob.get(v.func.id, None) return func(*args) elif isinstance(v, ast.List): return [get_value(e) for e in v.elts] elif isinstance(v, ast.Constant): return v.value seg = get_source_segment(source, v) return eval(seg, glob, loc)
def __init__(self, code, lamb, context=None): if context is not None: self.lamb = lamb self.context = context childs = ast.walk(lamb.body) num_expr = 0 for child in childs: if (type(child) in expr): num_expr += 1 self.expressions = num_expr self.linecontent = ast.get_source_segment(code, lamb) else: self.id = code[0] self.arguments = code[3] self.expressions = code[4] self.line = code[5] self.context = code[6] self.linecontent = code[7]
def visit_Call(self, node): """ Instrument all function calls """ # pylint: disable=invalid-name ast.NodeTransformer.generic_visit(self, node) call_code = ast.get_source_segment(self.source_code, node) self.add_before_call_used_value_capturing_call(call_code, node, self.source_code) self.add_before_call_used_args_capturing_call(call_code, node, self.source_code) self.add_before_call_used_kwargs_capturing_call( call_code, node, self.source_code) instrumented_call_node = self.add_after_call_used_capturing( call_code, node, False) return instrumented_call_node
def execute_patched_func(original_func, execute_inspections_func, *args, **kwargs): """ Detects whether the function call comes directly from user code and decides whether to execute the original function or the patched variant. """ # Performance aspects: https://gist.github.com/JettJones/c236494013f22723c1822126df944b12 # CPython implementation detail: This function should be used for internal and specialized purposes only. # It is not guaranteed to exist in all implementations of Python. # inspect.getcurrentframe() also only does return `sys._getframe(1) if hasattr(sys, "_getframe") else None` # We can execute one hasattr check right at the beginning of the mlinspect execution caller_filename = sys._getframe(2).f_code.co_filename # pylint: disable=protected-access if caller_filename != singleton.source_code_path: result = original_func(*args, **kwargs) elif singleton.track_code_references: call_ast_node = ast.Call( lineno=singleton.lineno_next_call_or_subscript, col_offset=singleton.col_offset_next_call_or_subscript, end_lineno=singleton.end_lineno_next_call_or_subscript, end_col_offset=singleton.end_col_offset_next_call_or_subscript) caller_source_code = ast.get_source_segment(singleton.source_code, node=call_ast_node) caller_lineno = singleton.lineno_next_call_or_subscript op_id = singleton.get_next_op_id() caller_code_reference = CodeReference( singleton.lineno_next_call_or_subscript, singleton.col_offset_next_call_or_subscript, singleton.end_lineno_next_call_or_subscript, singleton.end_col_offset_next_call_or_subscript) result = execute_inspections_func(op_id, caller_filename, caller_lineno, caller_code_reference, caller_source_code) else: op_id = singleton.get_next_op_id() caller_lineno = sys._getframe(2).f_lineno # pylint: disable=protected-access result = execute_inspections_func(op_id, caller_filename, caller_lineno, None, None) return result
def add_before_call_used_value_capturing_call(call_code, node, all_source_code): """ When the method of some object is called, capture the value of the object before executing the method """ if hasattr(node.func, "value"): old_value_node = node.func.value value_code = ast.get_source_segment(all_source_code, old_value_node) new_value_node = ast.Call( func=ast.Name(id='before_call_used_value', ctx=ast.Load()), args=[ ast.Constant(n=False, kind=None), ast.Constant(n=call_code, kind=None), ast.Constant(n=value_code, kind=None), old_value_node, ast.Constant(n=node.lineno, kind=None), ast.Constant(n=node.col_offset, kind=None), ast.Constant(n=node.end_lineno, kind=None), ast.Constant(n=node.end_col_offset, kind=None) ], keywords=[]) node.func.value = new_value_node
def get_calls(func: ast.FunctionDef, code: str, end_calls_only: bool = False) -> List[str]: calls = [] if not end_calls_only: # return the entire call string for child in ast.walk(func): if isinstance(child, ast.Call): calls.append(ast.get_source_segment(code, child)) else: # return just the name of the function that was called for child in ast.walk(func): if isinstance(child, ast.Call): if isinstance(child.func, ast.Name): calls.append(child.func.id) elif isinstance(child.func, ast.Attribute): calls.append(child.func.attr) calls = list(set(calls)) calls.sort() return calls # easy way to remove duplicates
def test_AssignedVariablesCollector_single_assignment_separate_variable_from_instance_attribute( class_self_id: str, assignment_code: str, annotation_as_str: str, self_attributes: list, variables: list): # the assignment is the first line of the body assignment_ast: AST = parse(assignment_code).body[0] # assignment without annotation (multiple targets, but only one in these test cases) if annotation_as_str is None: annotation = None assert len(assignment_ast.targets) == 1, 'unit test consistency' assignment_target = assignment_ast.targets[0] # assignment with annotation (only one target) else: annotation = assignment_ast.annotation assert get_source_segment( assignment_code, annotation) == annotation_as_str, 'unit test consistency' assignment_target = assignment_ast.target assignment_collector = AssignedVariablesCollector(class_self_id, annotation) assignment_collector.visit(assignment_target) # detection of self attributes assert len(assignment_collector.self_attributes) == len(self_attributes) for self_attribute, (variable_id, variable_type_str) in zip( assignment_collector.self_attributes, self_attributes): assert_Variable(self_attribute, variable_id, variable_type_str, assignment_code) # detection of new variables occupying the memory scope assert len(assignment_collector.variables) == len(variables) for variable, (variable_id, variable_type_str) in zip(assignment_collector.variables, variables): assert_Variable(variable, variable_id, variable_type_str, assignment_code)
def add_before_call_used_kwargs_capturing_call(call_code, node, all_source_code): """ When a method is called, capture the keyword arguments of the method before executing it """ old_kwargs_nodes_ast = node.keywords # old_kwargs_nodes_ast = ast.List(node.keywords, ctx=ast.Load()) old_kwargs_code = ast.List([ ast.Constant(n=ast.get_source_segment(all_source_code, kwarg), kind=None) for kwarg in node.keywords ], ctx=ast.Load()) new_kwargs_node = ast.keyword(value=ast.Call( func=ast.Name(id='before_call_used_kwargs', ctx=ast.Load()), args=[ ast.Constant(n=False, kind=None), ast.Constant(n=call_code, kind=None), old_kwargs_code, ast.Constant(n=node.lineno, kind=None), ast.Constant(n=node.col_offset, kind=None), ast.Constant(n=node.end_lineno, kind=None), ast.Constant(n=node.end_col_offset, kind=None) ], keywords=old_kwargs_nodes_ast), arg=None) node.keywords = [new_kwargs_node]
def replace_fluent_alias(source, fluent_mapping): fluent_mapping = {a: b for a, b in fluent_mapping} new_src = source for _ in range(100): # 100 is a (random) big enough number replaced = False tree = ast.parse(new_src) for node in ast.walk(tree): if (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) and node.func.value.id == 'd2l' and node.func.attr in fluent_mapping): new_node = ast.Call( ast.Attribute(value=node.args[0], attr=fluent_mapping[node.func.attr]), node.args[1:], node.keywords) new_src = new_src.replace( ast.get_source_segment(new_src, node), astor.to_source(new_node).rstrip()) replaced = True break if not replaced: break return new_src
def node_to_readable_file_location(code, node, child_node=None): location = "" if isinstance(node.parent, ast.Module): # The next node up is the root, don't go higher. pass else: location += node_to_readable_file_location(code, node.parent, node) location += " > " if isinstance(node, ast.Module): raise Exception("We shouldn't see a Module") elif isinstance(node, ast.If): assert child_node if child_node in node.body: location += "if " + ast.get_source_segment(code, node.test) else: location += "else-of-if " + ast.get_source_segment(code, node.test) elif isinstance(node, ast.For): location += ("for " + ast.get_source_segment(code, node.target) + " in " + ast.get_source_segment(code, node.iter)) elif isinstance(node, ast.AugAssign): if isinstance(node.target, ast.Name): location += node.target.id else: location += ast.get_source_segment(code, node.target) elif isinstance(node, ast.Assign): # This assert would fire if we did e.g. some_sources = all_sources = [ ... ] assert len( node.targets) == 1, "Assignment node contains more than one target" if isinstance(node.targets[0], ast.Name): location += node.targets[0].id else: location += ast.get_source_segment(code, node.targets[0]) else: raise Exception("Got a node type I don't know how to handle: " + str(node)) return location
def fstring_parts(node, source): return { ast.get_source_segment(source, part.value) # noqa for part in node.values if isinstance(part, ast.FormattedValue) }
def flatten_into(self, output_file): code_lines = _read_file_contents_as_lines(self.script_path) code = "".join(code_lines) tree = ast.parse(code, self.script_path) # insert a backward pointer in every node for node in ast.walk(tree): for child in ast.iter_child_nodes(node): child.parent = node imports = [] # list of all imports to be performed names = [ ] # list of all symbolic names that might need to be substituted definitions = [ ] # list of all class and function definitions that will need to be exported assigns = [ ] # list of all assignments to variables that will need to be exported base_path = os.path.dirname(self.script_path) for node in ast.walk(tree): if isinstance(node, ast.Import): if node.col_offset == 0: for name in node.names: imports.append( _ImportStatement( name.name, name.asname, node.lineno - 1, base_path, self.other_base_path, )) elif isinstance(node, ast.Name): symbol = node.id parent = node.parent end_offset = node.end_col_offset names.append( _Symbol(symbol, node.lineno - 1, node.col_offset, end_offset)) while isinstance(parent, ast.Attribute): symbol = symbol + "." + parent.attr end_offset = parent.end_col_offset names.append( _Symbol(symbol, node.lineno - 1, node.col_offset, end_offset)) parent = parent.parent elif not self.is_script(): # only look for things to export if this is not the top level script if isinstance(node, ast.ClassDef) or isinstance( node, ast.FunctionDef): # don't record class/function definitions made inside other definitions if node.col_offset == 0: name_offset = 6 if isinstance(node, ast.ClassDef) else 4 definitions.append( _DefStatement(node.name, self.module_name)) # the name of the class/function is not an AST 'name' so it won't # be added to the list to substitute automatically - add it here names.append( _Symbol( node.name, node.lineno - 1, name_offset, name_offset + len(node.name), )) elif isinstance(node, ast.Assign): if node.col_offset == 0: assigns.append( _AssignmentStatement( ast.get_source_segment(code, node), self.module_name)) for an_import in imports: previous_import = self.imports_done.get(an_import.module_path) if previous_import is None: an_import.flatten(into=output_file, imports_done=self.imports_done) self.imports_done[an_import.module_path] = an_import else: if previous_import.alias != an_import.alias: raise ImportError( f"Module '{an_import.module_path}' has already been inlined using alias '{previous_import.alias}' so cannot be inlined using alias '{an_import.alias}'", name=an_import.module_path, ) # always load the exported symbols into the current context, even if the import has already been done self.local_symbol_mappings.update( self.imports_done[an_import.module_path].exports) for a_definition in definitions: self.exported_symbol_mappings.update( a_definition.get_exported_symbol_mapping()) self.local_symbol_mappings.update( a_definition.get_local_symbol_mapping()) for an_assign in assigns: self.exported_symbol_mappings.update( an_assign.get_exported_symbol_mapping()) self.local_symbol_mappings.update( an_assign.get_local_symbol_mapping()) # sort the symbols by line number and then start column number names.sort(key=lambda x: (x.line_number * 10000) + x.start_col_offset) current_line_num = 0 import_lines = [x.line_number for x in imports] # copy all the lines up to next 'name' line while len(names) > 0 and current_line_num < len(code_lines): while current_line_num < names[0].line_number: if current_line_num not in import_lines: self.write_with_reference(output_file, code_lines[current_line_num], current_line_num) current_line_num += 1 # replace 'names' in the line up_to = 0 result = "" current_line = code_lines[current_line_num] while len(names) > 0 and current_line_num == names[0].line_number: next_symbol = names[0] del names[0] result += current_line[up_to:next_symbol.start_col_offset] up_to = max(up_to, next_symbol.start_col_offset) if next_symbol.name in self.local_symbol_mappings: result += self.local_symbol_mappings[next_symbol.name] up_to = next_symbol.end_col_offset elif next_symbol.name in self.exported_symbol_mappings: result += self.exported_symbol_mappings[next_symbol.name] up_to = next_symbol.end_col_offset elif len(names) > 0 and current_line_num == names[ 0].line_number and next_symbol.start_col_offset == names[ 0].start_col_offset: # There's another overlapping name that might match, so skip this one pass else: result += current_line[up_to:next_symbol.end_col_offset] up_to = next_symbol.end_col_offset result += current_line[up_to:] # write the updated line self.write_with_reference(output_file, result, current_line_num) current_line_num += 1 # copy any remaining lines while current_line_num < len(code_lines): self.write_with_reference(output_file, code_lines[current_line_num], current_line_num) current_line_num += 1 return self.exported_symbol_mappings
def get_exports(source, root): all_exports = [node for node in ast.walk(root) if isinstance(node, ast.AugAssign) and ( (isinstance(node.target, ast.Name) and node.target.id == "EXPORTS") or (isinstance(node.target, ast.Attribute) and "EXPORTS" in ast.get_source_segment(source, node)) ) ] all_exports.extend([node for node in ast.walk(root) if isinstance(node, ast.Assign) and ( (isinstance(node.targets[0], ast.Name) and node.targets[0].id in ["EXPORTS", "h_and_cpp"]) or (isinstance(node.targets[0], ast.Attribute) and "EXPORTS" in ast.get_source_segment(source, node)) ) ]) all_exports.extend([node for node in ast.walk(root) if isinstance(node, ast.Assign) and ( (isinstance(node.targets[0], ast.Subscript) and isinstance(node.targets[0].value, ast.Name) and node.targets[0].value.id == "EXPORTS") and (node.targets[0].slice.value.value == "double-conversion") # Limit it to just this one case to be safe ) ]) linux_exports = [] # For each export, see if it is contained within an if statement # whose conditional includes OS_ARCH or OS_TARGET and if so, only include it if it's Linux for e in all_exports: e_source = ast.get_source_segment(source, e) #if "XXXXXXX" in e_source: # pdb.set_trace() if e.parent == root: linux_exports.append((e, e_source)) continue node = e should_include = True while node.parent != root: code = ast.get_source_segment(source, node.parent) # This test will fail for a conditional like "OS_ARCH != Linux" # but presently we don't have any of those... if isinstance(node.parent, ast.If): conditional = ast.get_source_segment(source, node.parent.test) # We want to ensure that the node is excluded if it is directly underneath a conditional # that looks like it is for another platform if ("OS_ARCH" in conditional or "OS_TARGET" in conditional) and "Linux" not in conditional: if e in node.parent.body: should_include = False if "MOZ_WIDGET_TOOLKIT" in conditional and "gtk" not in conditional: if e in node.parent.body: should_include = False if "MOZ_BUILD_APP" in conditional and '== "memory"' in conditional: if e in node.parent.body: should_include = False node = node.parent if should_include: linux_exports.append((e, e_source)) return linux_exports
def process_calls(self, source: str): for call in self.expression_calls: for index, call_arg in enumerate(call['args']): if not isinstance(call_arg, _ast.Constant): source_call = ast.get_source_segment(source, call_arg) call['args'][index] = source_call
def fetch(filename, **loc_data): with tokenize.open(DATA_PATH / filename) as file: source = file.read() loc_node = LocationNode(**loc_data) return ast.get_source_segment(source, loc_node, padded=True)
def modify( dir_: Path, paths: List[Path], default_level: str = 'info', accept_all: bool = False, context_lines: int = 13, comment_sep: str = ' \\ ', **_kwargs, ) -> None: for path in paths: fpath = str(path.relative_to(dir_)) text = path.read_text() lines = text.splitlines() edited = False # Is there an accepted edit to write to disk? tree = ast.parse(text) # Make backlinks in order to later determine the context (function or class name) # of the print statement for node in ast.walk(tree): for child in ast.iter_child_nodes(node): child.parent = node # Find all the nodes in the AST with a print statement, print_nodes = [ node for node in ast.walk(tree) if (isinstance(node, ast.Call) and getattr(node.func, 'id', None) == 'print') ] if not print_nodes: continue # Fetch comments lineix_to_comment = dict() # Line index to comment with path.open() as fil: for toktype, tok, (start_rowno, _), _, _ in tokenize.generate_tokens( fil.readline): if toktype == tokenize.COMMENT: lineix_to_comment[start_rowno - 1] = re.sub( r'#\s*', '', tok) print_statements = [] for node in print_nodes: has_vars = False terms = [] # (term, is_var) for arg in node.args: term = ast.get_source_segment(text, arg) if isinstance(arg, ast.JoinedStr): # Is an f-string has_vars = True n_quotes = len( re.match(r'^f("""|\'\'\'|"|\')', term).group(1)) # Remove 'f' and quotes from string term = term[1 + n_quotes:-n_quotes] # Escape newlines etc term = term.encode('unicode_escape').decode('utf-8') terms.append((term, True)) elif term.startswith('"') or term.startswith("'"): term = arg.value.encode('unicode_escape').decode('utf-8') terms.append((term, False)) else: terms.append(('{' + term + '}', True)) has_vars = True # Escape {} in non-f strings if has_vars: for i, (term, is_var) in enumerate(terms): if not is_var: terms[i] = (term.replace('{', '{{').replace('}', '}}'), is_var) # Use custom separator from print statement sep = ' ' for kw in node.keywords: if kw.arg == 'sep': sep = getattr( kw.value, "id", getattr(kw.value, "value", ' ')).encode('unicode_escape').decode('utf-8') arg_line = sep.join(t[0] for t in terms) # Line numbers are 1-indexed lineix, end_lineix = node.lineno - 1, node.end_lineno - 1 # Find the whitespace predecing 'print' in the orginal statement old_line = lines[lineix] whitespace = re.match(r'(\s*)print', old_line).group(1) comments = [ lineix_to_comment[i] for i in range(lineix, end_lineix + 1) if i in lineix_to_comment ] comment = comment_sep.join(comments) or None source_context = node while not isinstance(source_context, (ast.AsyncFunctionDef, ast.ClassDef, ast.FunctionDef, ast.Module)): source_context = source_context.parent if isinstance(source_context, ast.Module): source_context = '' else: source_context = '/' + source_context.name print_statements.append( PrintStatement(lineix, end_lineix, whitespace, arg_line, has_vars, comment, source_context)) # Find out if logging is already imported, and if so, as what. has_logging_imported = False import_nodes = [] logging_asname = 'logging' for node in ast.iter_child_nodes(tree): if isinstance(node, ast.Import): import_nodes.append(node) if node.names[0].name == 'logging': has_logging_imported = True logging_asname = node.names[0].asname or 'logging' first_import_node = min(import_nodes, default=None, key=attrgetter('lineno')) first_import_lineno = first_import_node.lineno or 1 def get_line(stmt: PrintStatement, level: str) -> str: """ Construct the full source code line with whitespace, logging statement and optionally a comment """ f_string = 'f' if stmt.f_string else '' comment = f' # {stmt.comment}' if stmt.comment else '' return stmt.whitespace + f'{logging_asname}.{level}({f_string}"' \ + stmt.arg_line + '")' + comment print_statements.sort(key=lambda st: st.lineix) # Multiline print statements are always squashed to a single-line statement, # so there might be superfluous lines to delete to_del_lineixs = [] n_changes = 0 for stmt in print_statements: line = get_line(stmt, default_level) if not accept_all: clear_terminal() print( Bcolor.HEADER, f"{fpath}{stmt.source_context}:" f"{stmt.lineix+1}-{stmt.end_lineix+1}", Bcolor.ENDC) print() print_context(lines, stmt.lineix, stmt.end_lineix, line, context_lines) print() inp = None while inp not in [ '', 'y', 'n', 'A', 'i', 'w', 'e', 'c', 'x', 'q' ]: inp = input(Bcolor.OKCYAN + "Accept change? (" f"y = yes ({default_level}) [default], " "n = no, " "A = yes to all, " "i = yes (info), " "w = yes (warning), " "e = yes (error), " "c = yes (critical), " "x = yes (exception), " "q = quit): " + Bcolor.ENDC) if inp in ('q', 'Q'): sys.exit(0) elif inp in ['i', 'w', 'e', 'c', 'x']: level = levels[inp] line = get_line(stmt, level) elif inp == 'A': accept_all = True if accept_all or inp in ['', 'y', 'A', 'i', 'w', 'e', 'c', 'e']: lines[stmt.lineix] = line edited = True to_del_lineixs.extend( range(stmt.lineix + 1, stmt.end_lineix + 1)) n_changes += 1 for index in sorted(to_del_lineixs, reverse=True): del lines[index] if edited: # insert import statement, if necessacy: if not has_logging_imported: lines.insert(first_import_lineno - 1, 'import logging') print("Added `import logging`. And ", end=' ') path.write_text('\n'.join(lines)) print(f"Wrote {n_changes} changes to {fpath}")
sys.exit(1) path = sys.argv[1] print("# %s" % path) code = open(path, 'r').read() tree = ast.parse(code) found_ConanFile = False for node in ast.walk(tree): if isinstance(node, ast.ClassDef): baseIdList = [b.id for b in node.bases if hasattr(b, "id")] # class must inherit from class ConanFile if "ConanFile" not in baseIdList: continue print("class %s (%s):" % (node.name, ", ".join(baseIdList))) found_ConanFile = True if isinstance(node, ast.FunctionDef) and node.name == 'source': #print(node.name) print(ast.get_source_segment(code, node)) """ for child in node.body: print("child:") print(ast.get_source_segment(code, child)) """ if found_ConanFile == False: print("error: not implemented: no `class SomeClass(ConanFile)` was found")
def get_source_segment(self, *args): return Code(ast.get_source_segment(self._src, args[0]))
def __waiting(self, ttype, tstring, lineno): opts = self.__options # Do docstring extractions, if enabled if opts.docstrings: # Module docstring? if self.__fresh_module: if ttype == tokenize.STRING and is_literal_string(tstring): self.__add_entry( safe_eval(tstring), lineno, is_docstring=True ) self.__fresh_module = False elif ttype not in ( tokenize.ENCODING, tokenize.COMMENT, tokenize.NL, ): self.__fresh_module = False return # Class or func/method docstring? if ttype == tokenize.NAME and tstring in ('class', 'def'): self.__state = self.__suite_seen return if opts.cmd_docstrings: if ttype == tokenize.OP and tstring == '@': self.__state = self.__decorator_seen return elif ttype == tokenize.NAME and tstring == 'class': self.__state = self.__class_seen return if ttype == tokenize.NAME and tstring in opts.keywords: self.__state = self.__keyword_seen return if ttype == tokenize.COMMENT: if self.__comment and lineno == self.__comment[1] + 1: self.__comment[0] += tstring[1:] self.__comment[1] = lineno else: comment = tstring[1:].strip().split(' ', 1) if comment[0].rstrip(':') in opts.c_keywords: self.__comment = [comment[1], lineno] if ttype == tokenize.STRING: maybe_fstring = ast.parse(tstring, mode='eval').body if not isinstance(maybe_fstring, ast.JoinedStr): return for value in filter( lambda node: isinstance(node, ast.FormattedValue), maybe_fstring.values, ): for call in filter( lambda node: isinstance(node, ast.Call), ast.walk(value) ): func = call.func if isinstance(func, ast.Name): func_name = func.id elif isinstance(func, ast.Attribute): func_name = func.attr else: continue if func_name not in opts.keywords: continue if len(call.args) != 1 and not opts.multiple_args: print( ( '*** %(file)s:%(lineno)s: Seen unexpected amount of' ' positional arguments in gettext call: %(source_segment)s' ) % { 'source_segment': ast.get_source_segment( tstring, call ) or tstring, 'file': self.__cur_infile, 'lineno': lineno, }, file=sys.stderr, ) continue if call.keywords and not opts.multiple_args: print( ( '*** %(file)s:%(lineno)s: Seen unexpected keyword arguments' ' in gettext call: %(source_segment)s' ) % { 'source_segment': ast.get_source_segment( tstring, call ) or tstring, 'file': self.__cur_infile, 'lineno': lineno, }, file=sys.stderr, ) continue arg = call.args[0] if not isinstance(arg, ast.Constant): print( ( '*** %(file)s:%(lineno)s: Seen unexpected argument type' ' in gettext call: %(source_segment)s' ) % { 'source_segment': ast.get_source_segment( tstring, call ) or tstring, 'file': self.__cur_infile, 'lineno': lineno, }, file=sys.stderr, ) continue if isinstance(arg.value, str): self.__add_entry(arg.value, lineno)
def __waiting(self, ttype, tstring, lineno): opts = self.__options # Do docstring extractions, if enabled if opts.docstrings and not opts.nodocstrings.get(self.__curfile): # module docstring? if self.__freshmodule: if ttype == tokenize.STRING and is_literal_string(tstring): self.__addentry(safe_eval(tstring), lineno, isdocstring=1) self.__freshmodule = 0 elif ttype not in (tokenize.COMMENT, tokenize.NL): self.__freshmodule = 0 return # class or func/method docstring? if ttype == tokenize.NAME and tstring in ('class', 'def'): self.__state = self.__suiteseen return if ttype == tokenize.NAME and tstring in opts.keywords: self.__state = self.__keywordseen return if ttype == tokenize.STRING: maybe_fstring = ast.parse(tstring, mode='eval').body if not isinstance(maybe_fstring, ast.JoinedStr): return for value in filter( lambda node: isinstance(node, ast.FormattedValue), maybe_fstring.values): for call in filter(lambda node: isinstance(node, ast.Call), ast.walk(value)): func = call.func if isinstance(func, ast.Name): func_name = func.id elif isinstance(func, ast.Attribute): func_name = func.attr else: continue if func_name not in opts.keywords: continue if len(call.args) != 1: print(_( '*** %(file)s:%(lineno)s: Seen unexpected amount of' ' positional arguments in gettext call: %(source_segment)s' ) % { 'source_segment': ast.get_source_segment(tstring, call) or tstring, 'file': self.__curfile, 'lineno': lineno }, file=sys.stderr) continue if call.keywords: print(_( '*** %(file)s:%(lineno)s: Seen unexpected keyword arguments' ' in gettext call: %(source_segment)s') % { 'source_segment': ast.get_source_segment(tstring, call) or tstring, 'file': self.__curfile, 'lineno': lineno }, file=sys.stderr) continue arg = call.args[0] if not isinstance(arg, ast.Constant): print(_( '*** %(file)s:%(lineno)s: Seen unexpected argument type' ' in gettext call: %(source_segment)s') % { 'source_segment': ast.get_source_segment(tstring, call) or tstring, 'file': self.__curfile, 'lineno': lineno }, file=sys.stderr) continue if isinstance(arg.value, str): self.__addentry(arg.value, lineno)
def update_event(self, inp=-1): self.set_output_val( 0, ast.get_source_segment(self.input(0), self.input(1)))
def visit_Assign(self, node): expression = AssignExpression([t.id for t in node.targets], ast.get_source_segment(self.__sourcecode, node, padded=True)) self.__current = expression self.__expressions.append(self.__current) self.generic_visit(node)
def segment(self): return ast.get_source_segment(self.source.text, self.node, padded=False)
def source(self, node): return ast.get_source_segment(self.code, node).strip('\n')
def AST_FunctionDataExtract(f, fullCode): ''' Extracts function data from AST ''' # Init FunctionData = {"language": "python", "name": '', "params": '', "desc": "", "imports": [], "code": ""} # Extract # Name FunctionData['name'] = f.name # Desc # FunctionData['desc'] = [x.s for x in f.body] # Imports ImportsData = [] Imports = [x for x in f.body if isinstance(x, ast.Import)] FromImports = [x for x in f.body if isinstance(x, ast.ImportFrom)] for impG in Imports: impsDicts = [] for imp in impG.names: name = imp.name.split(".")[-1] module = imp.name.rstrip(name).rstrip(".") impData = { "name": name, "dependency_path": module.split(".") } impsDicts.append(impData) ImportsData.extend(impsDicts) for impG in FromImports: impsDicts = [] for imp in impG.names: name = imp.name.split(".")[-1] module = ".".join([impG.module, imp.name.rstrip(name).rstrip(".")]).rstrip(".") impData = { "name": name, "dependency_path": module.split(".") } impsDicts.append(impData) ImportsData.extend(impsDicts) FunctionData['imports'] = ImportsData # Params FunctionData['params'] = [] defaultCount = len(f.args.defaults) argsCount = len(f.args.args) for i in range(len(f.args.args)): a = f.args.args[i] d = None if i >= argsCount - defaultCount: d = f.args.defaults[i - (argsCount - defaultCount)].value d_dict = {"value": d} t = a.annotation.id if a.annotation is not None else None paramData = { "name": a.arg, "desc": "", "default_value": json.dumps(d_dict), "type": t } FunctionData['params'].append(paramData) # Code FunctionData['code'] = ast.get_source_segment(fullCode, f) return FunctionData