def check(source, encoding, exception=None, matches=True): encoded = source.encode(encoding) if exception: with self.assertRaises(exception): Source.decode_source(encoded) else: decoded = Source.decode_source(encoded) if matches: self.assertEqual(decoded, source) else: self.assertNotEqual(decoded, source)
def trace_func(frame, event, _arg): filename = frame.f_code.co_filename if event == "call": if include_file(filename): return trace_func elif event == "line": lineno = frame.f_lineno queues[filename].append(lineno) totals[filename][lineno] += 1 Source.lazycache(frame)
def tester(arg, returns=None): frame = inspect.currentframe().f_back Source.lazycache(frame) call = Source.executing(frame).node result = eval( compile(ast.Expression(only(call.args)), '<>', 'eval'), frame.f_globals, frame.f_locals, ) assert result == result, (result, arg) if returns is None: return arg return returns
def file_table_context(): filename = request.args['filename'] source = Source.for_filename(filename) queue = queues[filename] highlighted = highlight_ranges(source, frames_matching(filename)) highlighted = highlight_python_and_ranges(highlighted) highlighted_lines = list(enumerate(highlighted.splitlines())) counters = [queue_counter(queue, 2**i) for i in range(levels + 1)] ratios = [[ counter[i + 1] / min(2**c, len(queue) or 1) * (c + 1) / levels for c, counter in enumerate(counters) ] for i, _ in highlighted_lines] max_ratio = max(map(max, ratios)) or 1 rows = [( i + 1, totals[filename][i + 1] or '', reversed( [int(round(ratio / max_ratio * 100)) for ratio in ratios[i]]), line, ) for i, line in highlighted_lines] return dict( rows=rows, zip=zip, lightnesses=lightnesses, filename=filename, highlighted=highlighted, )
def test_invalid_python(self): path = os.path.join( os.path.dirname(__file__), 'not_code.txt', ) source = Source.for_filename(path) self.assertIsNone(source.tree)
def match(self, frame_no: int, frameinfos: List[inspect.FrameInfo]) -> bool: frame = frameinfos[frame_no].frame # module is None, check qualname only return fnmatch( Source.for_frame(frame).code_qualname(frame.f_code), self.qualname)
def get_node(frame: int, ignore: Optional[IgnoreType] = None, raise_exc: bool = True) -> Optional[ast.AST]: """Try to get node from the executing object. This can fail when a frame is failed to retrieve. One case should be when python code is executed in R pacakge `reticulate`, where only first frame is kept. When the node can not be retrieved, try to return the first statement. """ from .ignore import IgnoreList ignore = IgnoreList.create(ignore) try: frame = ignore.get_frame(frame) except VarnameRetrievingError: return None exect = Source.executing(frame) if exect.node: return exect.node if exect.source.text and exect.source.tree and raise_exc: raise VarnameRetrievingError( "Couldn't retrieve the call node. " "This may happen if you're using some other AST magic at the " "same time, such as pytest, ipython, macropy, or birdseye.") return None
def argnode_source(source: Source, node: ast.AST, vars_only: bool) -> Union[str, ast.AST]: """Get the source of an argument node Args: source: The executing source object node: The node to get the source from vars_only: Whether only allow variables and attributes Returns: The source of the node (node.id for ast.Name, node.attr for ast.Attribute). Or the node itself if the source cannot be fetched. """ if isinstance(node, ast.Constant): return repr(node.value) if sys.version_info < (3, 9): # pragma: no cover if isinstance(node, ast.Index): node = node.value if isinstance(node, ast.Num): return repr(node.n) if isinstance(node, (ast.Bytes, ast.Str)): return repr(node.s) if isinstance(node, ast.NameConstant): return repr(node.value) if vars_only: return (node.id if isinstance(node, ast.Name) else node.attr if isinstance(node, ast.Attribute) else node) # requires asttokens return source.asttokens().get_text(node)
def highlight_ranges(source, frames): text = source.text ranges = set() for frame in frames: executing = Source.executing(frame) if executing.node: text_range = executing.text_range() ranges.add(text_range) positions = [] for start, end in ranges: positions.append((start, open_sentinel)) positions.append((end, close_sentinel)) while True: start = text.find('\n', start + 1, end) if start == -1: break positions.append((start, close_sentinel)) positions.append((start + 1, open_sentinel)) # This just makes the loop below simpler positions.append((len(text), '')) positions.sort() parts = [] start = 0 for position, part in positions: parts.append(text[start:position]) parts.append(part) start = position return ''.join(parts)
def test_traceback(self): try: 134895 / 0 except: tb = sys.exc_info()[2] ex = Source.executing(tb) self.assertTrue(isinstance(ex.node, ast.BinOp)) self.assertEqual(ex.text(), "134895 / 0")
def _(evt): # Patch to ensure the executing module's cache is invalidated whenever # a source file is changed. cache = Source._class_local("__source_cache", {}) filename = evt.codefile.filename if filename in cache: del cache[filename] linecache.checkcache(filename)
def test_executing_methods(self): frame = inspect.currentframe() executing = Source.executing(frame) self.assertEqual(executing.code_qualname(), 'TestStuff.test_executing_methods') text = 'Source.executing(frame)' self.assertEqual(executing.text(), text) start, end = executing.text_range() self.assertEqual(executing.source.text[start:end], text)
def calling_env(funtype: str) -> Any: """Checking how the function is called: 1. PIPING_VERB: It is a verb that is piped directed. ie. data >> verb(...) 2. PIPING: It is a function called as (part of) the argument of a piping verb. ie.: >>> data >> verb(func(...)) Note that `func` here could also be a verb. When a function is called inside a lambda body, it should not be counted in this situation: >>> data >> verb(lambda: func(...)) In this case, func should be called as normal function. This function should return `None` 3. FUNC_ARG: It is an argument of any function call 4. None: None of the above situation fits This function should be only called inside register_*.wrapper """ if options.assume_all_piping: return (CallingEnvs.PIPING_VERB if funtype == 'Verb' else CallingEnvs.PIPING) # frame 1: register_*.wrapper # frame 2: func(...) frame = sys._getframe(2) my_node = Source.executing(frame).node if not my_node and options.warn_astnode_failure: warnings.warn( "Failed to fetch the node calling the function, " "please call the verbs regularly instead of `data >> verb(...)`.") return None piping_verb_node = _get_piping_verb_node(my_node) if piping_verb_node is my_node and piping_verb_node is not None: return CallingEnvs.PIPING_VERB if _is_piping_verb_argument_node(my_node, piping_verb_node): return CallingEnvs.PIPING parent_call_node = _argument_node_of(my_node) if parent_call_node is None: return None # check if parent call node is a function registered by # register_verb/register_func evaluator = Evaluator.from_frame(frame) try: func = evaluator[parent_call_node.func] except CannotEval: # pragma: no cover return None if functype(func) != "plain": return CallingEnvs.PIPING return None
def gen(): frame = current_frame() while frame: code = frame.f_code filename = code.co_filename name = Source.for_frame(frame).code_qualname(code) yield (filename, frame.f_lineno, name, highlight_stack_frame(frame), include_file(filename)) frame = frame.f_back
def __call__(self, *args, **kwargs): frame = sys._getframe(1) while frame.f_code in self._excluded_codes: frame = frame.f_back executing = Source.executing(frame) assert executing.node, "Failed to find call node" return self.at(FrameInfo(executing))(*args, **kwargs)
def _post_init(self) -> None: attach_ignore_id_to_module(self.module) # check uniqueness of qualname modfile = getattr(self.module, '__file__', None) if modfile is not None: check_qualname_by_source( Source.for_filename(modfile, self.module.__dict__), self.module.__name__, self.qualname)
def check_filename(self, filename): print(filename) source = Source.for_filename(filename) if PY3: code = compile(source.text, filename, "exec", dont_inherit=True) for subcode, qualname in find_qualnames(code): if not qualname.endswith(">"): code_qualname = source.code_qualname(subcode) self.assertEqual(code_qualname, qualname) nodes = defaultdict(list) for node in ast.walk(source.tree): if isinstance(node, ( ast.UnaryOp, ast.BinOp, ast.Subscript, ast.Call, ast.Compare, ast.Attribute )): nodes[node] = [] code = compile(source.tree, source.filename, 'exec') result = list(self.check_code(code, nodes)) if not re.search(r'^\s*if 0(:| and )', source.text, re.MULTILINE): for node, values in nodes.items(): if is_unary_not(node): continue if isinstance(getattr(node, 'ctx', None), (ast.Store, ast.Del)): assert not values continue if isinstance(node, ast.Compare): if len(node.ops) > 1: assert not values continue if is_unary_not(node.parent) and isinstance(node.ops[0], (ast.In, ast.Is)): continue if is_literal(node): continue if sys.version_info >= (3, 9) and in_finally(node): correct = len(values) > 1 else: correct = len(values) == 1 if not correct: print(source.text, '---', node_string(source, node), node.lineno, len(values), correct, values, file=sys.stderr, sep='\n') self.fail() return result
def check_code(self, code, nodes): linestarts = dict(dis.findlinestarts(code)) instructions = get_instructions(code) lineno = None for inst in instructions: if time.time() - self.start_time > 45 * 60: # Avoid travis time limit of 50 minutes raise TimeOut lineno = linestarts.get(inst.offset, lineno) if not inst.opname.startswith(( 'BINARY_', 'UNARY_', 'LOAD_ATTR', 'LOAD_METHOD', 'LOOKUP_METHOD', 'SLICE+', 'COMPARE_OP', 'CALL_', 'IS_OP', 'CONTAINS_OP', )): continue frame = C() frame.f_lasti = inst.offset frame.f_code = code frame.f_globals = globals() frame.f_lineno = lineno source = Source.for_frame(frame) node = None try: try: node = Source.executing(frame).node except Exception: if inst.opname.startswith(('COMPARE_OP', 'CALL_')): continue if isinstance(only(source.statements_at_line(lineno)), (ast.AugAssign, ast.Import)): continue raise except Exception: print(source.text, lineno, inst, node and ast.dump(node), code, file=sys.stderr, sep='\n') raise nodes[node].append((inst, frame.__dict__)) yield [inst.opname, node_string(source, node)] for const in code.co_consts: if isinstance(const, type(code)): for x in self.check_code(const, nodes): yield x
def test_many_calls(self): node = None start = time.time() for i in range(10000): new_node = Source.executing(inspect.currentframe()).node if node is None: node = new_node else: self.assertIs(node, new_node) self.assertLess(time.time() - start, 1)
def assert_name_error(self): try: yield except NameError as e: tb = sys.exc_info()[2] ex = Source.executing(tb.tb_next) self.assertEqual(type(ex.node), ast.Name) self.assertIn(ex.node.id, str(e)) self.assertEqual(ex.text(), ex.node.id) else: self.fail("NameError not raised")
def highlight_stack_frame(frame): executing = Source.executing(frame) node = executing.node source = executing.source if node: source.asttokens() start = node.first_token.start[0] end = node.last_token.end[0] else: start = end = frame.f_lineno highlighted = '\n'.join(highlight_ranges(source, [frame]).splitlines()[start - 1:end]) return highlight_python_and_ranges(highlighted)
def match(self, frame_no: int, frameinfos: List[inspect.FrameInfo]) -> bool: frame = frameinfos[frame_no].frame frame_filename = path.realpath(frame.f_code.co_filename) preset_filename = path.realpath(self.filename) # return earlier to avoid qualname uniqueness check if frame_filename != preset_filename: return False source = Source.for_frame(frame) check_qualname_by_source(source, self.filename, self.qualname) return fnmatch(source.code_qualname(frame.f_code), self.qualname)
def test_ignore_module_qualname_no_source(tmp_path): module = module_from_source( 'ignore_module_qualname_no_source', """ def bar(): return 1 """, tmp_path) source = Source.for_filename(module.__file__) # simulate when source is not available # no way to check uniqueness of qualname source.tree = None def foo(): return varname(ignore=(module, 'bar')) f = foo()
def get_node_by_frame(frame: FrameType, raise_exc: bool = True) -> Optional[ast.AST]: """Get the node by frame, raise errors if possible""" exect = Source.executing(frame) if exect.node: return exect.node if exect.source.text and exect.source.tree and raise_exc: raise VarnameRetrievingError( "Couldn't retrieve the call node. " "This may happen if you're using some other AST magic at the " "same time, such as pytest, ipython, macropy, or birdseye.") return None
def get_node_by_frame(frame: FrameType, raise_exc: bool = True) -> ast.AST: """Get the node by frame, raise errors if possible""" exect = Source.executing(frame) if exect.node: # attach the frame for better exception message # (ie. where ImproperUseError happens) exect.node.__frame__ = frame return exect.node if exect.source.text and exect.source.tree and raise_exc: raise VarnameRetrievingError( "Couldn't retrieve the call node. " "This may happen if you're using some other AST magic at the " "same time, such as pytest, ipython, macropy, or birdseye.") return None
def match(self, frame_no: int, frameinfos: List[inspect.FrameInfo]) -> bool: frame = frameinfos[frame_no].frame module = cached_getmodule(frame.f_code) # Return earlier to avoid qualname uniqueness check if module and module != self.module: return False if (not module and not frame_matches_module_by_ignore_id(frame, self.module)): return False source = Source.for_frame(frame) check_qualname_by_source(source, self.module.__name__, self.qualname) return fnmatch(source.code_qualname(frame.f_code), self.qualname)
def _getattr(name: str): # Using get_option("import_names_conflict") to get the value # instead of `import_names_conflict` # OPTIONS changed in lifetime opt_maybe_changed = get_option("import_names_conflict") if (name == "__path__" or name not in conflict_names or opt_maybe_changed == "underscore_suffixed"): raise AttributeError # from ... import xxx if (name not in WARNED and opt_maybe_changed == "warn" and not Source.executing(sys._getframe(1)).node): WARNED.add(name) logger.warning( 'Builtin name "%s" has been masked by datar.', name, ) return imports[f"{name}_"]
def check_filename(self, filename): print(filename) source = Source.for_filename(filename) nodes = {} for node in ast.walk(source.tree): if isinstance(node, (ast.UnaryOp, ast.BinOp, ast.Subscript, ast.Call, ast.Compare, ast.Attribute)): nodes[node] = None code = compile(source.tree, source.filename, 'exec') result = list(self.check_code(code, nodes)) if not re.search(r'^\s*if 0(:| and )', source.text, re.MULTILINE): for node, value in nodes.items(): if is_unary_not(node): continue if isinstance(getattr(node, 'ctx', None), (ast.Store, ast.Del)): assert value is None continue if isinstance(node, ast.Compare): if len(node.ops) > 1: assert value is None continue if is_unary_not(node.parent) and isinstance( node.ops[0], (ast.In, ast.Is)): continue if is_literal(node): continue if value is None: print(source.text, '---', node_string(source, node), file=sys.stderr, sep='\n') self.fail() return result
def argnode_source(source: Source, node: ast.AST, vars_only: bool) -> Union[str, ast.AST]: """Get the source of an argument node Args: source: The executing source object node: The node to get the source from vars_only: Whether only allow variables and attributes Returns: The source of the node (node.id for ast.Name, node.attr for ast.Attribute). Or the node itself if the source cannot be fetched. """ if vars_only: return (node.id if isinstance(node, ast.Name) else node.attr if isinstance(node, ast.Attribute) else node) # requires asttokens return source.asttokens().get_text(node)
def argname(arg: Any, # pylint: disable=unused-argument *more_args: Any, # *, keyword-only argument, only available with python3.8+ func: Optional[Callable] = None, frame: int = 1, vars_only: bool = True, pos_only: bool = False) -> Union[str, Tuple[str]]: """Get the argument names/sources passed to a function Args: arg: Parameter of the function, used to map the argument passed to the function *more_args: Other parameters of the function, used to map more arguments passed to the function func: The target function. If not provided, the AST node of the function call will be used to fetch the function: - If a variable (ast.Name) used as function, the `node.id` will be used to get the function from `locals()` or `globals()`. - If variable (ast.Name), attributes (ast.Attribute), subscripts (ast.Subscript), and combinations of those and literals used as function, `pure_eval` will be used to evaluate the node - If `pure_eval` is not installed or failed to evaluate, `eval` will be used. A warning will be shown since unwanted side effects may happen in this case. You are encouraged to always pass the function explicitly. frame: The frame where target function is called from this call. The intermediate calls will be the wrappers of this function. However, keep in mind that the wrappers must have the same signature as this function. When `pos_only` is `True`, only the positional arguments have to be the same vars_only: Require the arguments to be variables only, pos_only: Only fetch the names/sources for positional arguments. Returns: The argument source when no more_args passed, otherwise a tuple of argument sources Raises: NonVariableArgumentError: When vars_only is True, and we are trying to retrieve the source of an argument that is not a variable (i.e. an expression) VarnameRetrievingError: When failed to get the frame or node ValueError: When the arguments passed to this function is invalid. Only variables and subscripts of variables are allow to be passed to this function. """ ignore_list = IgnoreList.create( ignore_lambda=False, ignore_varname=False ) # where argname(...) is called argname_frame = ignore_list.get_frame(frame) argname_node = get_node_by_frame(argname_frame) # where func(...) is called func_frame = ignore_list.get_frame(frame + 1) func_node = get_node_by_frame(func_frame) # Only do it when both nodes are available if not argname_node or not func_node: # We can do something at bytecode level, when a single positional # argument passed to both functions (argname and the target function) # However, it's hard to ensure that there is only a single positional # arguments passed to the target function, at bytecode level. raise VarnameRetrievingError( "The source code of 'argname' calling is not available." ) if not func: func = get_function_called_argname(func_frame, func_node) # don't pass the target arguments so that we can cache the sources in # the same call. For example: # >>> def func(a, b): # >>> a_name = argname(a) # >>> b_name = argname(b) argument_sources = get_argument_sources( Source.for_frame(func_frame), func_node, func, vars_only=vars_only, pos_only=pos_only ) ret = [] for argnode in argname_node.args: if not isinstance(argnode, (ast.Name, ast.Subscript, ast.Starred)): raise ValueError( "Arguments of 'argname' must be " "function arguments themselves or subscripts of them." ) if isinstance(argnode, ast.Starred): if ( not isinstance(argnode.value, ast.Name) or argnode.value.id not in argument_sources or not isinstance(argument_sources[argnode.value.id], tuple) ): posvar = argnode.value posvar = getattr(posvar, 'id', posvar) raise ValueError( f"No such variable positional argument {posvar!r}" ) ret.extend(argument_sources[argnode.value.id]) elif isinstance(argnode, ast.Name): if argnode.id not in argument_sources: raise ValueError( f"No value passed for argument {argnode.id!r}, " "or it is not an argument at all." ) ret.append(argument_sources[argnode.id]) else: name, subscript = parse_argname_subscript(argnode) if name not in argument_sources: raise ValueError(f"{name!r} is not an argument.") if (isinstance(subscript, int) and not isinstance(argument_sources[name], tuple)): raise ValueError( f"{name!r} is not a positional argument " "(*args, for example)." ) if (isinstance(subscript, str) and not isinstance(argument_sources[name], dict)): raise ValueError( f"{name!r} is not a keyword argument " "(**kwargs, for example)." ) ret.append(argument_sources[name][subscript]) if vars_only: for source in ret: if isinstance(source, ast.AST): raise NonVariableArgumentError( f'Argument {ast.dump(source)} is not a variable ' 'or an attribute.' ) return ret[0] if not more_args else tuple(ret)