def check(source, encoding, exception=None, matches=True):
encoded = source.encode(encoding)
if exception:
with self.assertRaises(exception):
Source.decode_source(encoded)
else:
decoded = Source.decode_source(encoded)
if matches:
self.assertEqual(decoded, source)
else:
self.assertNotEqual(decoded, source)
def trace_func(frame, event, _arg):
filename = frame.f_code.co_filename
if event == "call":
if include_file(filename):
return trace_func
elif event == "line":
lineno = frame.f_lineno
queues[filename].append(lineno)
totals[filename][lineno] += 1
Source.lazycache(frame)
def tester(arg, returns=None):
frame = inspect.currentframe().f_back
Source.lazycache(frame)
call = Source.executing(frame).node
result = eval(
compile(ast.Expression(only(call.args)), '<>', 'eval'),
frame.f_globals,
frame.f_locals,
)
assert result == result, (result, arg)
if returns is None:
return arg
return returns
def file_table_context():
filename = request.args['filename']
source = Source.for_filename(filename)
queue = queues[filename]
highlighted = highlight_ranges(source, frames_matching(filename))
highlighted = highlight_python_and_ranges(highlighted)
highlighted_lines = list(enumerate(highlighted.splitlines()))
counters = [queue_counter(queue, 2**i) for i in range(levels + 1)]
ratios = [[
counter[i + 1] / min(2**c,
len(queue) or 1) * (c + 1) / levels
for c, counter in enumerate(counters)
] for i, _ in highlighted_lines]
max_ratio = max(map(max, ratios)) or 1
rows = [(
i + 1,
totals[filename][i + 1] or '',
reversed(
[int(round(ratio / max_ratio * 100)) for ratio in ratios[i]]),
line,
) for i, line in highlighted_lines]
return dict(
rows=rows,
zip=zip,
lightnesses=lightnesses,
filename=filename,
highlighted=highlighted,
)
def test_invalid_python(self):
path = os.path.join(
os.path.dirname(__file__),
'not_code.txt',
)
source = Source.for_filename(path)
self.assertIsNone(source.tree)
def match(self, frame_no: int,
frameinfos: List[inspect.FrameInfo]) -> bool:
frame = frameinfos[frame_no].frame
# module is None, check qualname only
return fnmatch(
Source.for_frame(frame).code_qualname(frame.f_code), self.qualname)
def get_node(frame: int,
ignore: Optional[IgnoreType] = None,
raise_exc: bool = True) -> Optional[ast.AST]:
"""Try to get node from the executing object.
This can fail when a frame is failed to retrieve.
One case should be when python code is executed in
R pacakge `reticulate`, where only first frame is kept.
When the node can not be retrieved, try to return the first statement.
"""
from .ignore import IgnoreList
ignore = IgnoreList.create(ignore)
try:
frame = ignore.get_frame(frame)
except VarnameRetrievingError:
return None
exect = Source.executing(frame)
if exect.node:
return exect.node
if exect.source.text and exect.source.tree and raise_exc:
raise VarnameRetrievingError(
"Couldn't retrieve the call node. "
"This may happen if you're using some other AST magic at the "
"same time, such as pytest, ipython, macropy, or birdseye.")
return None
def argnode_source(source: Source, node: ast.AST,
vars_only: bool) -> Union[str, ast.AST]:
"""Get the source of an argument node
Args:
source: The executing source object
node: The node to get the source from
vars_only: Whether only allow variables and attributes
Returns:
The source of the node (node.id for ast.Name,
node.attr for ast.Attribute). Or the node itself if the source
cannot be fetched.
"""
if isinstance(node, ast.Constant):
return repr(node.value)
if sys.version_info < (3, 9): # pragma: no cover
if isinstance(node, ast.Index):
node = node.value
if isinstance(node, ast.Num):
return repr(node.n)
if isinstance(node, (ast.Bytes, ast.Str)):
return repr(node.s)
if isinstance(node, ast.NameConstant):
return repr(node.value)
if vars_only:
return (node.id if isinstance(node, ast.Name) else
node.attr if isinstance(node, ast.Attribute) else node)
# requires asttokens
return source.asttokens().get_text(node)
def highlight_ranges(source, frames):
text = source.text
ranges = set()
for frame in frames:
executing = Source.executing(frame)
if executing.node:
text_range = executing.text_range()
ranges.add(text_range)
positions = []
for start, end in ranges:
positions.append((start, open_sentinel))
positions.append((end, close_sentinel))
while True:
start = text.find('\n', start + 1, end)
if start == -1:
break
positions.append((start, close_sentinel))
positions.append((start + 1, open_sentinel))
# This just makes the loop below simpler
positions.append((len(text), ''))
positions.sort()
parts = []
start = 0
for position, part in positions:
parts.append(text[start:position])
parts.append(part)
start = position
return ''.join(parts)
def test_traceback(self):
try:
134895 / 0
except:
tb = sys.exc_info()[2]
ex = Source.executing(tb)
self.assertTrue(isinstance(ex.node, ast.BinOp))
self.assertEqual(ex.text(), "134895 / 0")
def _(evt):
# Patch to ensure the executing module's cache is invalidated whenever
# a source file is changed.
cache = Source._class_local("__source_cache", {})
filename = evt.codefile.filename
if filename in cache:
del cache[filename]
linecache.checkcache(filename)
def test_executing_methods(self):
frame = inspect.currentframe()
executing = Source.executing(frame)
self.assertEqual(executing.code_qualname(), 'TestStuff.test_executing_methods')
text = 'Source.executing(frame)'
self.assertEqual(executing.text(), text)
start, end = executing.text_range()
self.assertEqual(executing.source.text[start:end], text)
def calling_env(funtype: str) -> Any:
"""Checking how the function is called:
1. PIPING_VERB: It is a verb that is piped directed. ie. data >> verb(...)
2. PIPING: It is a function called as (part of) the argument
of a piping verb. ie.:
>>> data >> verb(func(...))
Note that `func` here could also be a verb. When a function is called
inside a lambda body, it should not be counted in this situation:
>>> data >> verb(lambda: func(...))
In this case, func should be called as normal function.
This function should return `None`
3. FUNC_ARG: It is an argument of any function call
4. None: None of the above situation fits
This function should be only called inside register_*.wrapper
"""
if options.assume_all_piping:
return (CallingEnvs.PIPING_VERB
if funtype == 'Verb' else CallingEnvs.PIPING)
# frame 1: register_*.wrapper
# frame 2: func(...)
frame = sys._getframe(2)
my_node = Source.executing(frame).node
if not my_node and options.warn_astnode_failure:
warnings.warn(
"Failed to fetch the node calling the function, "
"please call the verbs regularly instead of `data >> verb(...)`.")
return None
piping_verb_node = _get_piping_verb_node(my_node)
if piping_verb_node is my_node and piping_verb_node is not None:
return CallingEnvs.PIPING_VERB
if _is_piping_verb_argument_node(my_node, piping_verb_node):
return CallingEnvs.PIPING
parent_call_node = _argument_node_of(my_node)
if parent_call_node is None:
return None
# check if parent call node is a function registered by
# register_verb/register_func
evaluator = Evaluator.from_frame(frame)
try:
func = evaluator[parent_call_node.func]
except CannotEval: # pragma: no cover
return None
if functype(func) != "plain":
return CallingEnvs.PIPING
return None
def gen():
frame = current_frame()
while frame:
code = frame.f_code
filename = code.co_filename
name = Source.for_frame(frame).code_qualname(code)
yield (filename, frame.f_lineno, name,
highlight_stack_frame(frame), include_file(filename))
frame = frame.f_back
def __call__(self, *args, **kwargs):
frame = sys._getframe(1)
while frame.f_code in self._excluded_codes:
frame = frame.f_back
executing = Source.executing(frame)
assert executing.node, "Failed to find call node"
return self.at(FrameInfo(executing))(*args, **kwargs)
def _post_init(self) -> None:
attach_ignore_id_to_module(self.module)
# check uniqueness of qualname
modfile = getattr(self.module, '__file__', None)
if modfile is not None:
check_qualname_by_source(
Source.for_filename(modfile, self.module.__dict__),
self.module.__name__, self.qualname)
def check_filename(self, filename):
print(filename)
source = Source.for_filename(filename)
if PY3:
code = compile(source.text, filename, "exec", dont_inherit=True)
for subcode, qualname in find_qualnames(code):
if not qualname.endswith(">"):
code_qualname = source.code_qualname(subcode)
self.assertEqual(code_qualname, qualname)
nodes = defaultdict(list)
for node in ast.walk(source.tree):
if isinstance(node, (
ast.UnaryOp,
ast.BinOp,
ast.Subscript,
ast.Call,
ast.Compare,
ast.Attribute
)):
nodes[node] = []
code = compile(source.tree, source.filename, 'exec')
result = list(self.check_code(code, nodes))
if not re.search(r'^\s*if 0(:| and )', source.text, re.MULTILINE):
for node, values in nodes.items():
if is_unary_not(node):
continue
if isinstance(getattr(node, 'ctx', None), (ast.Store, ast.Del)):
assert not values
continue
if isinstance(node, ast.Compare):
if len(node.ops) > 1:
assert not values
continue
if is_unary_not(node.parent) and isinstance(node.ops[0], (ast.In, ast.Is)):
continue
if is_literal(node):
continue
if sys.version_info >= (3, 9) and in_finally(node):
correct = len(values) > 1
else:
correct = len(values) == 1
if not correct:
print(source.text, '---', node_string(source, node), node.lineno,
len(values), correct, values, file=sys.stderr, sep='\n')
self.fail()
return result
def check_code(self, code, nodes):
linestarts = dict(dis.findlinestarts(code))
instructions = get_instructions(code)
lineno = None
for inst in instructions:
if time.time() - self.start_time > 45 * 60:
# Avoid travis time limit of 50 minutes
raise TimeOut
lineno = linestarts.get(inst.offset, lineno)
if not inst.opname.startswith((
'BINARY_', 'UNARY_', 'LOAD_ATTR', 'LOAD_METHOD', 'LOOKUP_METHOD',
'SLICE+', 'COMPARE_OP', 'CALL_', 'IS_OP', 'CONTAINS_OP',
)):
continue
frame = C()
frame.f_lasti = inst.offset
frame.f_code = code
frame.f_globals = globals()
frame.f_lineno = lineno
source = Source.for_frame(frame)
node = None
try:
try:
node = Source.executing(frame).node
except Exception:
if inst.opname.startswith(('COMPARE_OP', 'CALL_')):
continue
if isinstance(only(source.statements_at_line(lineno)), (ast.AugAssign, ast.Import)):
continue
raise
except Exception:
print(source.text, lineno, inst, node and ast.dump(node), code, file=sys.stderr, sep='\n')
raise
nodes[node].append((inst, frame.__dict__))
yield [inst.opname, node_string(source, node)]
for const in code.co_consts:
if isinstance(const, type(code)):
for x in self.check_code(const, nodes):
yield x
def test_many_calls(self):
node = None
start = time.time()
for i in range(10000):
new_node = Source.executing(inspect.currentframe()).node
if node is None:
node = new_node
else:
self.assertIs(node, new_node)
self.assertLess(time.time() - start, 1)
def assert_name_error(self):
try:
yield
except NameError as e:
tb = sys.exc_info()[2]
ex = Source.executing(tb.tb_next)
self.assertEqual(type(ex.node), ast.Name)
self.assertIn(ex.node.id, str(e))
self.assertEqual(ex.text(), ex.node.id)
else:
self.fail("NameError not raised")
def highlight_stack_frame(frame):
executing = Source.executing(frame)
node = executing.node
source = executing.source
if node:
source.asttokens()
start = node.first_token.start[0]
end = node.last_token.end[0]
else:
start = end = frame.f_lineno
highlighted = '\n'.join(highlight_ranges(source, [frame]).splitlines()[start - 1:end])
return highlight_python_and_ranges(highlighted)
def match(self, frame_no: int,
frameinfos: List[inspect.FrameInfo]) -> bool:
frame = frameinfos[frame_no].frame
frame_filename = path.realpath(frame.f_code.co_filename)
preset_filename = path.realpath(self.filename)
# return earlier to avoid qualname uniqueness check
if frame_filename != preset_filename:
return False
source = Source.for_frame(frame)
check_qualname_by_source(source, self.filename, self.qualname)
return fnmatch(source.code_qualname(frame.f_code), self.qualname)
def test_ignore_module_qualname_no_source(tmp_path):
module = module_from_source(
'ignore_module_qualname_no_source', """
def bar():
return 1
""", tmp_path)
source = Source.for_filename(module.__file__)
# simulate when source is not available
# no way to check uniqueness of qualname
source.tree = None
def foo():
return varname(ignore=(module, 'bar'))
f = foo()
def get_node_by_frame(frame: FrameType,
raise_exc: bool = True) -> Optional[ast.AST]:
"""Get the node by frame, raise errors if possible"""
exect = Source.executing(frame)
if exect.node:
return exect.node
if exect.source.text and exect.source.tree and raise_exc:
raise VarnameRetrievingError(
"Couldn't retrieve the call node. "
"This may happen if you're using some other AST magic at the "
"same time, such as pytest, ipython, macropy, or birdseye.")
return None
def get_node_by_frame(frame: FrameType, raise_exc: bool = True) -> ast.AST:
"""Get the node by frame, raise errors if possible"""
exect = Source.executing(frame)
if exect.node:
# attach the frame for better exception message
# (ie. where ImproperUseError happens)
exect.node.__frame__ = frame
return exect.node
if exect.source.text and exect.source.tree and raise_exc:
raise VarnameRetrievingError(
"Couldn't retrieve the call node. "
"This may happen if you're using some other AST magic at the "
"same time, such as pytest, ipython, macropy, or birdseye.")
return None
def match(self, frame_no: int,
frameinfos: List[inspect.FrameInfo]) -> bool:
frame = frameinfos[frame_no].frame
module = cached_getmodule(frame.f_code)
# Return earlier to avoid qualname uniqueness check
if module and module != self.module:
return False
if (not module
and not frame_matches_module_by_ignore_id(frame, self.module)):
return False
source = Source.for_frame(frame)
check_qualname_by_source(source, self.module.__name__, self.qualname)
return fnmatch(source.code_qualname(frame.f_code), self.qualname)
def _getattr(name: str):
# Using get_option("import_names_conflict") to get the value
# instead of `import_names_conflict`
# OPTIONS changed in lifetime
opt_maybe_changed = get_option("import_names_conflict")
if (name == "__path__" or name not in conflict_names
or opt_maybe_changed == "underscore_suffixed"):
raise AttributeError
# from ... import xxx
if (name not in WARNED and opt_maybe_changed == "warn"
and not Source.executing(sys._getframe(1)).node):
WARNED.add(name)
logger.warning(
'Builtin name "%s" has been masked by datar.',
name,
)
return imports[f"{name}_"]
def check_filename(self, filename):
print(filename)
source = Source.for_filename(filename)
nodes = {}
for node in ast.walk(source.tree):
if isinstance(node, (ast.UnaryOp, ast.BinOp, ast.Subscript,
ast.Call, ast.Compare, ast.Attribute)):
nodes[node] = None
code = compile(source.tree, source.filename, 'exec')
result = list(self.check_code(code, nodes))
if not re.search(r'^\s*if 0(:| and )', source.text, re.MULTILINE):
for node, value in nodes.items():
if is_unary_not(node):
continue
if isinstance(getattr(node, 'ctx', None),
(ast.Store, ast.Del)):
assert value is None
continue
if isinstance(node, ast.Compare):
if len(node.ops) > 1:
assert value is None
continue
if is_unary_not(node.parent) and isinstance(
node.ops[0], (ast.In, ast.Is)):
continue
if is_literal(node):
continue
if value is None:
print(source.text,
'---',
node_string(source, node),
file=sys.stderr,
sep='\n')
self.fail()
return result
def argnode_source(source: Source, node: ast.AST,
vars_only: bool) -> Union[str, ast.AST]:
"""Get the source of an argument node
Args:
source: The executing source object
node: The node to get the source from
vars_only: Whether only allow variables and attributes
Returns:
The source of the node (node.id for ast.Name,
node.attr for ast.Attribute). Or the node itself if the source
cannot be fetched.
"""
if vars_only:
return (node.id if isinstance(node, ast.Name) else
node.attr if isinstance(node, ast.Attribute) else node)
# requires asttokens
return source.asttokens().get_text(node)
def argname(arg: Any, # pylint: disable=unused-argument
*more_args: Any,
# *, keyword-only argument, only available with python3.8+
func: Optional[Callable] = None,
frame: int = 1,
vars_only: bool = True,
pos_only: bool = False) -> Union[str, Tuple[str]]:
"""Get the argument names/sources passed to a function
Args:
arg: Parameter of the function, used to map the argument passed to
the function
*more_args: Other parameters of the function, used to map more arguments
passed to the function
func: The target function. If not provided, the AST node of the
function call will be used to fetch the function:
- If a variable (ast.Name) used as function, the `node.id` will
be used to get the function from `locals()` or `globals()`.
- If variable (ast.Name), attributes (ast.Attribute),
subscripts (ast.Subscript), and combinations of those and
literals used as function, `pure_eval` will be used to evaluate
the node
- If `pure_eval` is not installed or failed to evaluate, `eval`
will be used. A warning will be shown since unwanted side
effects may happen in this case.
You are encouraged to always pass the function explicitly.
frame: The frame where target function is called from this call.
The intermediate calls will be the wrappers of this function.
However, keep in mind that the wrappers must have the same
signature as this function. When `pos_only` is `True`, only the
positional arguments have to be the same
vars_only: Require the arguments to be variables only,
pos_only: Only fetch the names/sources for positional arguments.
Returns:
The argument source when no more_args passed, otherwise a tuple of
argument sources
Raises:
NonVariableArgumentError: When vars_only is True, and we are trying
to retrieve the source of an argument that is not a variable
(i.e. an expression)
VarnameRetrievingError: When failed to get the frame or node
ValueError: When the arguments passed to this function is invalid.
Only variables and subscripts of variables are allow to be passed
to this function.
"""
ignore_list = IgnoreList.create(
ignore_lambda=False,
ignore_varname=False
)
# where argname(...) is called
argname_frame = ignore_list.get_frame(frame)
argname_node = get_node_by_frame(argname_frame)
# where func(...) is called
func_frame = ignore_list.get_frame(frame + 1)
func_node = get_node_by_frame(func_frame)
# Only do it when both nodes are available
if not argname_node or not func_node:
# We can do something at bytecode level, when a single positional
# argument passed to both functions (argname and the target function)
# However, it's hard to ensure that there is only a single positional
# arguments passed to the target function, at bytecode level.
raise VarnameRetrievingError(
"The source code of 'argname' calling is not available."
)
if not func:
func = get_function_called_argname(func_frame, func_node)
# don't pass the target arguments so that we can cache the sources in
# the same call. For example:
# >>> def func(a, b):
# >>> a_name = argname(a)
# >>> b_name = argname(b)
argument_sources = get_argument_sources(
Source.for_frame(func_frame),
func_node,
func,
vars_only=vars_only,
pos_only=pos_only
)
ret = []
for argnode in argname_node.args:
if not isinstance(argnode, (ast.Name, ast.Subscript, ast.Starred)):
raise ValueError(
"Arguments of 'argname' must be "
"function arguments themselves or subscripts of them."
)
if isinstance(argnode, ast.Starred):
if (
not isinstance(argnode.value, ast.Name) or
argnode.value.id not in argument_sources or
not isinstance(argument_sources[argnode.value.id], tuple)
):
posvar = argnode.value
posvar = getattr(posvar, 'id', posvar)
raise ValueError(
f"No such variable positional argument {posvar!r}"
)
ret.extend(argument_sources[argnode.value.id])
elif isinstance(argnode, ast.Name):
if argnode.id not in argument_sources:
raise ValueError(
f"No value passed for argument {argnode.id!r}, "
"or it is not an argument at all."
)
ret.append(argument_sources[argnode.id])
else:
name, subscript = parse_argname_subscript(argnode)
if name not in argument_sources:
raise ValueError(f"{name!r} is not an argument.")
if (isinstance(subscript, int) and
not isinstance(argument_sources[name], tuple)):
raise ValueError(
f"{name!r} is not a positional argument "
"(*args, for example)."
)
if (isinstance(subscript, str) and
not isinstance(argument_sources[name], dict)):
raise ValueError(
f"{name!r} is not a keyword argument "
"(**kwargs, for example)."
)
ret.append(argument_sources[name][subscript])
if vars_only:
for source in ret:
if isinstance(source, ast.AST):
raise NonVariableArgumentError(
f'Argument {ast.dump(source)} is not a variable '
'or an attribute.'
)
return ret[0] if not more_args else tuple(ret)
