def __init__(self, *args, **kwargs):
super(Source, self).__init__(*args, **kwargs)
if self.tree:
self.lines = self.text.splitlines()
self.pieces = list(self._clean_pieces())
self.tokens_by_lineno = group_by_key_func(self.asttokens().tokens,
lambda tok: tok.start[0])
else:
self.lines = []
def precompute(words, length):
words = [w for w in words if len(w) == length]
by_letters = group_by_key_func(words, lambda w: "".join(sorted(w)))
graph = Graph()
by_subletters = defaultdict(list)
for key in by_letters:
counts = tuple(Counter(key).items())
for differing_letters in [1, 2]:
for subletters in combinations(counts, len(counts) - differing_letters):
by_subletters[subletters].append(key)
for group in by_subletters.values():
for u, v in combinations(group, 2):
u_counts = Counter(u)
v_counts = Counter(v)
less = None
more = None
for letter in u_counts.keys() | v_counts.keys():
if u_counts[letter] == v_counts[letter]:
continue
elif u_counts[letter] == v_counts[letter] + 1 and more is None:
more = letter
elif u_counts[letter] == v_counts[letter] - 1 and less is None:
less = letter
else:
break
else:
assert less and more
graph.add_edge(u, v)
components = list(networkx.connected_components(graph))
nodes = list(max(components, key=len))
graph: networkx.Graph = graph.subgraph(nodes).copy()
degree = graph.degree
leaves = [node for node in nodes if degree[node] == 1]
for leaf in leaves:
while True:
neighbors = list(graph.neighbors(leaf))
if len(neighbors) != 1:
break
graph.remove_node(leaf)
leaf = neighbors[0]
result = {}
for node in graph:
neighbors = list(graph.neighbors(node))
assert len(neighbors) > 1
result[node] = [by_letters[node], neighbors]
return result
def _separate_comprehensions(self, nodes, end_lineno, positions,
traced_file):
# type: (list, int, List[HTMLPosition], TracedFile) -> int
"""
Comprehensions (e.g. list comprehensions) are troublesome because they can
be navigated like loops, and the buttons for these need to be on separate lines.
This function inserts newlines to turn:
[x + y for x in range(3) for y in range(5)] and
[[x + y for x in range(3)] for y in range(5)]
into
[x + y for x in range(3)
for y in range(5)] and
[[x + y for x in range(3)]
for y in range(5)]
"""
comprehensions = group_by_key_func(
of_type((ast.comprehension, ast.While, ast.For),
nodes), lambda c: c.first_token.start[0]
) # type: Dict[Any, Iterable[ast.comprehension]]
def get_start(n):
# type: (ast.AST) -> int
return traced_file.tokens.get_text_range(n)[0]
for comp_list in comprehensions.values():
prev_start = None # type: Optional[int]
for comp in sorted(comp_list,
key=lambda c: c.first_token.startpos):
if isinstance(comp, ast.comprehension
) and comp is comp.parent.generators[0]:
start = get_start(comp.parent)
if prev_start is not None and start < prev_start:
start = get_start(comp)
else:
start = get_start(comp)
if prev_start is not None:
positions.append(HTMLPosition(start, True, 0, '\n '))
end_lineno += 1
prev_start = start
return end_lineno
def variables(self):
if not self.source.tree:
return []
evaluator = Evaluator.from_frame(self.frame)
get_text = self.source.asttokens().get_text
scope = self.scope
node_values = evaluator.find_expressions(scope)
if isinstance(scope, ast.FunctionDef):
for node in ast.walk(scope.args):
if not isinstance(node, ast.arg):
continue
name = node.arg
try:
value = evaluator.names[name]
except KeyError:
pass
else:
node_values.append((node, value))
# TODO use compile(...).co_code instead of ast.dump?
# Group equivalent nodes together
grouped = group_by_key_func(
node_values,
# Add parens to avoid syntax errors for multiline expressions
lambda nv: ast.dump(ast.parse('(' + get_text(nv[0]) + ')')),
)
result = []
for group in grouped.values():
nodes, values = zip(*group)
if not all(value is values[0] for value in values):
# Distinct values found for same expression
# Probably indicates another thread is messing with things
# Since no single correct value exists, ignore this expression
continue
value = values[0]
text = get_text(nodes[0])
result.append(Variable(text, nodes, value))
return result
def compare_versions():
out = [""]
def prn(*args):
out[0] += " ".join(map(str, args)) + "\n"
samples_dir = os.path.join(tests_dir, "samples")
results_dir = os.path.join(tests_dir, "sample_results")
versions = sorted(os.listdir(results_dir))
for filename in sorted(os.listdir(samples_dir)):
if not filename.endswith(".py"):
continue
module_name = filename[:-3]
if module_name == "__init__":
continue
doesnt_exist = "Doesn't exist:"
def get_results(version):
path = os.path.join(results_dir, version, module_name + ".txt")
if os.path.exists(path):
return file_to_string(path)
else:
return doesnt_exist
grouped = group_by_key_func(versions, get_results)
if len(grouped) > 1:
prn(module_name)
doesnt_exist_versions = grouped.pop(doesnt_exist, None)
if doesnt_exist_versions:
prn(doesnt_exist, ", ".join(doesnt_exist_versions))
if len(grouped) > 1:
prn("Differing versions:")
for version_group in sorted(grouped.values()):
prn(", ".join(version_group))
prn()
compare_to_file(out[0], os.path.join(tests_dir, "version_differences.txt"))
def __call__(self, func):
if inspect.isclass(func):
cls = func
for name, meth in iteritems(cls.__dict__):
if inspect.ismethod(meth) or inspect.isfunction(meth):
setattr(cls, name, self.__call__(meth))
return cls
new_func = super(BirdsEye, self).__call__(func)
code_info = self._code_infos.get(new_func.__code__)
if code_info:
return new_func
lines, start_lineno = inspect.getsourcelines(func)
end_lineno = start_lineno + len(lines)
name = safe_qualname(func)
filename = os.path.abspath(inspect.getsourcefile(func))
traced_file = new_func.traced_file
traced_file.root._depth = 0
for node in ast.walk(traced_file.root):
for child in ast.iter_child_nodes(node):
child._depth = node._depth + 1
positions = []
node_loops = {}
for node in traced_file.nodes:
if isinstance(node, ast.expr):
node_type = 'expr'
if not node._is_interesting_expression:
continue
elif (isinstance(node, (ast.While, ast.For, ast.comprehension))
and not isinstance(node.parent, ast.GeneratorExp)):
node_type = 'loop'
elif isinstance(node, ast.stmt):
node_type = 'stmt'
else:
continue
assert isinstance(node, ast.AST)
# In particular FormattedValue is missing this
if not hasattr(node, 'first_token'):
continue
if not start_lineno <= node.first_token.start[0] <= end_lineno:
continue
start, end = traced_file.tokens.get_text_range(node)
if start == end == 0:
continue
positions.append((start, 1, node._depth,
'<span data-index="%s" data-type="%s">' %
(node._tree_index, node_type)))
positions.append((end, 0, node._depth, '</span>'))
if node._loops:
node_loops[node._tree_index] = [
n._tree_index for n in node._loops
]
comprehensions = group_by_key_func([
comp for comp in traced_file.nodes
if isinstance(comp, ast.comprehension)
], lambda c: c.first_token.line)
def get_start(n):
return traced_file.tokens.get_text_range(n)[0]
for comp_list in comprehensions.values():
prev_start = None
for comp in sorted(comp_list,
key=lambda c: c.first_token.startpos):
if comp is comp.parent.generators[0]:
start = get_start(comp.parent)
if prev_start is not None and start < prev_start:
start = get_start(comp)
else:
start = get_start(comp)
if prev_start is not None:
positions.append((start, 1, 0, '\n '))
end_lineno += 1
prev_start = start
positions.append((len(traced_file.source), None, None, ''))
positions.sort()
html_body = []
start = 0
for pos, _, _, part in positions:
html_body.append(html.escape(traced_file.source[start:pos]))
html_body.append(part)
start = pos
html_body = ''.join(html_body)
html_body = '\n'.join(
html_body.split('\n')[start_lineno - 1:end_lineno - 1])
db_args = dict(file=filename,
name=name,
html_body=html_body,
lineno=start_lineno,
data=json.dumps(
dict(node_loops=node_loops, ),
sort_keys=True,
))
db_func = one_or_none(session.query(Function).filter_by(**db_args))
if not db_func:
db_func = Function(**db_args)
session.add(db_func)
session.commit()
self._code_infos[new_func.__code__] = CodeInfo(db_func, traced_file)
return new_func
