def __init__(self, *args, **kwargs): super(Source, self).__init__(*args, **kwargs) if self.tree: self.lines = self.text.splitlines() self.pieces = list(self._clean_pieces()) self.tokens_by_lineno = group_by_key_func(self.asttokens().tokens, lambda tok: tok.start[0]) else: self.lines = []
def precompute(words, length): words = [w for w in words if len(w) == length] by_letters = group_by_key_func(words, lambda w: "".join(sorted(w))) graph = Graph() by_subletters = defaultdict(list) for key in by_letters: counts = tuple(Counter(key).items()) for differing_letters in [1, 2]: for subletters in combinations(counts, len(counts) - differing_letters): by_subletters[subletters].append(key) for group in by_subletters.values(): for u, v in combinations(group, 2): u_counts = Counter(u) v_counts = Counter(v) less = None more = None for letter in u_counts.keys() | v_counts.keys(): if u_counts[letter] == v_counts[letter]: continue elif u_counts[letter] == v_counts[letter] + 1 and more is None: more = letter elif u_counts[letter] == v_counts[letter] - 1 and less is None: less = letter else: break else: assert less and more graph.add_edge(u, v) components = list(networkx.connected_components(graph)) nodes = list(max(components, key=len)) graph: networkx.Graph = graph.subgraph(nodes).copy() degree = graph.degree leaves = [node for node in nodes if degree[node] == 1] for leaf in leaves: while True: neighbors = list(graph.neighbors(leaf)) if len(neighbors) != 1: break graph.remove_node(leaf) leaf = neighbors[0] result = {} for node in graph: neighbors = list(graph.neighbors(node)) assert len(neighbors) > 1 result[node] = [by_letters[node], neighbors] return result
def _separate_comprehensions(self, nodes, end_lineno, positions, traced_file): # type: (list, int, List[HTMLPosition], TracedFile) -> int """ Comprehensions (e.g. list comprehensions) are troublesome because they can be navigated like loops, and the buttons for these need to be on separate lines. This function inserts newlines to turn: [x + y for x in range(3) for y in range(5)] and [[x + y for x in range(3)] for y in range(5)] into [x + y for x in range(3) for y in range(5)] and [[x + y for x in range(3)] for y in range(5)] """ comprehensions = group_by_key_func( of_type((ast.comprehension, ast.While, ast.For), nodes), lambda c: c.first_token.start[0] ) # type: Dict[Any, Iterable[ast.comprehension]] def get_start(n): # type: (ast.AST) -> int return traced_file.tokens.get_text_range(n)[0] for comp_list in comprehensions.values(): prev_start = None # type: Optional[int] for comp in sorted(comp_list, key=lambda c: c.first_token.startpos): if isinstance(comp, ast.comprehension ) and comp is comp.parent.generators[0]: start = get_start(comp.parent) if prev_start is not None and start < prev_start: start = get_start(comp) else: start = get_start(comp) if prev_start is not None: positions.append(HTMLPosition(start, True, 0, '\n ')) end_lineno += 1 prev_start = start return end_lineno
def variables(self): if not self.source.tree: return [] evaluator = Evaluator.from_frame(self.frame) get_text = self.source.asttokens().get_text scope = self.scope node_values = evaluator.find_expressions(scope) if isinstance(scope, ast.FunctionDef): for node in ast.walk(scope.args): if not isinstance(node, ast.arg): continue name = node.arg try: value = evaluator.names[name] except KeyError: pass else: node_values.append((node, value)) # TODO use compile(...).co_code instead of ast.dump? # Group equivalent nodes together grouped = group_by_key_func( node_values, # Add parens to avoid syntax errors for multiline expressions lambda nv: ast.dump(ast.parse('(' + get_text(nv[0]) + ')')), ) result = [] for group in grouped.values(): nodes, values = zip(*group) if not all(value is values[0] for value in values): # Distinct values found for same expression # Probably indicates another thread is messing with things # Since no single correct value exists, ignore this expression continue value = values[0] text = get_text(nodes[0]) result.append(Variable(text, nodes, value)) return result
def compare_versions(): out = [""] def prn(*args): out[0] += " ".join(map(str, args)) + "\n" samples_dir = os.path.join(tests_dir, "samples") results_dir = os.path.join(tests_dir, "sample_results") versions = sorted(os.listdir(results_dir)) for filename in sorted(os.listdir(samples_dir)): if not filename.endswith(".py"): continue module_name = filename[:-3] if module_name == "__init__": continue doesnt_exist = "Doesn't exist:" def get_results(version): path = os.path.join(results_dir, version, module_name + ".txt") if os.path.exists(path): return file_to_string(path) else: return doesnt_exist grouped = group_by_key_func(versions, get_results) if len(grouped) > 1: prn(module_name) doesnt_exist_versions = grouped.pop(doesnt_exist, None) if doesnt_exist_versions: prn(doesnt_exist, ", ".join(doesnt_exist_versions)) if len(grouped) > 1: prn("Differing versions:") for version_group in sorted(grouped.values()): prn(", ".join(version_group)) prn() compare_to_file(out[0], os.path.join(tests_dir, "version_differences.txt"))
def __call__(self, func): if inspect.isclass(func): cls = func for name, meth in iteritems(cls.__dict__): if inspect.ismethod(meth) or inspect.isfunction(meth): setattr(cls, name, self.__call__(meth)) return cls new_func = super(BirdsEye, self).__call__(func) code_info = self._code_infos.get(new_func.__code__) if code_info: return new_func lines, start_lineno = inspect.getsourcelines(func) end_lineno = start_lineno + len(lines) name = safe_qualname(func) filename = os.path.abspath(inspect.getsourcefile(func)) traced_file = new_func.traced_file traced_file.root._depth = 0 for node in ast.walk(traced_file.root): for child in ast.iter_child_nodes(node): child._depth = node._depth + 1 positions = [] node_loops = {} for node in traced_file.nodes: if isinstance(node, ast.expr): node_type = 'expr' if not node._is_interesting_expression: continue elif (isinstance(node, (ast.While, ast.For, ast.comprehension)) and not isinstance(node.parent, ast.GeneratorExp)): node_type = 'loop' elif isinstance(node, ast.stmt): node_type = 'stmt' else: continue assert isinstance(node, ast.AST) # In particular FormattedValue is missing this if not hasattr(node, 'first_token'): continue if not start_lineno <= node.first_token.start[0] <= end_lineno: continue start, end = traced_file.tokens.get_text_range(node) if start == end == 0: continue positions.append((start, 1, node._depth, '<span data-index="%s" data-type="%s">' % (node._tree_index, node_type))) positions.append((end, 0, node._depth, '</span>')) if node._loops: node_loops[node._tree_index] = [ n._tree_index for n in node._loops ] comprehensions = group_by_key_func([ comp for comp in traced_file.nodes if isinstance(comp, ast.comprehension) ], lambda c: c.first_token.line) def get_start(n): return traced_file.tokens.get_text_range(n)[0] for comp_list in comprehensions.values(): prev_start = None for comp in sorted(comp_list, key=lambda c: c.first_token.startpos): if comp is comp.parent.generators[0]: start = get_start(comp.parent) if prev_start is not None and start < prev_start: start = get_start(comp) else: start = get_start(comp) if prev_start is not None: positions.append((start, 1, 0, '\n ')) end_lineno += 1 prev_start = start positions.append((len(traced_file.source), None, None, '')) positions.sort() html_body = [] start = 0 for pos, _, _, part in positions: html_body.append(html.escape(traced_file.source[start:pos])) html_body.append(part) start = pos html_body = ''.join(html_body) html_body = '\n'.join( html_body.split('\n')[start_lineno - 1:end_lineno - 1]) db_args = dict(file=filename, name=name, html_body=html_body, lineno=start_lineno, data=json.dumps( dict(node_loops=node_loops, ), sort_keys=True, )) db_func = one_or_none(session.query(Function).filter_by(**db_args)) if not db_func: db_func = Function(**db_args) session.add(db_func) session.commit() self._code_infos[new_func.__code__] = CodeInfo(db_func, traced_file) return new_func