def reprocess_nodes(manager: BuildManager, graph: Dict[str, State], module_id: str, nodeset: Set[DeferredNode], deps: Dict[str, Set[str]]) -> Set[str]: """Reprocess a set of nodes within a single module. Return fired triggers. """ file_node = manager.modules[module_id] def key(node: DeferredNode) -> str: fullname = node.node.fullname() if isinstance(node.node, FuncDef) and fullname is None: assert node.node.info is not None fullname = '%s.%s' % (node.node.info.fullname(), node.node.name()) return fullname # Some nodes by full name so that the order of processing is deterministic. nodes = sorted(nodeset, key=key) # Strip semantic analysis information. for deferred in nodes: strip_target(deferred.node) semantic_analyzer = manager.semantic_analyzer # Second pass of semantic analysis. We don't redo the first pass, because it only # does local things that won't go stale. for deferred in nodes: with semantic_analyzer.file_context( file_node=file_node, fnam=file_node.path, options=manager.options, active_type=deferred.active_typeinfo): manager.semantic_analyzer.refresh_partial(deferred.node) # Third pass of semantic analysis. for deferred in nodes: with semantic_analyzer.file_context( file_node=file_node, fnam=file_node.path, options=manager.options, active_type=deferred.active_typeinfo): manager.semantic_analyzer_pass3.refresh_partial(deferred.node) # Keep track of potentially affected attribute types before type checking. old_types_map = get_enclosing_namespace_types(nodes) # Type check. graph[module_id].type_checker.check_second_pass( nodes) # TODO: check return value # Check if any attribute types were changed and need to be propagated further. new_triggered = get_triggered_namespace_items(old_types_map) # Dependencies may have changed. update_deps(module_id, nodes, graph, deps, manager.options) return new_triggered
def reprocess_nodes(manager: BuildManager, graph: Dict[str, State], module_id: str, nodeset: Set[DeferredNode], deps: Dict[str, Set[str]]) -> Set[str]: """Reprocess a set of nodes within a single module. Return fired triggers. """ if module_id not in graph: manager.log_fine_grained( '%s not in graph (blocking errors or deleted?)' % module_id) return set() file_node = manager.modules[module_id] old_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names) old_symbols = {name: names.copy() for name, names in old_symbols.items()} old_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names) def key(node: DeferredNode) -> int: # Unlike modules which are sorted by name within SCC, # nodes within the same module are sorted by line number, because # this is how they are processed in normal mode. return node.node.line nodes = sorted(nodeset, key=key) # TODO: ignore_all argument to set_file_ignored_lines manager.errors.set_file_ignored_lines(file_node.path, file_node.ignored_lines) # Strip semantic analysis information. for deferred in nodes: strip_target(deferred.node) semantic_analyzer = manager.semantic_analyzer patches = [] # type: List[Tuple[int, Callable[[], None]]] # Second pass of semantic analysis. We don't redo the first pass, because it only # does local things that won't go stale. for deferred in nodes: with semantic_analyzer.file_context( file_node=file_node, fnam=file_node.path, options=manager.options, active_type=deferred.active_typeinfo): manager.semantic_analyzer.refresh_partial(deferred.node, patches) # Third pass of semantic analysis. for deferred in nodes: with semantic_analyzer.file_context( file_node=file_node, fnam=file_node.path, options=manager.options, active_type=deferred.active_typeinfo): manager.semantic_analyzer_pass3.refresh_partial( deferred.node, patches) apply_semantic_analyzer_patches(patches) # Merge symbol tables to preserve identities of AST nodes. The file node will remain # the same, but other nodes may have been recreated with different identities, such as # NamedTuples defined using assignment statements. new_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names) for name in old_symbols: if name in new_symbols: merge_asts(file_node, old_symbols[name], file_node, new_symbols[name]) # Type check. checker = graph[module_id].type_checker() checker.reset() # We seem to need additional passes in fine-grained incremental mode. checker.pass_num = 0 checker.last_pass = 3 more = checker.check_second_pass(nodes) while more: more = False if graph[module_id].type_checker().check_second_pass(): more = True new_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names) # Check if any attribute types were changed and need to be propagated further. changed = compare_symbol_table_snapshots(file_node.fullname(), old_symbols_snapshot, new_symbols_snapshot) new_triggered = {make_trigger(name) for name in changed} # Dependencies may have changed. update_deps(module_id, nodes, graph, deps, manager.options) # Report missing imports. verify_dependencies(graph[module_id], manager) return new_triggered
def reprocess_nodes(manager: BuildManager, graph: Dict[str, State], module_id: str, nodeset: Set[FineGrainedDeferredNode], deps: Dict[str, Set[str]], processed_targets: List[str]) -> Set[str]: """Reprocess a set of nodes within a single module. Return fired triggers. """ if module_id not in graph: manager.log_fine_grained('%s not in graph (blocking errors or deleted?)' % module_id) return set() file_node = manager.modules[module_id] old_symbols = find_symbol_tables_recursive(file_node.fullname, file_node.names) old_symbols = {name: names.copy() for name, names in old_symbols.items()} old_symbols_snapshot = snapshot_symbol_table(file_node.fullname, file_node.names) def key(node: FineGrainedDeferredNode) -> int: # Unlike modules which are sorted by name within SCC, # nodes within the same module are sorted by line number, because # this is how they are processed in normal mode. return node.node.line nodes = sorted(nodeset, key=key) options = graph[module_id].options manager.errors.set_file_ignored_lines( file_node.path, file_node.ignored_lines, options.ignore_errors) targets = set() for node in nodes: target = target_from_node(module_id, node.node) if target is not None: targets.add(target) manager.errors.clear_errors_in_targets(file_node.path, targets) # If one of the nodes is the module itself, emit any errors that # happened before semantic analysis. for target in targets: if target == module_id: for info in graph[module_id].early_errors: manager.errors.add_error_info(info) # Strip semantic analysis information. saved_attrs = {} # type: SavedAttributes for deferred in nodes: processed_targets.append(deferred.node.fullname) strip_target(deferred.node, saved_attrs) semantic_analysis_for_targets(graph[module_id], nodes, graph, saved_attrs) # Merge symbol tables to preserve identities of AST nodes. The file node will remain # the same, but other nodes may have been recreated with different identities, such as # NamedTuples defined using assignment statements. new_symbols = find_symbol_tables_recursive(file_node.fullname, file_node.names) for name in old_symbols: if name in new_symbols: merge_asts(file_node, old_symbols[name], file_node, new_symbols[name]) # Type check. checker = graph[module_id].type_checker() checker.reset() # We seem to need additional passes in fine-grained incremental mode. checker.pass_num = 0 checker.last_pass = 3 more = checker.check_second_pass(nodes) while more: more = False if graph[module_id].type_checker().check_second_pass(): more = True if manager.options.export_types: manager.all_types.update(graph[module_id].type_map()) new_symbols_snapshot = snapshot_symbol_table(file_node.fullname, file_node.names) # Check if any attribute types were changed and need to be propagated further. changed = compare_symbol_table_snapshots(file_node.fullname, old_symbols_snapshot, new_symbols_snapshot) new_triggered = {make_trigger(name) for name in changed} # Dependencies may have changed. update_deps(module_id, nodes, graph, deps, options) # Report missing imports. graph[module_id].verify_dependencies() graph[module_id].free_state() return new_triggered
def reprocess_nodes(manager: BuildManager, graph: Dict[str, State], module_id: str, nodeset: Set[DeferredNode], deps: Dict[str, Set[str]]) -> Set[str]: """Reprocess a set of nodes within a single module. Return fired triggers. """ if module_id not in graph: manager.log_fine_grained('%s not in graph (blocking errors or deleted?)' % module_id) return set() file_node = manager.modules[module_id] old_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names) old_symbols = {name: names.copy() for name, names in old_symbols.items()} old_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names) def key(node: DeferredNode) -> int: # Unlike modules which are sorted by name within SCC, # nodes within the same module are sorted by line number, because # this is how they are processed in normal mode. return node.node.line nodes = sorted(nodeset, key=key) # TODO: ignore_all argument to set_file_ignored_lines manager.errors.set_file_ignored_lines(file_node.path, file_node.ignored_lines) targets = set() for node in nodes: target = target_from_node(module_id, node.node) if target is not None: targets.add(target) manager.errors.clear_errors_in_targets(file_node.path, targets) # Strip semantic analysis information. for deferred in nodes: strip_target(deferred.node) semantic_analyzer = manager.semantic_analyzer patches = [] # type: List[Tuple[int, Callable[[], None]]] # Second pass of semantic analysis. We don't redo the first pass, because it only # does local things that won't go stale. for deferred in nodes: with semantic_analyzer.file_context( file_node=file_node, fnam=file_node.path, options=manager.options, active_type=deferred.active_typeinfo): manager.semantic_analyzer.refresh_partial(deferred.node, patches) # Third pass of semantic analysis. for deferred in nodes: with semantic_analyzer.file_context( file_node=file_node, fnam=file_node.path, options=manager.options, active_type=deferred.active_typeinfo, scope=manager.semantic_analyzer_pass3.scope): manager.semantic_analyzer_pass3.refresh_partial(deferred.node, patches) with semantic_analyzer.file_context( file_node=file_node, fnam=file_node.path, options=manager.options, active_type=None): apply_semantic_analyzer_patches(patches) # Merge symbol tables to preserve identities of AST nodes. The file node will remain # the same, but other nodes may have been recreated with different identities, such as # NamedTuples defined using assignment statements. new_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names) for name in old_symbols: if name in new_symbols: merge_asts(file_node, old_symbols[name], file_node, new_symbols[name]) # Type check. checker = graph[module_id].type_checker() checker.reset() # We seem to need additional passes in fine-grained incremental mode. checker.pass_num = 0 checker.last_pass = 3 more = checker.check_second_pass(nodes) while more: more = False if graph[module_id].type_checker().check_second_pass(): more = True new_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names) # Check if any attribute types were changed and need to be propagated further. changed = compare_symbol_table_snapshots(file_node.fullname(), old_symbols_snapshot, new_symbols_snapshot) new_triggered = {make_trigger(name) for name in changed} # Dependencies may have changed. update_deps(module_id, nodes, graph, deps, manager.options) # Report missing imports. verify_dependencies(graph[module_id], manager) return new_triggered
def reprocess_nodes(manager: BuildManager, graph: Dict[str, State], module_id: str, nodeset: Set[DeferredNode], deps: Dict[str, Set[str]]) -> Set[str]: """Reprocess a set of nodes within a single module. Return fired triggers. """ if module_id not in manager.saved_cache or module_id not in graph: if DEBUG: print( '%s not in saved cache or graph (blocking errors or deleted?)' % module_id) return set() file_node = manager.modules[module_id] old_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names) old_symbols = {name: names.copy() for name, names in old_symbols.items()} def key(node: DeferredNode) -> str: fullname = node.node.fullname() if fullname is None: if isinstance(node.node, FuncDef): info = node.node.info elif isinstance(node.node, OverloadedFuncDef): info = node.node.items[0].info else: assert False, "'None' fullname for %s instance" % type( node.node) assert info is not None fullname = '%s.%s' % (info.fullname(), node.node.name()) return fullname # Sort nodes by full name so that the order of processing is deterministic. nodes = sorted(nodeset, key=key) # TODO: ignore_all argument to set_file_ignored_lines manager.errors.set_file_ignored_lines(file_node.path, file_node.ignored_lines) # Keep track of potentially affected attribute types before type checking. old_types_map = get_enclosing_namespace_types(nodes) # Strip semantic analysis information. for deferred in nodes: strip_target(deferred.node) semantic_analyzer = manager.semantic_analyzer # Second pass of semantic analysis. We don't redo the first pass, because it only # does local things that won't go stale. for deferred in nodes: with semantic_analyzer.file_context( file_node=file_node, fnam=file_node.path, options=manager.options, active_type=deferred.active_typeinfo): manager.semantic_analyzer.refresh_partial(deferred.node) # Third pass of semantic analysis. for deferred in nodes: with semantic_analyzer.file_context( file_node=file_node, fnam=file_node.path, options=manager.options, active_type=deferred.active_typeinfo): manager.semantic_analyzer_pass3.refresh_partial(deferred.node) # Merge symbol tables to preserve identities of AST nodes. The file node will remain # the same, but other nodes may have been recreated with different identities, such as # NamedTuples defined using assignment statements. new_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names) for name in old_symbols: if name in new_symbols: merge_asts(file_node, old_symbols[name], file_node, new_symbols[name]) # Type check. meta, file_node, type_map = manager.saved_cache[module_id] graph[module_id].tree = file_node graph[module_id].type_checker().type_map = type_map checker = graph[module_id].type_checker() # We seem to need additional passes in fine-grained incremental mode. checker.pass_num = 0 checker.last_pass = 3 more = checker.check_second_pass(nodes) while more: more = False if graph[module_id].type_checker().check_second_pass(): more = True # Check if any attribute types were changed and need to be propagated further. new_triggered = get_triggered_namespace_items(old_types_map) # Dependencies may have changed. update_deps(module_id, nodes, graph, deps, manager.options) # Report missing imports. verify_dependencies(graph[module_id], manager) return new_triggered
def reprocess_nodes(manager: BuildManager, graph: Dict[str, State], module_id: str, nodeset: Set[FineGrainedDeferredNode], deps: Dict[str, Set[str]], processed_targets: List[str]) -> Set[str]: """Reprocess a set of nodes within a single module. Return fired triggers. """ if module_id not in graph: manager.log_fine_grained('%s not in graph (blocking errors or deleted?)' % module_id) return set() file_node = manager.modules[module_id] old_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names) old_symbols = {name: names.copy() for name, names in old_symbols.items()} old_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names) def key(node: FineGrainedDeferredNode) -> int: # Unlike modules which are sorted by name within SCC, # nodes within the same module are sorted by line number, because # this is how they are processed in normal mode. return node.node.line nodes = sorted(nodeset, key=key) options = graph[module_id].options manager.errors.set_file_ignored_lines( file_node.path, file_node.ignored_lines, options.ignore_errors) targets = set() for node in nodes: target = target_from_node(module_id, node.node) if target is not None: targets.add(target) manager.errors.clear_errors_in_targets(file_node.path, targets) # Strip semantic analysis information. patches = [] # type: List[Callable[[], None]] for deferred in nodes: processed_targets.append(deferred.node.fullname()) if not manager.options.new_semantic_analyzer: strip_target(deferred.node) else: patches = strip_target_new(deferred.node) if not options.new_semantic_analyzer: re_analyze_nodes(file_node, nodes, manager, options) else: process_selected_targets(graph[module_id], nodes, graph, patches) # Merge symbol tables to preserve identities of AST nodes. The file node will remain # the same, but other nodes may have been recreated with different identities, such as # NamedTuples defined using assignment statements. new_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names) for name in old_symbols: if name in new_symbols: merge_asts(file_node, old_symbols[name], file_node, new_symbols[name]) # Type check. checker = graph[module_id].type_checker() checker.reset() # We seem to need additional passes in fine-grained incremental mode. checker.pass_num = 0 checker.last_pass = 3 more = checker.check_second_pass(nodes) while more: more = False if graph[module_id].type_checker().check_second_pass(): more = True if manager.options.export_types: manager.all_types.update(graph[module_id].type_map()) new_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names) # Check if any attribute types were changed and need to be propagated further. changed = compare_symbol_table_snapshots(file_node.fullname(), old_symbols_snapshot, new_symbols_snapshot) new_triggered = {make_trigger(name) for name in changed} # Dependencies may have changed. update_deps(module_id, nodes, graph, deps, options) # Report missing imports. graph[module_id].verify_dependencies() return new_triggered