def replace_modules_with_new_variants( manager: BuildManager, graph: Dict[str, State], old_modules: Dict[str, MypyFile], new_modules: Dict[str, MypyFile]) -> None: """Replace modules with newly builds versions. Retain the identities of externally visible AST nodes in the old ASTs so that references to the affected modules from other modules will still be valid (unless something was deleted or replaced with an incompatible definition, in which case there will be dangling references that will be handled by propagate_changes_using_dependencies). """ for id in new_modules: merge_asts(old_modules[id], old_modules[id].names, new_modules[id], new_modules[id].names) manager.modules[id] = old_modules[id]
def replace_modules_with_new_variants( manager: BuildManager, graph: Dict[str, State], old_modules: Dict[str, MypyFile], new_modules: Dict[str, Optional[MypyFile]]) -> None: """Replace modules with newly builds versions. Retain the identities of externally visible AST nodes in the old ASTs so that references to the affected modules from other modules will still be valid (unless something was deleted or replaced with an incompatible definition, in which case there will be dangling references that will be handled by propagate_changes_using_dependencies). """ for id in new_modules: new_module = new_modules[id] if id in old_modules and new_module is not None: preserved_module = old_modules[id] merge_asts(preserved_module, old_modules[id].names, new_module, new_module.names) manager.modules[id] = preserved_module graph[id].tree = preserved_module
def reprocess_nodes(manager: BuildManager, graph: Dict[str, State], module_id: str, nodeset: Set[DeferredNode], deps: Dict[str, Set[str]]) -> Set[str]: """Reprocess a set of nodes within a single module. Return fired triggers. """ if module_id not in graph: manager.log_fine_grained( '%s not in graph (blocking errors or deleted?)' % module_id) return set() file_node = manager.modules[module_id] old_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names) old_symbols = {name: names.copy() for name, names in old_symbols.items()} old_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names) def key(node: DeferredNode) -> int: # Unlike modules which are sorted by name within SCC, # nodes within the same module are sorted by line number, because # this is how they are processed in normal mode. return node.node.line nodes = sorted(nodeset, key=key) # TODO: ignore_all argument to set_file_ignored_lines manager.errors.set_file_ignored_lines(file_node.path, file_node.ignored_lines) # Strip semantic analysis information. for deferred in nodes: strip_target(deferred.node) semantic_analyzer = manager.semantic_analyzer patches = [] # type: List[Tuple[int, Callable[[], None]]] # Second pass of semantic analysis. We don't redo the first pass, because it only # does local things that won't go stale. for deferred in nodes: with semantic_analyzer.file_context( file_node=file_node, fnam=file_node.path, options=manager.options, active_type=deferred.active_typeinfo): manager.semantic_analyzer.refresh_partial(deferred.node, patches) # Third pass of semantic analysis. for deferred in nodes: with semantic_analyzer.file_context( file_node=file_node, fnam=file_node.path, options=manager.options, active_type=deferred.active_typeinfo): manager.semantic_analyzer_pass3.refresh_partial( deferred.node, patches) apply_semantic_analyzer_patches(patches) # Merge symbol tables to preserve identities of AST nodes. The file node will remain # the same, but other nodes may have been recreated with different identities, such as # NamedTuples defined using assignment statements. new_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names) for name in old_symbols: if name in new_symbols: merge_asts(file_node, old_symbols[name], file_node, new_symbols[name]) # Type check. checker = graph[module_id].type_checker() checker.reset() # We seem to need additional passes in fine-grained incremental mode. checker.pass_num = 0 checker.last_pass = 3 more = checker.check_second_pass(nodes) while more: more = False if graph[module_id].type_checker().check_second_pass(): more = True new_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names) # Check if any attribute types were changed and need to be propagated further. changed = compare_symbol_table_snapshots(file_node.fullname(), old_symbols_snapshot, new_symbols_snapshot) new_triggered = {make_trigger(name) for name in changed} # Dependencies may have changed. update_deps(module_id, nodes, graph, deps, manager.options) # Report missing imports. verify_dependencies(graph[module_id], manager) return new_triggered
def reprocess_nodes(manager: BuildManager, graph: Dict[str, State], module_id: str, nodeset: Set[FineGrainedDeferredNode], deps: Dict[str, Set[str]], processed_targets: List[str]) -> Set[str]: """Reprocess a set of nodes within a single module. Return fired triggers. """ if module_id not in graph: manager.log_fine_grained('%s not in graph (blocking errors or deleted?)' % module_id) return set() file_node = manager.modules[module_id] old_symbols = find_symbol_tables_recursive(file_node.fullname, file_node.names) old_symbols = {name: names.copy() for name, names in old_symbols.items()} old_symbols_snapshot = snapshot_symbol_table(file_node.fullname, file_node.names) def key(node: FineGrainedDeferredNode) -> int: # Unlike modules which are sorted by name within SCC, # nodes within the same module are sorted by line number, because # this is how they are processed in normal mode. return node.node.line nodes = sorted(nodeset, key=key) options = graph[module_id].options manager.errors.set_file_ignored_lines( file_node.path, file_node.ignored_lines, options.ignore_errors) targets = set() for node in nodes: target = target_from_node(module_id, node.node) if target is not None: targets.add(target) manager.errors.clear_errors_in_targets(file_node.path, targets) # If one of the nodes is the module itself, emit any errors that # happened before semantic analysis. for target in targets: if target == module_id: for info in graph[module_id].early_errors: manager.errors.add_error_info(info) # Strip semantic analysis information. saved_attrs = {} # type: SavedAttributes for deferred in nodes: processed_targets.append(deferred.node.fullname) strip_target(deferred.node, saved_attrs) semantic_analysis_for_targets(graph[module_id], nodes, graph, saved_attrs) # Merge symbol tables to preserve identities of AST nodes. The file node will remain # the same, but other nodes may have been recreated with different identities, such as # NamedTuples defined using assignment statements. new_symbols = find_symbol_tables_recursive(file_node.fullname, file_node.names) for name in old_symbols: if name in new_symbols: merge_asts(file_node, old_symbols[name], file_node, new_symbols[name]) # Type check. checker = graph[module_id].type_checker() checker.reset() # We seem to need additional passes in fine-grained incremental mode. checker.pass_num = 0 checker.last_pass = 3 more = checker.check_second_pass(nodes) while more: more = False if graph[module_id].type_checker().check_second_pass(): more = True if manager.options.export_types: manager.all_types.update(graph[module_id].type_map()) new_symbols_snapshot = snapshot_symbol_table(file_node.fullname, file_node.names) # Check if any attribute types were changed and need to be propagated further. changed = compare_symbol_table_snapshots(file_node.fullname, old_symbols_snapshot, new_symbols_snapshot) new_triggered = {make_trigger(name) for name in changed} # Dependencies may have changed. update_deps(module_id, nodes, graph, deps, options) # Report missing imports. graph[module_id].verify_dependencies() graph[module_id].free_state() return new_triggered
def reprocess_nodes(manager: BuildManager, graph: Dict[str, State], module_id: str, nodeset: Set[DeferredNode], deps: Dict[str, Set[str]]) -> Set[str]: """Reprocess a set of nodes within a single module. Return fired triggers. """ if module_id not in graph: manager.log_fine_grained('%s not in graph (blocking errors or deleted?)' % module_id) return set() file_node = manager.modules[module_id] old_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names) old_symbols = {name: names.copy() for name, names in old_symbols.items()} old_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names) def key(node: DeferredNode) -> int: # Unlike modules which are sorted by name within SCC, # nodes within the same module are sorted by line number, because # this is how they are processed in normal mode. return node.node.line nodes = sorted(nodeset, key=key) # TODO: ignore_all argument to set_file_ignored_lines manager.errors.set_file_ignored_lines(file_node.path, file_node.ignored_lines) targets = set() for node in nodes: target = target_from_node(module_id, node.node) if target is not None: targets.add(target) manager.errors.clear_errors_in_targets(file_node.path, targets) # Strip semantic analysis information. for deferred in nodes: strip_target(deferred.node) semantic_analyzer = manager.semantic_analyzer patches = [] # type: List[Tuple[int, Callable[[], None]]] # Second pass of semantic analysis. We don't redo the first pass, because it only # does local things that won't go stale. for deferred in nodes: with semantic_analyzer.file_context( file_node=file_node, fnam=file_node.path, options=manager.options, active_type=deferred.active_typeinfo): manager.semantic_analyzer.refresh_partial(deferred.node, patches) # Third pass of semantic analysis. for deferred in nodes: with semantic_analyzer.file_context( file_node=file_node, fnam=file_node.path, options=manager.options, active_type=deferred.active_typeinfo, scope=manager.semantic_analyzer_pass3.scope): manager.semantic_analyzer_pass3.refresh_partial(deferred.node, patches) with semantic_analyzer.file_context( file_node=file_node, fnam=file_node.path, options=manager.options, active_type=None): apply_semantic_analyzer_patches(patches) # Merge symbol tables to preserve identities of AST nodes. The file node will remain # the same, but other nodes may have been recreated with different identities, such as # NamedTuples defined using assignment statements. new_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names) for name in old_symbols: if name in new_symbols: merge_asts(file_node, old_symbols[name], file_node, new_symbols[name]) # Type check. checker = graph[module_id].type_checker() checker.reset() # We seem to need additional passes in fine-grained incremental mode. checker.pass_num = 0 checker.last_pass = 3 more = checker.check_second_pass(nodes) while more: more = False if graph[module_id].type_checker().check_second_pass(): more = True new_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names) # Check if any attribute types were changed and need to be propagated further. changed = compare_symbol_table_snapshots(file_node.fullname(), old_symbols_snapshot, new_symbols_snapshot) new_triggered = {make_trigger(name) for name in changed} # Dependencies may have changed. update_deps(module_id, nodes, graph, deps, manager.options) # Report missing imports. verify_dependencies(graph[module_id], manager) return new_triggered
def reprocess_nodes(manager: BuildManager, graph: Dict[str, State], module_id: str, nodeset: Set[DeferredNode], deps: Dict[str, Set[str]]) -> Set[str]: """Reprocess a set of nodes within a single module. Return fired triggers. """ if module_id not in manager.saved_cache or module_id not in graph: if DEBUG: print( '%s not in saved cache or graph (blocking errors or deleted?)' % module_id) return set() file_node = manager.modules[module_id] old_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names) old_symbols = {name: names.copy() for name, names in old_symbols.items()} def key(node: DeferredNode) -> str: fullname = node.node.fullname() if fullname is None: if isinstance(node.node, FuncDef): info = node.node.info elif isinstance(node.node, OverloadedFuncDef): info = node.node.items[0].info else: assert False, "'None' fullname for %s instance" % type( node.node) assert info is not None fullname = '%s.%s' % (info.fullname(), node.node.name()) return fullname # Sort nodes by full name so that the order of processing is deterministic. nodes = sorted(nodeset, key=key) # TODO: ignore_all argument to set_file_ignored_lines manager.errors.set_file_ignored_lines(file_node.path, file_node.ignored_lines) # Keep track of potentially affected attribute types before type checking. old_types_map = get_enclosing_namespace_types(nodes) # Strip semantic analysis information. for deferred in nodes: strip_target(deferred.node) semantic_analyzer = manager.semantic_analyzer # Second pass of semantic analysis. We don't redo the first pass, because it only # does local things that won't go stale. for deferred in nodes: with semantic_analyzer.file_context( file_node=file_node, fnam=file_node.path, options=manager.options, active_type=deferred.active_typeinfo): manager.semantic_analyzer.refresh_partial(deferred.node) # Third pass of semantic analysis. for deferred in nodes: with semantic_analyzer.file_context( file_node=file_node, fnam=file_node.path, options=manager.options, active_type=deferred.active_typeinfo): manager.semantic_analyzer_pass3.refresh_partial(deferred.node) # Merge symbol tables to preserve identities of AST nodes. The file node will remain # the same, but other nodes may have been recreated with different identities, such as # NamedTuples defined using assignment statements. new_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names) for name in old_symbols: if name in new_symbols: merge_asts(file_node, old_symbols[name], file_node, new_symbols[name]) # Type check. meta, file_node, type_map = manager.saved_cache[module_id] graph[module_id].tree = file_node graph[module_id].type_checker().type_map = type_map checker = graph[module_id].type_checker() # We seem to need additional passes in fine-grained incremental mode. checker.pass_num = 0 checker.last_pass = 3 more = checker.check_second_pass(nodes) while more: more = False if graph[module_id].type_checker().check_second_pass(): more = True # Check if any attribute types were changed and need to be propagated further. new_triggered = get_triggered_namespace_items(old_types_map) # Dependencies may have changed. update_deps(module_id, nodes, graph, deps, manager.options) # Report missing imports. verify_dependencies(graph[module_id], manager) return new_triggered
def reprocess_nodes(manager: BuildManager, graph: Dict[str, State], module_id: str, nodeset: Set[FineGrainedDeferredNode], deps: Dict[str, Set[str]], processed_targets: List[str]) -> Set[str]: """Reprocess a set of nodes within a single module. Return fired triggers. """ if module_id not in graph: manager.log_fine_grained('%s not in graph (blocking errors or deleted?)' % module_id) return set() file_node = manager.modules[module_id] old_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names) old_symbols = {name: names.copy() for name, names in old_symbols.items()} old_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names) def key(node: FineGrainedDeferredNode) -> int: # Unlike modules which are sorted by name within SCC, # nodes within the same module are sorted by line number, because # this is how they are processed in normal mode. return node.node.line nodes = sorted(nodeset, key=key) options = graph[module_id].options manager.errors.set_file_ignored_lines( file_node.path, file_node.ignored_lines, options.ignore_errors) targets = set() for node in nodes: target = target_from_node(module_id, node.node) if target is not None: targets.add(target) manager.errors.clear_errors_in_targets(file_node.path, targets) # Strip semantic analysis information. patches = [] # type: List[Callable[[], None]] for deferred in nodes: processed_targets.append(deferred.node.fullname()) if not manager.options.new_semantic_analyzer: strip_target(deferred.node) else: patches = strip_target_new(deferred.node) if not options.new_semantic_analyzer: re_analyze_nodes(file_node, nodes, manager, options) else: process_selected_targets(graph[module_id], nodes, graph, patches) # Merge symbol tables to preserve identities of AST nodes. The file node will remain # the same, but other nodes may have been recreated with different identities, such as # NamedTuples defined using assignment statements. new_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names) for name in old_symbols: if name in new_symbols: merge_asts(file_node, old_symbols[name], file_node, new_symbols[name]) # Type check. checker = graph[module_id].type_checker() checker.reset() # We seem to need additional passes in fine-grained incremental mode. checker.pass_num = 0 checker.last_pass = 3 more = checker.check_second_pass(nodes) while more: more = False if graph[module_id].type_checker().check_second_pass(): more = True if manager.options.export_types: manager.all_types.update(graph[module_id].type_map()) new_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names) # Check if any attribute types were changed and need to be propagated further. changed = compare_symbol_table_snapshots(file_node.fullname(), old_symbols_snapshot, new_symbols_snapshot) new_triggered = {make_trigger(name) for name in changed} # Dependencies may have changed. update_deps(module_id, nodes, graph, deps, options) # Report missing imports. graph[module_id].verify_dependencies() return new_triggered