def update_single_isolated(module: str, path: str, manager: BuildManager, previous_modules: Dict[str, str], graph: Graph) -> UpdateResult: """Build a new version of one changed module only. Don't propagate changes to elsewhere in the program. Raise CompleError on encountering a blocking error. Args: module: Changed module (modified, created or deleted) path: Path of the changed module manager: Build manager graph: Build graph Returns a named tuple describing the result (see above for details). """ if module in manager.modules: assert_equivalent_paths(path, manager.modules[module].path) else: manager.log_fine_grained('new module %r' % module) old_modules = dict(manager.modules) sources = get_sources(previous_modules, [(module, path)]) if module in manager.missing_modules: manager.missing_modules.remove(module) try: if module in graph: del graph[module] load_graph(sources, manager, graph) except CompileError as err: # Parse error somewhere in the program -- a blocker assert err.module_with_blocker if err.module_with_blocker != module: # Blocker is in a fresh module. Delete the state of the original target module # since it will be stale. # # TODO: It would be more efficient to store the original target module path = manager.modules[module].path del manager.modules[module] remaining_modules = [(module, path)] else: remaining_modules = [] return BlockedUpdate(err.module_with_blocker, path, remaining_modules, err.messages) if not os.path.isfile(path): delete_module(module, graph, manager) return NormalUpdate(module, path, [], None) # Find any other modules brought in by imports. changed_modules = get_all_changed_modules(module, path, previous_modules, graph) # If there are multiple modules to process, only process one of them and return # the remaining ones to the caller. if len(changed_modules) > 1: # As an optimization, look for a module that imports no other changed modules. module, path = find_relative_leaf_module(changed_modules, graph) changed_modules.remove((module, path)) remaining_modules = changed_modules # The remaining modules haven't been processed yet so drop them. for id, _ in remaining_modules: if id in old_modules: manager.modules[id] = old_modules[id] else: del manager.modules[id] del graph[id] manager.log_fine_grained('--> %r (newly imported)' % module) else: remaining_modules = [] state = graph[module] # Process the changed file. state.parse_file() # TODO: state.fix_suppressed_dependencies()? try: state.semantic_analysis() except CompileError as err: # There was a blocking error, so module AST is incomplete. Restore old modules. manager.modules.clear() manager.modules.update(old_modules) del graph[module] return BlockedUpdate(module, path, remaining_modules, err.messages) state.semantic_analysis_pass_three() state.semantic_analysis_apply_patches() # Merge old and new ASTs. assert state.tree is not None, "file must be at least parsed" new_modules = {module: state.tree} # type: Dict[str, Optional[MypyFile]] replace_modules_with_new_variants(manager, graph, old_modules, new_modules) # Perform type checking. state.type_checker().reset() state.type_check_first_pass() state.type_check_second_pass() state.compute_fine_grained_deps() state.finish_passes() # TODO: state.write_cache()? # TODO: state.mark_as_rechecked()? graph[module] = state return NormalUpdate(module, path, remaining_modules, state.tree)
def update_module_isolated(module: str, path: str, manager: BuildManager, previous_modules: Dict[str, str], graph: Graph, force_removed: bool) -> UpdateResult: """Build a new version of one changed module only. Don't propagate changes to elsewhere in the program. Raise CompileError on encountering a blocking error. Args: module: Changed module (modified, created or deleted) path: Path of the changed module manager: Build manager graph: Build graph force_removed: If True, consider the module removed from the build even it the file exists Returns a named tuple describing the result (see above for details). """ if module not in graph: manager.log_fine_grained('new module %r' % module) if not manager.fscache.isfile(path) or force_removed: delete_module(module, path, graph, manager) return NormalUpdate(module, path, [], None) sources = get_sources(manager.fscache, previous_modules, [(module, path)]) if module in manager.missing_modules: manager.missing_modules.remove(module) orig_module = module orig_state = graph.get(module) orig_tree = manager.modules.get(module) def restore(ids: List[str]) -> None: # For each of the modules in ids, restore that id's old # manager.modules and graphs entries. (Except for the original # module, this means deleting them.) for id in ids: if id == orig_module and orig_tree: manager.modules[id] = orig_tree elif id in manager.modules: del manager.modules[id] if id == orig_module and orig_state: graph[id] = orig_state elif id in graph: del graph[id] new_modules = [] # type: List[State] try: if module in graph: del graph[module] load_graph(sources, manager, graph, new_modules) except CompileError as err: # Parse error somewhere in the program -- a blocker assert err.module_with_blocker restore([module] + [st.id for st in new_modules]) return BlockedUpdate(err.module_with_blocker, path, [], err.messages) # Reparsing the file may have brought in dependencies that we # didn't have before. Make sure that they are loaded to restore # the invariant that a module having a loaded tree implies that # its dependencies do as well. ensure_trees_loaded(manager, graph, graph[module].dependencies) # Find any other modules brought in by imports. changed_modules = [(st.id, st.xpath) for st in new_modules] # If there are multiple modules to process, only process one of them and return # the remaining ones to the caller. if len(changed_modules) > 1: # As an optimization, look for a module that imports no other changed modules. module, path = find_relative_leaf_module(changed_modules, graph) changed_modules.remove((module, path)) remaining_modules = changed_modules # The remaining modules haven't been processed yet so drop them. restore([id for id, _ in remaining_modules]) manager.log_fine_grained('--> %r (newly imported)' % module) else: remaining_modules = [] state = graph[module] # Process the changed file. state.parse_file() assert state.tree is not None, "file must be at least parsed" t0 = time.time() # TODO: state.fix_suppressed_dependencies()? try: semantic_analysis_for_scc(graph, [state.id], manager.errors) except CompileError as err: # There was a blocking error, so module AST is incomplete. Restore old modules. restore([module]) return BlockedUpdate(module, path, remaining_modules, err.messages) # Merge old and new ASTs. new_modules_dict = {module: state.tree} # type: Dict[str, Optional[MypyFile]] replace_modules_with_new_variants(manager, graph, {orig_module: orig_tree}, new_modules_dict) t1 = time.time() # Perform type checking. state.type_checker().reset() state.type_check_first_pass() state.type_check_second_pass() t2 = time.time() state.finish_passes() t3 = time.time() manager.add_stats( semanal_time=t1 - t0, typecheck_time=t2 - t1, finish_passes_time=t3 - t2) graph[module] = state return NormalUpdate(module, path, remaining_modules, state.tree)
def update_module_isolated(module: str, path: str, manager: BuildManager, previous_modules: Dict[str, str], graph: Graph, force_removed: bool) -> UpdateResult: """Build a new version of one changed module only. Don't propagate changes to elsewhere in the program. Raise CompleError on encountering a blocking error. Args: module: Changed module (modified, created or deleted) path: Path of the changed module manager: Build manager graph: Build graph force_removed: If True, consider the module removed from the build even it the file exists Returns a named tuple describing the result (see above for details). """ if module in manager.modules: assert_equivalent_paths(path, manager.modules[module].path) else: manager.log_fine_grained('new module %r' % module) if not manager.fscache.isfile(path) or force_removed: delete_module(module, graph, manager) return NormalUpdate(module, path, [], None) old_modules = dict(manager.modules) sources = get_sources(manager.fscache, previous_modules, [(module, path)]) if module in manager.missing_modules: manager.missing_modules.remove(module) try: if module in graph: del graph[module] load_graph(sources, manager, graph) except CompileError as err: # Parse error somewhere in the program -- a blocker assert err.module_with_blocker if err.module_with_blocker != module: # Blocker is in a fresh module. Delete the state of the original target module # since it will be stale. # # TODO: It would be more efficient to store the original target module path = manager.modules[module].path del manager.modules[module] remaining_modules = [(module, path)] else: remaining_modules = [] return BlockedUpdate(err.module_with_blocker, path, remaining_modules, err.messages) # Find any other modules brought in by imports. changed_modules = get_all_changed_modules(module, path, previous_modules, graph) # If there are multiple modules to process, only process one of them and return # the remaining ones to the caller. if len(changed_modules) > 1: # As an optimization, look for a module that imports no other changed modules. module, path = find_relative_leaf_module(changed_modules, graph) changed_modules.remove((module, path)) remaining_modules = changed_modules # The remaining modules haven't been processed yet so drop them. for id, _ in remaining_modules: if id in old_modules: manager.modules[id] = old_modules[id] else: del manager.modules[id] del graph[id] manager.log_fine_grained('--> %r (newly imported)' % module) else: remaining_modules = [] state = graph[module] # Process the changed file. state.parse_file() # TODO: state.fix_suppressed_dependencies()? try: state.semantic_analysis() except CompileError as err: # There was a blocking error, so module AST is incomplete. Restore old modules. manager.modules.clear() manager.modules.update(old_modules) del graph[module] return BlockedUpdate(module, path, remaining_modules, err.messages) state.semantic_analysis_pass_three() state.semantic_analysis_apply_patches() # Merge old and new ASTs. assert state.tree is not None, "file must be at least parsed" new_modules = {module: state.tree} # type: Dict[str, Optional[MypyFile]] replace_modules_with_new_variants(manager, graph, old_modules, new_modules) # Perform type checking. state.type_checker().reset() state.type_check_first_pass() state.type_check_second_pass() state.compute_fine_grained_deps() state.finish_passes() graph[module] = state return NormalUpdate(module, path, remaining_modules, state.tree)
def build_incremental_step( manager: BuildManager, changed_modules: List[Tuple[str, str]], graph: Dict[str, State]) -> Tuple[Dict[str, Optional[MypyFile]], Graph]: """Build new versions of changed modules only. Raise CompleError on encountering a blocking error. Return the new ASTs for the changed modules and the entire build graph. """ # TODO: Handle multiple changed modules per step assert len(changed_modules) == 1 id, path = changed_modules[0] if id in manager.modules: path1 = os.path.normpath(path) path2 = os.path.normpath(manager.modules[id].path) assert path1 == path2, '%s != %s' % (path1, path2) old_modules = dict(manager.modules) sources = get_sources(graph, changed_modules) changed_set = {id for id, _ in changed_modules} invalidate_stale_cache_entries(manager.saved_cache, changed_modules) if not os.path.isfile(path): graph = delete_module(id, graph, manager) return {id: None}, graph old_graph = graph manager.missing_modules = set() graph = load_graph(sources, manager) # Find any other modules brought in by imports. for st in graph.values(): if st.id not in old_graph and st.id not in changed_set: changed_set.add(st.id) assert st.path changed_modules.append((st.id, st.path)) # TODO: Handle multiple changed modules per step assert len(changed_modules) == 1, changed_modules state = graph[id] # Parse file and run first pass of semantic analysis. state.parse_file() # TODO: state.fix_suppressed_dependencies()? # Run remaining passes of semantic analysis. try: state.semantic_analysis() except CompileError as err: # TODO: What if there are multiple changed modules? # There was a blocking error, so module AST is incomplete. Restore old modules. manager.modules.clear() manager.modules.update(old_modules) raise err state.semantic_analysis_pass_three() state.semantic_analysis_apply_patches() # Merge old and new ASTs. assert state.tree is not None, "file must be at least parsed" new_modules = {id: state.tree} # type: Dict[str, Optional[MypyFile]] replace_modules_with_new_variants(manager, graph, old_modules, new_modules) # Perform type checking. state.type_check_first_pass() # TODO: state.type_check_second_pass()? state.finish_passes() # TODO: state.write_cache()? # TODO: state.mark_as_rechecked()? # TODO: Store new State in graph, as it has updated dependencies etc. graph[id] = state return new_modules, graph
def update_module_isolated(module: str, path: str, manager: BuildManager, previous_modules: Dict[str, str], graph: Graph, force_removed: bool) -> UpdateResult: """Build a new version of one changed module only. Don't propagate changes to elsewhere in the program. Raise CompileError on encountering a blocking error. Args: module: Changed module (modified, created or deleted) path: Path of the changed module manager: Build manager graph: Build graph force_removed: If True, consider the module removed from the build even it the file exists Returns a named tuple describing the result (see above for details). """ if module not in graph: manager.log_fine_grained('new module %r' % module) if not manager.fscache.isfile(path) or force_removed: delete_module(module, path, graph, manager) return NormalUpdate(module, path, [], None) sources = get_sources(manager.fscache, previous_modules, [(module, path)]) if module in manager.missing_modules: manager.missing_modules.remove(module) orig_module = module orig_state = graph.get(module) orig_tree = manager.modules.get(module) def restore(ids: List[str]) -> None: # For each of the modules in ids, restore that id's old # manager.modules and graphs entries. (Except for the original # module, this means deleting them.) for id in ids: if id == orig_module and orig_tree: manager.modules[id] = orig_tree elif id in manager.modules: del manager.modules[id] if id == orig_module and orig_state: graph[id] = orig_state elif id in graph: del graph[id] new_modules = [] # type: List[State] try: if module in graph: del graph[module] load_graph(sources, manager, graph, new_modules) except CompileError as err: # Parse error somewhere in the program -- a blocker assert err.module_with_blocker restore([module] + [st.id for st in new_modules]) return BlockedUpdate(err.module_with_blocker, path, [], err.messages) # Reparsing the file may have brought in dependencies that we # didn't have before. Make sure that they are loaded to restore # the invariant that a module having a loaded tree implies that # its dependencies do as well. ensure_trees_loaded(manager, graph, graph[module].dependencies) # Find any other modules brought in by imports. changed_modules = [(st.id, st.xpath) for st in new_modules] # If there are multiple modules to process, only process one of them and return # the remaining ones to the caller. if len(changed_modules) > 1: # As an optimization, look for a module that imports no other changed modules. module, path = find_relative_leaf_module(changed_modules, graph) changed_modules.remove((module, path)) remaining_modules = changed_modules # The remaining modules haven't been processed yet so drop them. restore([id for id, _ in remaining_modules]) manager.log_fine_grained('--> %r (newly imported)' % module) else: remaining_modules = [] state = graph[module] # Process the changed file. state.parse_file() assert state.tree is not None, "file must be at least parsed" t0 = time.time() # TODO: state.fix_suppressed_dependencies()? if module == 'typing': # We need to manually add typing aliases to builtins, like we # do in process_stale_scc. Because this can't be done until # builtins is also loaded, there isn't an obvious way to # refactor this. manager.semantic_analyzer.add_builtin_aliases(state.tree) try: state.semantic_analysis() except CompileError as err: # There was a blocking error, so module AST is incomplete. Restore old modules. restore([module]) return BlockedUpdate(module, path, remaining_modules, err.messages) state.semantic_analysis_pass_three() state.semantic_analysis_apply_patches() # Merge old and new ASTs. new_modules_dict = {module: state.tree} # type: Dict[str, Optional[MypyFile]] replace_modules_with_new_variants(manager, graph, {orig_module: orig_tree}, new_modules_dict) t1 = time.time() # Perform type checking. state.type_checker().reset() state.type_check_first_pass() state.type_check_second_pass() t2 = time.time() state.compute_fine_grained_deps() t3 = time.time() state.finish_passes() t4 = time.time() manager.add_stats( semanal_time=t1 - t0, typecheck_time=t2 - t1, deps_time=t3 - t2, finish_passes_time=t4 - t3) graph[module] = state return NormalUpdate(module, path, remaining_modules, state.tree)