def preserve_full_cache(graph: Graph, manager: BuildManager) -> SavedCache:
"""Preserve every module with an AST in the graph, including modules with errors."""
saved_cache = {}
for id, state in graph.items():
assert state.id == id
if state.tree is not None:
meta = state.meta
if meta is None:
# No metadata, likely because of an error. We still want to retain the AST.
# There is no corresponding JSON so create partial "memory-only" metadata.
assert state.path
dep_prios = state.dependency_priorities()
meta = memory_only_cache_meta(
id,
state.path,
state.dependencies,
state.suppressed,
list(state.child_modules),
dep_prios,
state.source_hash,
state.ignore_all,
manager)
else:
meta = meta._replace(memory_only=True)
saved_cache[id] = (meta, state.tree, state.type_map())
return saved_cache
def get_sources(graph: Graph,
changed_modules: List[Tuple[str, str]]) -> List[BuildSource]:
sources = [BuildSource(st.path, st.id, None) for st in graph.values()]
for id, path in changed_modules:
if id not in graph:
sources.append(BuildSource(path, id, None))
return sources
def refresh_suppressed_submodules(module: str, path: Optional[str],
deps: Dict[str, Set[str]], graph: Graph,
fscache: FileSystemCache) -> None:
"""Look for submodules that are now suppressed in target package.
If a submodule a.b gets added, we need to mark it as suppressed
in modules that contain "from a import b". Previously we assumed
that 'a.b' is not a module but a regular name.
This is only relevant when following imports normally.
Args:
module: target package in which to look for submodules
path: path of the module
"""
if path is None or not path.endswith(INIT_SUFFIXES):
# Only packages have submodules.
return
# Find any submodules present in the directory.
pkgdir = os.path.dirname(path)
for fnam in fscache.listdir(pkgdir):
if (not fnam.endswith(('.py', '.pyi')) or fnam.startswith("__init__.")
or fnam.count('.') != 1):
continue
shortname = fnam.split('.')[0]
submodule = module + '.' + shortname
trigger = make_trigger(submodule)
if trigger in deps:
for dep in deps[trigger]:
# TODO: <...> deps, etc.
state = graph.get(dep)
if not state:
# Maybe it's a non-top-level target. We only care about the module.
dep_module = module_prefix(graph, dep)
if dep_module is not None:
state = graph.get(dep_module)
if state:
tree = state.tree
assert tree # TODO: What if doesn't exist?
for imp in tree.imports:
if isinstance(imp, ImportFrom):
if (imp.id == module
and any(name == shortname
for name, _ in imp.names)):
# TODO: Only if does not exist already
state.suppressed.append(submodule)
state.suppressed_set.add(submodule)
def get_module_to_path_map(graph: Graph) -> Dict[str, str]:
return {module: node.xpath
for module, node in graph.items()}
def update_module_isolated(module: str,
path: str,
manager: BuildManager,
previous_modules: Dict[str, str],
graph: Graph,
force_removed: bool) -> UpdateResult:
"""Build a new version of one changed module only.
Don't propagate changes to elsewhere in the program. Raise CompileError on
encountering a blocking error.
Args:
module: Changed module (modified, created or deleted)
path: Path of the changed module
manager: Build manager
graph: Build graph
force_removed: If True, consider the module removed from the build even it the
file exists
Returns a named tuple describing the result (see above for details).
"""
if module not in graph:
manager.log_fine_grained('new module %r' % module)
if not manager.fscache.isfile(path) or force_removed:
delete_module(module, path, graph, manager)
return NormalUpdate(module, path, [], None)
sources = get_sources(manager.fscache, previous_modules, [(module, path)])
if module in manager.missing_modules:
manager.missing_modules.remove(module)
orig_module = module
orig_state = graph.get(module)
orig_tree = manager.modules.get(module)
def restore(ids: List[str]) -> None:
# For each of the modules in ids, restore that id's old
# manager.modules and graphs entries. (Except for the original
# module, this means deleting them.)
for id in ids:
if id == orig_module and orig_tree:
manager.modules[id] = orig_tree
elif id in manager.modules:
del manager.modules[id]
if id == orig_module and orig_state:
graph[id] = orig_state
elif id in graph:
del graph[id]
new_modules = [] # type: List[State]
try:
if module in graph:
del graph[module]
load_graph(sources, manager, graph, new_modules)
except CompileError as err:
# Parse error somewhere in the program -- a blocker
assert err.module_with_blocker
restore([module] + [st.id for st in new_modules])
return BlockedUpdate(err.module_with_blocker, path, [], err.messages)
# Reparsing the file may have brought in dependencies that we
# didn't have before. Make sure that they are loaded to restore
# the invariant that a module having a loaded tree implies that
# its dependencies do as well.
ensure_trees_loaded(manager, graph, graph[module].dependencies)
# Find any other modules brought in by imports.
changed_modules = [(st.id, st.xpath) for st in new_modules]
# If there are multiple modules to process, only process one of them and return
# the remaining ones to the caller.
if len(changed_modules) > 1:
# As an optimization, look for a module that imports no other changed modules.
module, path = find_relative_leaf_module(changed_modules, graph)
changed_modules.remove((module, path))
remaining_modules = changed_modules
# The remaining modules haven't been processed yet so drop them.
restore([id for id, _ in remaining_modules])
manager.log_fine_grained('--> %r (newly imported)' % module)
else:
remaining_modules = []
state = graph[module]
# Process the changed file.
state.parse_file()
assert state.tree is not None, "file must be at least parsed"
t0 = time.time()
# TODO: state.fix_suppressed_dependencies()?
try:
semantic_analysis_for_scc(graph, [state.id], manager.errors)
except CompileError as err:
# There was a blocking error, so module AST is incomplete. Restore old modules.
restore([module])
return BlockedUpdate(module, path, remaining_modules, err.messages)
# Merge old and new ASTs.
new_modules_dict = {module: state.tree} # type: Dict[str, Optional[MypyFile]]
replace_modules_with_new_variants(manager, graph, {orig_module: orig_tree}, new_modules_dict)
t1 = time.time()
# Perform type checking.
state.type_checker().reset()
state.type_check_first_pass()
state.type_check_second_pass()
t2 = time.time()
state.finish_passes()
t3 = time.time()
manager.add_stats(
semanal_time=t1 - t0,
typecheck_time=t2 - t1,
finish_passes_time=t3 - t2)
graph[module] = state
return NormalUpdate(module, path, remaining_modules, state.tree)
def extract_type_maps(graph: Graph) -> Dict[str, Dict[Expression, Type]]:
return {id: state.type_map() for id, state in graph.items()}
def extract_type_maps(graph: Graph) -> Dict[str, Dict[Expression, Type]]:
# This is used to export information used only by the testmerge harness.
return {id: state.type_map() for id, state in graph.items() if state.tree}
def get_module_to_path_map(graph: Graph) -> Dict[str, str]:
return {module: node.xpath
for module, node in graph.items()}
def update_module_isolated(module: str,
path: str,
manager: BuildManager,
previous_modules: Dict[str, str],
graph: Graph,
force_removed: bool) -> UpdateResult:
"""Build a new version of one changed module only.
Don't propagate changes to elsewhere in the program. Raise CompileError on
encountering a blocking error.
Args:
module: Changed module (modified, created or deleted)
path: Path of the changed module
manager: Build manager
graph: Build graph
force_removed: If True, consider the module removed from the build even it the
file exists
Returns a named tuple describing the result (see above for details).
"""
if module not in graph:
manager.log_fine_grained('new module %r' % module)
if not manager.fscache.isfile(path) or force_removed:
delete_module(module, path, graph, manager)
return NormalUpdate(module, path, [], None)
sources = get_sources(manager.fscache, previous_modules, [(module, path)])
if module in manager.missing_modules:
manager.missing_modules.remove(module)
orig_module = module
orig_state = graph.get(module)
orig_tree = manager.modules.get(module)
def restore(ids: List[str]) -> None:
# For each of the modules in ids, restore that id's old
# manager.modules and graphs entries. (Except for the original
# module, this means deleting them.)
for id in ids:
if id == orig_module and orig_tree:
manager.modules[id] = orig_tree
elif id in manager.modules:
del manager.modules[id]
if id == orig_module and orig_state:
graph[id] = orig_state
elif id in graph:
del graph[id]
new_modules = [] # type: List[State]
try:
if module in graph:
del graph[module]
load_graph(sources, manager, graph, new_modules)
except CompileError as err:
# Parse error somewhere in the program -- a blocker
assert err.module_with_blocker
restore([module] + [st.id for st in new_modules])
return BlockedUpdate(err.module_with_blocker, path, [], err.messages)
# Reparsing the file may have brought in dependencies that we
# didn't have before. Make sure that they are loaded to restore
# the invariant that a module having a loaded tree implies that
# its dependencies do as well.
ensure_trees_loaded(manager, graph, graph[module].dependencies)
# Find any other modules brought in by imports.
changed_modules = [(st.id, st.xpath) for st in new_modules]
# If there are multiple modules to process, only process one of them and return
# the remaining ones to the caller.
if len(changed_modules) > 1:
# As an optimization, look for a module that imports no other changed modules.
module, path = find_relative_leaf_module(changed_modules, graph)
changed_modules.remove((module, path))
remaining_modules = changed_modules
# The remaining modules haven't been processed yet so drop them.
restore([id for id, _ in remaining_modules])
manager.log_fine_grained('--> %r (newly imported)' % module)
else:
remaining_modules = []
state = graph[module]
# Process the changed file.
state.parse_file()
assert state.tree is not None, "file must be at least parsed"
t0 = time.time()
# TODO: state.fix_suppressed_dependencies()?
if module == 'typing':
# We need to manually add typing aliases to builtins, like we
# do in process_stale_scc. Because this can't be done until
# builtins is also loaded, there isn't an obvious way to
# refactor this.
manager.semantic_analyzer.add_builtin_aliases(state.tree)
try:
state.semantic_analysis()
except CompileError as err:
# There was a blocking error, so module AST is incomplete. Restore old modules.
restore([module])
return BlockedUpdate(module, path, remaining_modules, err.messages)
state.semantic_analysis_pass_three()
state.semantic_analysis_apply_patches()
# Merge old and new ASTs.
new_modules_dict = {module: state.tree} # type: Dict[str, Optional[MypyFile]]
replace_modules_with_new_variants(manager, graph, {orig_module: orig_tree}, new_modules_dict)
t1 = time.time()
# Perform type checking.
state.type_checker().reset()
state.type_check_first_pass()
state.type_check_second_pass()
t2 = time.time()
state.compute_fine_grained_deps()
t3 = time.time()
state.finish_passes()
t4 = time.time()
manager.add_stats(
semanal_time=t1 - t0,
typecheck_time=t2 - t1,
deps_time=t3 - t2,
finish_passes_time=t4 - t3)
graph[module] = state
return NormalUpdate(module, path, remaining_modules, state.tree)
def refresh_suppressed_submodules(
module: str, path: Optional[str], deps: Dict[str, Set[str]],
graph: Graph, fscache: FileSystemCache,
refresh_file: Callable[[str, str], List[str]]) -> Optional[List[str]]:
"""Look for submodules that are now suppressed in target package.
If a submodule a.b gets added, we need to mark it as suppressed
in modules that contain "from a import b". Previously we assumed
that 'a.b' is not a module but a regular name.
This is only relevant when following imports normally.
Args:
module: target package in which to look for submodules
path: path of the module
refresh_file: function that reads the AST of a module (returns error messages)
Return a list of errors from refresh_file() if it was called. If the
return value is None, we didn't call refresh_file().
"""
messages = None
if path is None or not path.endswith(INIT_SUFFIXES):
# Only packages have submodules.
return None
# Find any submodules present in the directory.
pkgdir = os.path.dirname(path)
for fnam in fscache.listdir(pkgdir):
if (not fnam.endswith(('.py', '.pyi')) or fnam.startswith("__init__.")
or fnam.count('.') != 1):
continue
shortname = fnam.split('.')[0]
submodule = module + '.' + shortname
trigger = make_trigger(submodule)
# We may be missing the required fine-grained deps.
ensure_deps_loaded(module, deps, graph)
if trigger in deps:
for dep in deps[trigger]:
# We can ignore <...> deps since a submodule can't trigger any.
state = graph.get(dep)
if not state:
# Maybe it's a non-top-level target. We only care about the module.
dep_module = module_prefix(graph, dep)
if dep_module is not None:
state = graph.get(dep_module)
if state:
# Is the file may missing an AST in case it's read from cache?
if state.tree is None:
# Create AST for the file. This may produce some new errors
# that we need to propagate.
assert state.path is not None
messages = refresh_file(state.id, state.path)
tree = state.tree
assert tree # Will be fine, due to refresh_file() above
for imp in tree.imports:
if isinstance(imp, ImportFrom):
if (imp.id == module
and any(name == shortname
for name, _ in imp.names)
and submodule not in state.suppressed_set):
state.suppressed.append(submodule)
state.suppressed_set.add(submodule)
return messages