def find_targets_recursive(
manager: BuildManager,
graph: Graph,
triggers: Set[str],
deps: Dict[str, Set[str]],
up_to_date_modules: Set[str]) -> Tuple[Dict[str, Set[FineGrainedDeferredNode]],
Set[str], Set[TypeInfo]]:
"""Find names of all targets that need to reprocessed, given some triggers.
Returns: A tuple containing a:
* Dictionary from module id to a set of stale targets.
* A set of module ids for unparsed modules with stale targets.
"""
result = {} # type: Dict[str, Set[FineGrainedDeferredNode]]
worklist = triggers
processed = set() # type: Set[str]
stale_protos = set() # type: Set[TypeInfo]
unloaded_files = set() # type: Set[str]
# Find AST nodes corresponding to each target.
#
# TODO: Don't rely on a set, since the items are in an unpredictable order.
while worklist:
processed |= worklist
current = worklist
worklist = set()
for target in current:
if target.startswith('<'):
module_id = module_prefix(graph, trigger_to_target(target))
if module_id:
ensure_deps_loaded(module_id, deps, graph)
worklist |= deps.get(target, set()) - processed
else:
module_id = module_prefix(graph, target)
if module_id is None:
# Deleted module.
continue
if module_id in up_to_date_modules:
# Already processed.
continue
if (module_id not in manager.modules
or manager.modules[module_id].is_cache_skeleton):
# We haven't actually parsed and checked the module, so we don't have
# access to the actual nodes.
# Add it to the queue of files that need to be processed fully.
unloaded_files.add(module_id)
continue
if module_id not in result:
result[module_id] = set()
manager.log_fine_grained('process: %s' % target)
deferred, stale_proto = lookup_target(manager, target)
if stale_proto:
stale_protos.add(stale_proto)
result[module_id].update(deferred)
return result, unloaded_files, stale_protos
def propagate_changes_using_dependencies(
manager: BuildManager, graph: Dict[str, State], deps: Dict[str,
Set[str]],
triggered: Set[str], up_to_date_modules: Set[str],
targets_with_errors: Set[str]) -> List[Tuple[str, str]]:
"""Transitively rechecks targets based on triggers and the dependency map.
Returns a list (module id, path) tuples representing modules that contain
a target that needs to be reprocessed but that has not been parsed yet."""
num_iter = 0
remaining_modules = [] # type: List[Tuple[str, str]]
# Propagate changes until nothing visible has changed during the last
# iteration.
while triggered or targets_with_errors:
num_iter += 1
if num_iter > MAX_ITER:
raise RuntimeError(
'Max number of iterations (%d) reached (endless loop?)' %
MAX_ITER)
todo, unloaded, stale_protos = find_targets_recursive(
manager, graph, triggered, deps, up_to_date_modules)
# TODO: we sort to make it deterministic, but this is *incredibly* ad hoc
remaining_modules.extend(
(id, graph[id].xpath) for id in sorted(unloaded))
# Also process targets that used to have errors, as otherwise some
# errors might be lost.
for target in targets_with_errors:
id = module_prefix(graph, target)
if id is not None and id not in up_to_date_modules:
if id not in todo:
todo[id] = set()
manager.log_fine_grained('process target with error: %s' %
target)
more_nodes, _ = lookup_target(manager, target)
todo[id].update(more_nodes)
triggered = set()
# First invalidate subtype caches in all stale protocols.
# We need to do this to avoid false negatives if the protocol itself is
# unchanged, but was marked stale because its sub- (or super-) type changed.
for info in stale_protos:
TypeState.reset_subtype_caches_for(info)
# Then fully reprocess all targets.
# TODO: Preserve order (set is not optimal)
for id, nodes in sorted(todo.items(), key=lambda x: x[0]):
assert id not in up_to_date_modules
triggered |= reprocess_nodes(manager, graph, id, nodes, deps)
# Changes elsewhere may require us to reprocess modules that were
# previously considered up to date. For example, there may be a
# dependency loop that loops back to an originally processed module.
up_to_date_modules = set()
targets_with_errors = set()
if is_verbose(manager):
manager.log_fine_grained('triggered: %r' % list(triggered))
return remaining_modules
def find_targets_recursive(
manager: BuildManager,
graph: Graph,
triggers: Set[str],
deps: Dict[str, Set[str]],
up_to_date_modules: Set[str]) -> Tuple[Dict[str, Set[FineGrainedDeferredNode]],
Set[str], Set[TypeInfo]]:
"""Find names of all targets that need to reprocessed, given some triggers.
Returns: A tuple containing a:
* Dictionary from module id to a set of stale targets.
* A set of module ids for unparsed modules with stale targets.
"""
result = {} # type: Dict[str, Set[FineGrainedDeferredNode]]
worklist = triggers
processed = set() # type: Set[str]
stale_protos = set() # type: Set[TypeInfo]
unloaded_files = set() # type: Set[str]
# Find AST nodes corresponding to each target.
#
# TODO: Don't rely on a set, since the items are in an unpredictable order.
while worklist:
processed |= worklist
current = worklist
worklist = set()
for target in current:
if target.startswith('<'):
module_id = module_prefix(graph, trigger_to_target(target))
if module_id:
ensure_deps_loaded(module_id, deps, graph)
worklist |= deps.get(target, set()) - processed
else:
module_id = module_prefix(graph, target)
if module_id is None:
# Deleted module.
continue
if module_id in up_to_date_modules:
# Already processed.
continue
if (module_id not in manager.modules
or manager.modules[module_id].is_cache_skeleton):
# We haven't actually parsed and checked the module, so we don't have
# access to the actual nodes.
# Add it to the queue of files that need to be processed fully.
unloaded_files.add(module_id)
continue
if module_id not in result:
result[module_id] = set()
manager.log_fine_grained('process: %s' % target)
deferred, stale_proto = lookup_target(manager, target)
if stale_proto:
stale_protos.add(stale_proto)
result[module_id].update(deferred)
return result, unloaded_files, stale_protos
def ensure_trees_loaded(manager: BuildManager, graph: Dict[str, State],
initial: Sequence[str]) -> None:
"""Ensure that the modules in initial and their deps have loaded trees."""
to_process = find_unloaded_deps(manager, graph, initial)
if to_process:
if is_verbose(manager):
manager.log_fine_grained("Calling process_fresh_modules on set of size {} ({})".format(
len(to_process), sorted(to_process)))
process_fresh_modules(graph, to_process, manager)
def ensure_trees_loaded(manager: BuildManager, graph: Dict[str, State],
initial: Sequence[str]) -> None:
"""Ensure that the modules in initial and their deps have loaded trees."""
to_process = find_unloaded_deps(manager, graph, initial)
if to_process:
if is_verbose(manager):
manager.log_fine_grained("Calling process_fresh_modules on set of size {} ({})".format(
len(to_process), sorted(to_process)))
process_fresh_modules(graph, to_process, manager)
def propagate_changes_using_dependencies(
manager: BuildManager,
graph: Dict[str, State],
deps: Dict[str, Set[str]],
triggered: Set[str],
up_to_date_modules: Set[str],
targets_with_errors: Set[str]) -> List[Tuple[str, str]]:
"""Transitively rechecks targets based on triggers and the dependency map.
Returns a list (module id, path) tuples representing modules that contain
a target that needs to be reprocessed but that has not been parsed yet."""
num_iter = 0
remaining_modules = [] # type: List[Tuple[str, str]]
# Propagate changes until nothing visible has changed during the last
# iteration.
while triggered or targets_with_errors:
num_iter += 1
if num_iter > MAX_ITER:
raise RuntimeError('Max number of iterations (%d) reached (endless loop?)' % MAX_ITER)
todo, unloaded, stale_protos = find_targets_recursive(manager, graph,
triggered, deps, up_to_date_modules)
# TODO: we sort to make it deterministic, but this is *incredibly* ad hoc
remaining_modules.extend((id, graph[id].xpath) for id in sorted(unloaded))
# Also process targets that used to have errors, as otherwise some
# errors might be lost.
for target in targets_with_errors:
id = module_prefix(graph, target)
if id is not None and id not in up_to_date_modules:
if id not in todo:
todo[id] = set()
manager.log_fine_grained('process target with error: %s' % target)
more_nodes, _ = lookup_target(manager, target)
todo[id].update(more_nodes)
triggered = set()
# First invalidate subtype caches in all stale protocols.
# We need to do this to avoid false negatives if the protocol itself is
# unchanged, but was marked stale because its sub- (or super-) type changed.
for info in stale_protos:
TypeState.reset_subtype_caches_for(info)
# Then fully reprocess all targets.
# TODO: Preserve order (set is not optimal)
for id, nodes in sorted(todo.items(), key=lambda x: x[0]):
assert id not in up_to_date_modules
triggered |= reprocess_nodes(manager, graph, id, nodes, deps)
# Changes elsewhere may require us to reprocess modules that were
# previously considered up to date. For example, there may be a
# dependency loop that loops back to an originally processed module.
up_to_date_modules = set()
targets_with_errors = set()
if is_verbose(manager):
manager.log_fine_grained('triggered: %r' % list(triggered))
return remaining_modules
def propagate_changes_using_dependencies(
manager: BuildManager, graph: Dict[str, State], deps: Dict[str,
Set[str]],
triggered: Set[str], up_to_date_modules: Set[str],
targets_with_errors: Set[str]) -> List[Tuple[str, str]]:
"""Transitively rechecks targets based on triggers and the dependency map.
Returns a list (module id, path) tuples representing modules that contain
a target that needs to be reprocessed but that has not been parsed yet."""
# TODO: Multiple type checking passes
num_iter = 0
remaining_modules = []
# Propagate changes until nothing visible has changed during the last
# iteration.
while triggered or targets_with_errors:
num_iter += 1
if num_iter > MAX_ITER:
raise RuntimeError(
'Max number of iterations (%d) reached (endless loop?)' %
MAX_ITER)
todo = find_targets_recursive(manager, triggered, deps,
up_to_date_modules)
# Also process targets that used to have errors, as otherwise some
# errors might be lost.
for target in targets_with_errors:
id = module_prefix(manager.modules, target)
if id is not None and id not in up_to_date_modules:
if id not in todo:
todo[id] = set()
manager.log_fine_grained('process target with error: %s' %
target)
todo[id].update(lookup_target(manager, target))
triggered = set()
# TODO: Preserve order (set is not optimal)
for id, nodes in sorted(todo.items(), key=lambda x: x[0]):
assert id not in up_to_date_modules
if manager.modules[id].is_cache_skeleton:
# We have only loaded the cache for this file, not the actual file,
# so we can't access the nodes to reprocess.
# Add it to the queue of files that need to be processed fully.
remaining_modules.append((id, manager.modules[id].path))
else:
triggered |= reprocess_nodes(manager, graph, id, nodes, deps)
# Changes elsewhere may require us to reprocess modules that were
# previously considered up to date. For example, there may be a
# dependency loop that loops back to an originally processed module.
up_to_date_modules = set()
targets_with_errors = set()
if is_verbose(manager):
manager.log_fine_grained('triggered: %r' % list(triggered))
return remaining_modules
def propagate_changes_using_dependencies(
manager: BuildManager,
graph: Dict[str, State],
deps: Dict[str, Set[str]],
triggered: Set[str],
up_to_date_modules: Set[str],
targets_with_errors: Set[str]) -> List[Tuple[str, str]]:
"""Transitively rechecks targets based on triggers and the dependency map.
Returns a list (module id, path) tuples representing modules that contain
a target that needs to be reprocessed but that has not been parsed yet."""
num_iter = 0
remaining_modules = []
# Propagate changes until nothing visible has changed during the last
# iteration.
while triggered or targets_with_errors:
num_iter += 1
if num_iter > MAX_ITER:
raise RuntimeError('Max number of iterations (%d) reached (endless loop?)' % MAX_ITER)
todo = find_targets_recursive(manager, triggered, deps, up_to_date_modules)
# Also process targets that used to have errors, as otherwise some
# errors might be lost.
for target in targets_with_errors:
id = module_prefix(manager.modules, target)
if id is not None and id not in up_to_date_modules:
if id not in todo:
todo[id] = set()
manager.log_fine_grained('process target with error: %s' % target)
todo[id].update(lookup_target(manager, target))
triggered = set()
# TODO: Preserve order (set is not optimal)
for id, nodes in sorted(todo.items(), key=lambda x: x[0]):
assert id not in up_to_date_modules
if manager.modules[id].is_cache_skeleton:
# We have only loaded the cache for this file, not the actual file,
# so we can't access the nodes to reprocess.
# Add it to the queue of files that need to be processed fully.
remaining_modules.append((id, manager.modules[id].path))
else:
triggered |= reprocess_nodes(manager, graph, id, nodes, deps)
# Changes elsewhere may require us to reprocess modules that were
# previously considered up to date. For example, there may be a
# dependency loop that loops back to an originally processed module.
up_to_date_modules = set()
targets_with_errors = set()
if is_verbose(manager):
manager.log_fine_grained('triggered: %r' % list(triggered))
return remaining_modules
def delete_module(module_id: str, graph: Graph, manager: BuildManager) -> None:
manager.log_fine_grained('delete module %r' % module_id)
# TODO: Remove deps for the module (this only affects memory use, not correctness)
if module_id in graph:
del graph[module_id]
if module_id in manager.modules:
del manager.modules[module_id]
components = module_id.split('.')
if len(components) > 1:
# Delete reference to module in parent module.
parent_id = '.'.join(components[:-1])
# If parent module is ignored, it won't be included in the modules dictionary.
if parent_id in manager.modules:
parent = manager.modules[parent_id]
if components[-1] in parent.names:
del parent.names[components[-1]]
def delete_module(module_id: str,
graph: Graph,
manager: BuildManager) -> None:
manager.log_fine_grained('delete module %r' % module_id)
# TODO: Remove deps for the module (this only affects memory use, not correctness)
if module_id in graph:
del graph[module_id]
if module_id in manager.modules:
del manager.modules[module_id]
components = module_id.split('.')
if len(components) > 1:
# Delete reference to module in parent module.
parent_id = '.'.join(components[:-1])
# If parent module is ignored, it won't be included in the modules dictionary.
if parent_id in manager.modules:
parent = manager.modules[parent_id]
if components[-1] in parent.names:
del parent.names[components[-1]]
def delete_module(module_id: str, path: str, graph: Graph,
manager: BuildManager) -> None:
manager.log_fine_grained('delete module %r' % module_id)
# TODO: Remove deps for the module (this only affects memory use, not correctness)
if module_id in graph:
del graph[module_id]
if module_id in manager.modules:
del manager.modules[module_id]
components = module_id.split('.')
if len(components) > 1:
# Delete reference to module in parent module.
parent_id = '.'.join(components[:-1])
# If parent module is ignored, it won't be included in the modules dictionary.
if parent_id in manager.modules:
parent = manager.modules[parent_id]
if components[-1] in parent.names:
del parent.names[components[-1]]
# If the module is removed from the build but still exists, then
# we mark it as missing so that it will get picked up by import from still.
if manager.fscache.isfile(path):
manager.missing_modules.add(module_id)
def delete_module(module_id: str, graph: Dict[str, State],
manager: BuildManager) -> Dict[str, State]:
manager.log_fine_grained('delete module %r' % module_id)
# TODO: Deletion of a package
# TODO: Remove deps for the module (this only affects memory use, not correctness)
assert module_id not in graph
new_graph = graph.copy()
if module_id in manager.modules:
del manager.modules[module_id]
if module_id in manager.saved_cache:
del manager.saved_cache[module_id]
components = module_id.split('.')
if len(components) > 1:
# Delete reference to module in parent module.
parent_id = '.'.join(components[:-1])
# If parent module is ignored, it won't be included in the modules dictionary.
if parent_id in manager.modules:
parent = manager.modules[parent_id]
if components[-1] in parent.names:
del parent.names[components[-1]]
return new_graph
def find_targets_recursive(
manager: BuildManager,
triggers: Set[str],
deps: Dict[str, Set[str]],
modules: Dict[str, MypyFile],
up_to_date_modules: Set[str]) -> Dict[str, Set[DeferredNode]]:
"""Find names of all targets that need to reprocessed, given some triggers.
Returns: Dictionary from module id to a set of stale targets.
"""
result = {} # type: Dict[str, Set[DeferredNode]]
worklist = triggers
processed = set() # type: Set[str]
# Find AST nodes corresponding to each target.
#
# TODO: Don't rely on a set, since the items are in an unpredictable order.
while worklist:
processed |= worklist
current = worklist
worklist = set()
for target in current:
if target.startswith('<'):
worklist |= deps.get(target, set()) - processed
else:
module_id = module_prefix(modules, target)
if module_id is None:
# Deleted module.
continue
if module_id in up_to_date_modules:
# Already processed.
continue
if module_id not in result:
result[module_id] = set()
manager.log_fine_grained('process %s' % target)
deferred = lookup_target(modules, target)
result[module_id].update(deferred)
return result
def find_targets_recursive(
manager: BuildManager,
triggers: Set[str],
deps: Dict[str, Set[str]],
up_to_date_modules: Set[str]) -> Dict[str, Set[DeferredNode]]:
"""Find names of all targets that need to reprocessed, given some triggers.
Returns: Dictionary from module id to a set of stale targets.
"""
result = {} # type: Dict[str, Set[DeferredNode]]
worklist = triggers
processed = set() # type: Set[str]
# Find AST nodes corresponding to each target.
#
# TODO: Don't rely on a set, since the items are in an unpredictable order.
while worklist:
processed |= worklist
current = worklist
worklist = set()
for target in current:
if target.startswith('<'):
worklist |= deps.get(target, set()) - processed
else:
module_id = module_prefix(manager.modules, target)
if module_id is None:
# Deleted module.
continue
if module_id in up_to_date_modules:
# Already processed.
continue
if module_id not in result:
result[module_id] = set()
manager.log_fine_grained('process: %s' % target)
deferred = lookup_target(manager, target)
result[module_id].update(deferred)
return result
def delete_module(module_id: str,
path: str,
graph: Graph,
manager: BuildManager) -> None:
manager.log_fine_grained('delete module %r' % module_id)
# TODO: Remove deps for the module (this only affects memory use, not correctness)
if module_id in graph:
del graph[module_id]
if module_id in manager.modules:
del manager.modules[module_id]
components = module_id.split('.')
if len(components) > 1:
# Delete reference to module in parent module.
parent_id = '.'.join(components[:-1])
# If parent module is ignored, it won't be included in the modules dictionary.
if parent_id in manager.modules:
parent = manager.modules[parent_id]
if components[-1] in parent.names:
del parent.names[components[-1]]
# If the module is removed from the build but still exists, then
# we mark it as missing so that it will get picked up by import from still.
if manager.fscache.isfile(path):
manager.missing_modules.add(module_id)
def reprocess_nodes(manager: BuildManager, graph: Dict[str, State],
module_id: str, nodeset: Set[DeferredNode],
deps: Dict[str, Set[str]]) -> Set[str]:
"""Reprocess a set of nodes within a single module.
Return fired triggers.
"""
if module_id not in graph:
manager.log_fine_grained(
'%s not in graph (blocking errors or deleted?)' % module_id)
return set()
file_node = manager.modules[module_id]
old_symbols = find_symbol_tables_recursive(file_node.fullname(),
file_node.names)
old_symbols = {name: names.copy() for name, names in old_symbols.items()}
old_symbols_snapshot = snapshot_symbol_table(file_node.fullname(),
file_node.names)
def key(node: DeferredNode) -> int:
# Unlike modules which are sorted by name within SCC,
# nodes within the same module are sorted by line number, because
# this is how they are processed in normal mode.
return node.node.line
nodes = sorted(nodeset, key=key)
# TODO: ignore_all argument to set_file_ignored_lines
manager.errors.set_file_ignored_lines(file_node.path,
file_node.ignored_lines)
# Strip semantic analysis information.
for deferred in nodes:
strip_target(deferred.node)
semantic_analyzer = manager.semantic_analyzer
patches = [] # type: List[Tuple[int, Callable[[], None]]]
# Second pass of semantic analysis. We don't redo the first pass, because it only
# does local things that won't go stale.
for deferred in nodes:
with semantic_analyzer.file_context(
file_node=file_node,
fnam=file_node.path,
options=manager.options,
active_type=deferred.active_typeinfo):
manager.semantic_analyzer.refresh_partial(deferred.node, patches)
# Third pass of semantic analysis.
for deferred in nodes:
with semantic_analyzer.file_context(
file_node=file_node,
fnam=file_node.path,
options=manager.options,
active_type=deferred.active_typeinfo):
manager.semantic_analyzer_pass3.refresh_partial(
deferred.node, patches)
apply_semantic_analyzer_patches(patches)
# Merge symbol tables to preserve identities of AST nodes. The file node will remain
# the same, but other nodes may have been recreated with different identities, such as
# NamedTuples defined using assignment statements.
new_symbols = find_symbol_tables_recursive(file_node.fullname(),
file_node.names)
for name in old_symbols:
if name in new_symbols:
merge_asts(file_node, old_symbols[name], file_node,
new_symbols[name])
# Type check.
checker = graph[module_id].type_checker()
checker.reset()
# We seem to need additional passes in fine-grained incremental mode.
checker.pass_num = 0
checker.last_pass = 3
more = checker.check_second_pass(nodes)
while more:
more = False
if graph[module_id].type_checker().check_second_pass():
more = True
new_symbols_snapshot = snapshot_symbol_table(file_node.fullname(),
file_node.names)
# Check if any attribute types were changed and need to be propagated further.
changed = compare_symbol_table_snapshots(file_node.fullname(),
old_symbols_snapshot,
new_symbols_snapshot)
new_triggered = {make_trigger(name) for name in changed}
# Dependencies may have changed.
update_deps(module_id, nodes, graph, deps, manager.options)
# Report missing imports.
verify_dependencies(graph[module_id], manager)
return new_triggered
def update_single_isolated(module: str, path: str, manager: BuildManager,
previous_modules: Dict[str, str],
graph: Graph) -> UpdateResult:
"""Build a new version of one changed module only.
Don't propagate changes to elsewhere in the program. Raise CompleError on
encountering a blocking error.
Args:
module: Changed module (modified, created or deleted)
path: Path of the changed module
manager: Build manager
graph: Build graph
Returns a named tuple describing the result (see above for details).
"""
if module in manager.modules:
assert_equivalent_paths(path, manager.modules[module].path)
else:
manager.log_fine_grained('new module %r' % module)
old_modules = dict(manager.modules)
sources = get_sources(previous_modules, [(module, path)])
if module in manager.missing_modules:
manager.missing_modules.remove(module)
try:
if module in graph:
del graph[module]
load_graph(sources, manager, graph)
except CompileError as err:
# Parse error somewhere in the program -- a blocker
assert err.module_with_blocker
if err.module_with_blocker != module:
# Blocker is in a fresh module. Delete the state of the original target module
# since it will be stale.
#
# TODO: It would be more efficient to store the original target module
path = manager.modules[module].path
del manager.modules[module]
remaining_modules = [(module, path)]
else:
remaining_modules = []
return BlockedUpdate(err.module_with_blocker, path, remaining_modules,
err.messages)
if not os.path.isfile(path):
delete_module(module, graph, manager)
return NormalUpdate(module, path, [], None)
# Find any other modules brought in by imports.
changed_modules = get_all_changed_modules(module, path, previous_modules,
graph)
# If there are multiple modules to process, only process one of them and return
# the remaining ones to the caller.
if len(changed_modules) > 1:
# As an optimization, look for a module that imports no other changed modules.
module, path = find_relative_leaf_module(changed_modules, graph)
changed_modules.remove((module, path))
remaining_modules = changed_modules
# The remaining modules haven't been processed yet so drop them.
for id, _ in remaining_modules:
if id in old_modules:
manager.modules[id] = old_modules[id]
else:
del manager.modules[id]
del graph[id]
manager.log_fine_grained('--> %r (newly imported)' % module)
else:
remaining_modules = []
state = graph[module]
# Process the changed file.
state.parse_file()
# TODO: state.fix_suppressed_dependencies()?
try:
state.semantic_analysis()
except CompileError as err:
# There was a blocking error, so module AST is incomplete. Restore old modules.
manager.modules.clear()
manager.modules.update(old_modules)
del graph[module]
return BlockedUpdate(module, path, remaining_modules, err.messages)
state.semantic_analysis_pass_three()
state.semantic_analysis_apply_patches()
# Merge old and new ASTs.
assert state.tree is not None, "file must be at least parsed"
new_modules = {module: state.tree} # type: Dict[str, Optional[MypyFile]]
replace_modules_with_new_variants(manager, graph, old_modules, new_modules)
# Perform type checking.
state.type_checker().reset()
state.type_check_first_pass()
state.type_check_second_pass()
state.compute_fine_grained_deps()
state.finish_passes()
# TODO: state.write_cache()?
# TODO: state.mark_as_rechecked()?
graph[module] = state
return NormalUpdate(module, path, remaining_modules, state.tree)
def reprocess_nodes(manager: BuildManager,
graph: Dict[str, State],
module_id: str,
nodeset: Set[FineGrainedDeferredNode],
deps: Dict[str, Set[str]],
processed_targets: List[str]) -> Set[str]:
"""Reprocess a set of nodes within a single module.
Return fired triggers.
"""
if module_id not in graph:
manager.log_fine_grained('%s not in graph (blocking errors or deleted?)' %
module_id)
return set()
file_node = manager.modules[module_id]
old_symbols = find_symbol_tables_recursive(file_node.fullname, file_node.names)
old_symbols = {name: names.copy() for name, names in old_symbols.items()}
old_symbols_snapshot = snapshot_symbol_table(file_node.fullname, file_node.names)
def key(node: FineGrainedDeferredNode) -> int:
# Unlike modules which are sorted by name within SCC,
# nodes within the same module are sorted by line number, because
# this is how they are processed in normal mode.
return node.node.line
nodes = sorted(nodeset, key=key)
options = graph[module_id].options
manager.errors.set_file_ignored_lines(
file_node.path, file_node.ignored_lines, options.ignore_errors)
targets = set()
for node in nodes:
target = target_from_node(module_id, node.node)
if target is not None:
targets.add(target)
manager.errors.clear_errors_in_targets(file_node.path, targets)
# If one of the nodes is the module itself, emit any errors that
# happened before semantic analysis.
for target in targets:
if target == module_id:
for info in graph[module_id].early_errors:
manager.errors.add_error_info(info)
# Strip semantic analysis information.
saved_attrs = {} # type: SavedAttributes
for deferred in nodes:
processed_targets.append(deferred.node.fullname)
strip_target(deferred.node, saved_attrs)
semantic_analysis_for_targets(graph[module_id], nodes, graph, saved_attrs)
# Merge symbol tables to preserve identities of AST nodes. The file node will remain
# the same, but other nodes may have been recreated with different identities, such as
# NamedTuples defined using assignment statements.
new_symbols = find_symbol_tables_recursive(file_node.fullname, file_node.names)
for name in old_symbols:
if name in new_symbols:
merge_asts(file_node, old_symbols[name], file_node, new_symbols[name])
# Type check.
checker = graph[module_id].type_checker()
checker.reset()
# We seem to need additional passes in fine-grained incremental mode.
checker.pass_num = 0
checker.last_pass = 3
more = checker.check_second_pass(nodes)
while more:
more = False
if graph[module_id].type_checker().check_second_pass():
more = True
if manager.options.export_types:
manager.all_types.update(graph[module_id].type_map())
new_symbols_snapshot = snapshot_symbol_table(file_node.fullname, file_node.names)
# Check if any attribute types were changed and need to be propagated further.
changed = compare_symbol_table_snapshots(file_node.fullname,
old_symbols_snapshot,
new_symbols_snapshot)
new_triggered = {make_trigger(name) for name in changed}
# Dependencies may have changed.
update_deps(module_id, nodes, graph, deps, options)
# Report missing imports.
graph[module_id].verify_dependencies()
graph[module_id].free_state()
return new_triggered
def update_module_isolated(module: str,
path: str,
manager: BuildManager,
previous_modules: Dict[str, str],
graph: Graph,
force_removed: bool) -> UpdateResult:
"""Build a new version of one changed module only.
Don't propagate changes to elsewhere in the program. Raise CompileError on
encountering a blocking error.
Args:
module: Changed module (modified, created or deleted)
path: Path of the changed module
manager: Build manager
graph: Build graph
force_removed: If True, consider the module removed from the build even it the
file exists
Returns a named tuple describing the result (see above for details).
"""
if module not in graph:
manager.log_fine_grained('new module %r' % module)
if not manager.fscache.isfile(path) or force_removed:
delete_module(module, path, graph, manager)
return NormalUpdate(module, path, [], None)
sources = get_sources(manager.fscache, previous_modules, [(module, path)])
if module in manager.missing_modules:
manager.missing_modules.remove(module)
orig_module = module
orig_state = graph.get(module)
orig_tree = manager.modules.get(module)
def restore(ids: List[str]) -> None:
# For each of the modules in ids, restore that id's old
# manager.modules and graphs entries. (Except for the original
# module, this means deleting them.)
for id in ids:
if id == orig_module and orig_tree:
manager.modules[id] = orig_tree
elif id in manager.modules:
del manager.modules[id]
if id == orig_module and orig_state:
graph[id] = orig_state
elif id in graph:
del graph[id]
new_modules = [] # type: List[State]
try:
if module in graph:
del graph[module]
load_graph(sources, manager, graph, new_modules)
except CompileError as err:
# Parse error somewhere in the program -- a blocker
assert err.module_with_blocker
restore([module] + [st.id for st in new_modules])
return BlockedUpdate(err.module_with_blocker, path, [], err.messages)
# Reparsing the file may have brought in dependencies that we
# didn't have before. Make sure that they are loaded to restore
# the invariant that a module having a loaded tree implies that
# its dependencies do as well.
ensure_trees_loaded(manager, graph, graph[module].dependencies)
# Find any other modules brought in by imports.
changed_modules = [(st.id, st.xpath) for st in new_modules]
# If there are multiple modules to process, only process one of them and return
# the remaining ones to the caller.
if len(changed_modules) > 1:
# As an optimization, look for a module that imports no other changed modules.
module, path = find_relative_leaf_module(changed_modules, graph)
changed_modules.remove((module, path))
remaining_modules = changed_modules
# The remaining modules haven't been processed yet so drop them.
restore([id for id, _ in remaining_modules])
manager.log_fine_grained('--> %r (newly imported)' % module)
else:
remaining_modules = []
state = graph[module]
# Process the changed file.
state.parse_file()
assert state.tree is not None, "file must be at least parsed"
t0 = time.time()
# TODO: state.fix_suppressed_dependencies()?
if module == 'typing':
# We need to manually add typing aliases to builtins, like we
# do in process_stale_scc. Because this can't be done until
# builtins is also loaded, there isn't an obvious way to
# refactor this.
manager.semantic_analyzer.add_builtin_aliases(state.tree)
try:
state.semantic_analysis()
except CompileError as err:
# There was a blocking error, so module AST is incomplete. Restore old modules.
restore([module])
return BlockedUpdate(module, path, remaining_modules, err.messages)
state.semantic_analysis_pass_three()
state.semantic_analysis_apply_patches()
# Merge old and new ASTs.
new_modules_dict = {module: state.tree} # type: Dict[str, Optional[MypyFile]]
replace_modules_with_new_variants(manager, graph, {orig_module: orig_tree}, new_modules_dict)
t1 = time.time()
# Perform type checking.
state.type_checker().reset()
state.type_check_first_pass()
state.type_check_second_pass()
t2 = time.time()
state.compute_fine_grained_deps()
t3 = time.time()
state.finish_passes()
t4 = time.time()
manager.add_stats(
semanal_time=t1 - t0,
typecheck_time=t2 - t1,
deps_time=t3 - t2,
finish_passes_time=t4 - t3)
graph[module] = state
return NormalUpdate(module, path, remaining_modules, state.tree)
def update_module_isolated(module: str,
path: str,
manager: BuildManager,
previous_modules: Dict[str, str],
graph: Graph,
force_removed: bool) -> UpdateResult:
"""Build a new version of one changed module only.
Don't propagate changes to elsewhere in the program. Raise CompileError on
encountering a blocking error.
Args:
module: Changed module (modified, created or deleted)
path: Path of the changed module
manager: Build manager
graph: Build graph
force_removed: If True, consider the module removed from the build even it the
file exists
Returns a named tuple describing the result (see above for details).
"""
if module not in graph:
manager.log_fine_grained('new module %r' % module)
if not manager.fscache.isfile(path) or force_removed:
delete_module(module, path, graph, manager)
return NormalUpdate(module, path, [], None)
sources = get_sources(manager.fscache, previous_modules, [(module, path)])
if module in manager.missing_modules:
manager.missing_modules.remove(module)
orig_module = module
orig_state = graph.get(module)
orig_tree = manager.modules.get(module)
def restore(ids: List[str]) -> None:
# For each of the modules in ids, restore that id's old
# manager.modules and graphs entries. (Except for the original
# module, this means deleting them.)
for id in ids:
if id == orig_module and orig_tree:
manager.modules[id] = orig_tree
elif id in manager.modules:
del manager.modules[id]
if id == orig_module and orig_state:
graph[id] = orig_state
elif id in graph:
del graph[id]
new_modules = [] # type: List[State]
try:
if module in graph:
del graph[module]
load_graph(sources, manager, graph, new_modules)
except CompileError as err:
# Parse error somewhere in the program -- a blocker
assert err.module_with_blocker
restore([module] + [st.id for st in new_modules])
return BlockedUpdate(err.module_with_blocker, path, [], err.messages)
# Reparsing the file may have brought in dependencies that we
# didn't have before. Make sure that they are loaded to restore
# the invariant that a module having a loaded tree implies that
# its dependencies do as well.
ensure_trees_loaded(manager, graph, graph[module].dependencies)
# Find any other modules brought in by imports.
changed_modules = [(st.id, st.xpath) for st in new_modules]
# If there are multiple modules to process, only process one of them and return
# the remaining ones to the caller.
if len(changed_modules) > 1:
# As an optimization, look for a module that imports no other changed modules.
module, path = find_relative_leaf_module(changed_modules, graph)
changed_modules.remove((module, path))
remaining_modules = changed_modules
# The remaining modules haven't been processed yet so drop them.
restore([id for id, _ in remaining_modules])
manager.log_fine_grained('--> %r (newly imported)' % module)
else:
remaining_modules = []
state = graph[module]
# Process the changed file.
state.parse_file()
assert state.tree is not None, "file must be at least parsed"
t0 = time.time()
# TODO: state.fix_suppressed_dependencies()?
try:
semantic_analysis_for_scc(graph, [state.id], manager.errors)
except CompileError as err:
# There was a blocking error, so module AST is incomplete. Restore old modules.
restore([module])
return BlockedUpdate(module, path, remaining_modules, err.messages)
# Merge old and new ASTs.
new_modules_dict = {module: state.tree} # type: Dict[str, Optional[MypyFile]]
replace_modules_with_new_variants(manager, graph, {orig_module: orig_tree}, new_modules_dict)
t1 = time.time()
# Perform type checking.
state.type_checker().reset()
state.type_check_first_pass()
state.type_check_second_pass()
t2 = time.time()
state.finish_passes()
t3 = time.time()
manager.add_stats(
semanal_time=t1 - t0,
typecheck_time=t2 - t1,
finish_passes_time=t3 - t2)
graph[module] = state
return NormalUpdate(module, path, remaining_modules, state.tree)
def reprocess_nodes(manager: BuildManager,
graph: Dict[str, State],
module_id: str,
nodeset: Set[DeferredNode],
deps: Dict[str, Set[str]]) -> Set[str]:
"""Reprocess a set of nodes within a single module.
Return fired triggers.
"""
if module_id not in graph:
manager.log_fine_grained('%s not in graph (blocking errors or deleted?)' %
module_id)
return set()
file_node = manager.modules[module_id]
old_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names)
old_symbols = {name: names.copy() for name, names in old_symbols.items()}
old_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names)
def key(node: DeferredNode) -> int:
# Unlike modules which are sorted by name within SCC,
# nodes within the same module are sorted by line number, because
# this is how they are processed in normal mode.
return node.node.line
nodes = sorted(nodeset, key=key)
# TODO: ignore_all argument to set_file_ignored_lines
manager.errors.set_file_ignored_lines(file_node.path, file_node.ignored_lines)
targets = set()
for node in nodes:
target = target_from_node(module_id, node.node)
if target is not None:
targets.add(target)
manager.errors.clear_errors_in_targets(file_node.path, targets)
# Strip semantic analysis information.
for deferred in nodes:
strip_target(deferred.node)
semantic_analyzer = manager.semantic_analyzer
patches = [] # type: List[Tuple[int, Callable[[], None]]]
# Second pass of semantic analysis. We don't redo the first pass, because it only
# does local things that won't go stale.
for deferred in nodes:
with semantic_analyzer.file_context(
file_node=file_node,
fnam=file_node.path,
options=manager.options,
active_type=deferred.active_typeinfo):
manager.semantic_analyzer.refresh_partial(deferred.node, patches)
# Third pass of semantic analysis.
for deferred in nodes:
with semantic_analyzer.file_context(
file_node=file_node,
fnam=file_node.path,
options=manager.options,
active_type=deferred.active_typeinfo,
scope=manager.semantic_analyzer_pass3.scope):
manager.semantic_analyzer_pass3.refresh_partial(deferred.node, patches)
with semantic_analyzer.file_context(
file_node=file_node,
fnam=file_node.path,
options=manager.options,
active_type=None):
apply_semantic_analyzer_patches(patches)
# Merge symbol tables to preserve identities of AST nodes. The file node will remain
# the same, but other nodes may have been recreated with different identities, such as
# NamedTuples defined using assignment statements.
new_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names)
for name in old_symbols:
if name in new_symbols:
merge_asts(file_node, old_symbols[name], file_node, new_symbols[name])
# Type check.
checker = graph[module_id].type_checker()
checker.reset()
# We seem to need additional passes in fine-grained incremental mode.
checker.pass_num = 0
checker.last_pass = 3
more = checker.check_second_pass(nodes)
while more:
more = False
if graph[module_id].type_checker().check_second_pass():
more = True
new_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names)
# Check if any attribute types were changed and need to be propagated further.
changed = compare_symbol_table_snapshots(file_node.fullname(),
old_symbols_snapshot,
new_symbols_snapshot)
new_triggered = {make_trigger(name) for name in changed}
# Dependencies may have changed.
update_deps(module_id, nodes, graph, deps, manager.options)
# Report missing imports.
verify_dependencies(graph[module_id], manager)
return new_triggered
def reprocess_nodes(manager: BuildManager,
graph: Dict[str, State],
module_id: str,
nodeset: Set[FineGrainedDeferredNode],
deps: Dict[str, Set[str]],
processed_targets: List[str]) -> Set[str]:
"""Reprocess a set of nodes within a single module.
Return fired triggers.
"""
if module_id not in graph:
manager.log_fine_grained('%s not in graph (blocking errors or deleted?)' %
module_id)
return set()
file_node = manager.modules[module_id]
old_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names)
old_symbols = {name: names.copy() for name, names in old_symbols.items()}
old_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names)
def key(node: FineGrainedDeferredNode) -> int:
# Unlike modules which are sorted by name within SCC,
# nodes within the same module are sorted by line number, because
# this is how they are processed in normal mode.
return node.node.line
nodes = sorted(nodeset, key=key)
options = graph[module_id].options
manager.errors.set_file_ignored_lines(
file_node.path, file_node.ignored_lines, options.ignore_errors)
targets = set()
for node in nodes:
target = target_from_node(module_id, node.node)
if target is not None:
targets.add(target)
manager.errors.clear_errors_in_targets(file_node.path, targets)
# Strip semantic analysis information.
patches = [] # type: List[Callable[[], None]]
for deferred in nodes:
processed_targets.append(deferred.node.fullname())
if not manager.options.new_semantic_analyzer:
strip_target(deferred.node)
else:
patches = strip_target_new(deferred.node)
if not options.new_semantic_analyzer:
re_analyze_nodes(file_node, nodes, manager, options)
else:
process_selected_targets(graph[module_id], nodes, graph, patches)
# Merge symbol tables to preserve identities of AST nodes. The file node will remain
# the same, but other nodes may have been recreated with different identities, such as
# NamedTuples defined using assignment statements.
new_symbols = find_symbol_tables_recursive(file_node.fullname(), file_node.names)
for name in old_symbols:
if name in new_symbols:
merge_asts(file_node, old_symbols[name], file_node, new_symbols[name])
# Type check.
checker = graph[module_id].type_checker()
checker.reset()
# We seem to need additional passes in fine-grained incremental mode.
checker.pass_num = 0
checker.last_pass = 3
more = checker.check_second_pass(nodes)
while more:
more = False
if graph[module_id].type_checker().check_second_pass():
more = True
if manager.options.export_types:
manager.all_types.update(graph[module_id].type_map())
new_symbols_snapshot = snapshot_symbol_table(file_node.fullname(), file_node.names)
# Check if any attribute types were changed and need to be propagated further.
changed = compare_symbol_table_snapshots(file_node.fullname(),
old_symbols_snapshot,
new_symbols_snapshot)
new_triggered = {make_trigger(name) for name in changed}
# Dependencies may have changed.
update_deps(module_id, nodes, graph, deps, options)
# Report missing imports.
graph[module_id].verify_dependencies()
return new_triggered
def update_module_isolated(module: str,
path: str,
manager: BuildManager,
previous_modules: Dict[str, str],
graph: Graph,
force_removed: bool) -> UpdateResult:
"""Build a new version of one changed module only.
Don't propagate changes to elsewhere in the program. Raise CompleError on
encountering a blocking error.
Args:
module: Changed module (modified, created or deleted)
path: Path of the changed module
manager: Build manager
graph: Build graph
force_removed: If True, consider the module removed from the build even it the
file exists
Returns a named tuple describing the result (see above for details).
"""
if module in manager.modules:
assert_equivalent_paths(path, manager.modules[module].path)
else:
manager.log_fine_grained('new module %r' % module)
if not manager.fscache.isfile(path) or force_removed:
delete_module(module, graph, manager)
return NormalUpdate(module, path, [], None)
old_modules = dict(manager.modules)
sources = get_sources(manager.fscache, previous_modules, [(module, path)])
if module in manager.missing_modules:
manager.missing_modules.remove(module)
try:
if module in graph:
del graph[module]
load_graph(sources, manager, graph)
except CompileError as err:
# Parse error somewhere in the program -- a blocker
assert err.module_with_blocker
if err.module_with_blocker != module:
# Blocker is in a fresh module. Delete the state of the original target module
# since it will be stale.
#
# TODO: It would be more efficient to store the original target module
path = manager.modules[module].path
del manager.modules[module]
remaining_modules = [(module, path)]
else:
remaining_modules = []
return BlockedUpdate(err.module_with_blocker, path, remaining_modules, err.messages)
# Find any other modules brought in by imports.
changed_modules = get_all_changed_modules(module, path, previous_modules, graph)
# If there are multiple modules to process, only process one of them and return
# the remaining ones to the caller.
if len(changed_modules) > 1:
# As an optimization, look for a module that imports no other changed modules.
module, path = find_relative_leaf_module(changed_modules, graph)
changed_modules.remove((module, path))
remaining_modules = changed_modules
# The remaining modules haven't been processed yet so drop them.
for id, _ in remaining_modules:
if id in old_modules:
manager.modules[id] = old_modules[id]
else:
del manager.modules[id]
del graph[id]
manager.log_fine_grained('--> %r (newly imported)' % module)
else:
remaining_modules = []
state = graph[module]
# Process the changed file.
state.parse_file()
# TODO: state.fix_suppressed_dependencies()?
try:
state.semantic_analysis()
except CompileError as err:
# There was a blocking error, so module AST is incomplete. Restore old modules.
manager.modules.clear()
manager.modules.update(old_modules)
del graph[module]
return BlockedUpdate(module, path, remaining_modules, err.messages)
state.semantic_analysis_pass_three()
state.semantic_analysis_apply_patches()
# Merge old and new ASTs.
assert state.tree is not None, "file must be at least parsed"
new_modules = {module: state.tree} # type: Dict[str, Optional[MypyFile]]
replace_modules_with_new_variants(manager, graph, old_modules, new_modules)
# Perform type checking.
state.type_checker().reset()
state.type_check_first_pass()
state.type_check_second_pass()
state.compute_fine_grained_deps()
state.finish_passes()
graph[module] = state
return NormalUpdate(module, path, remaining_modules, state.tree)
