def find_targets_recursive(
manager: BuildManager,
graph: Graph,
triggers: Set[str],
deps: Dict[str, Set[str]],
up_to_date_modules: Set[str]) -> Tuple[Dict[str, Set[FineGrainedDeferredNode]],
Set[str], Set[TypeInfo]]:
"""Find names of all targets that need to reprocessed, given some triggers.
Returns: A tuple containing a:
* Dictionary from module id to a set of stale targets.
* A set of module ids for unparsed modules with stale targets.
"""
result = {} # type: Dict[str, Set[FineGrainedDeferredNode]]
worklist = triggers
processed = set() # type: Set[str]
stale_protos = set() # type: Set[TypeInfo]
unloaded_files = set() # type: Set[str]
# Find AST nodes corresponding to each target.
#
# TODO: Don't rely on a set, since the items are in an unpredictable order.
while worklist:
processed |= worklist
current = worklist
worklist = set()
for target in current:
if target.startswith('<'):
module_id = module_prefix(graph, trigger_to_target(target))
if module_id:
ensure_deps_loaded(module_id, deps, graph)
worklist |= deps.get(target, set()) - processed
else:
module_id = module_prefix(graph, target)
if module_id is None:
# Deleted module.
continue
if module_id in up_to_date_modules:
# Already processed.
continue
if (module_id not in manager.modules
or manager.modules[module_id].is_cache_skeleton):
# We haven't actually parsed and checked the module, so we don't have
# access to the actual nodes.
# Add it to the queue of files that need to be processed fully.
unloaded_files.add(module_id)
continue
if module_id not in result:
result[module_id] = set()
manager.log_fine_grained('process: %s' % target)
deferred, stale_proto = lookup_target(manager, target)
if stale_proto:
stale_protos.add(stale_proto)
result[module_id].update(deferred)
return result, unloaded_files, stale_protos
def find_targets_recursive(
manager: BuildManager, graph: Graph, triggers: Set[str],
deps: Dict[str, Set[str]], up_to_date_modules: Set[str]
) -> Tuple[Dict[str, Set[FineGrainedDeferredNode]], Set[str], Set[TypeInfo]]:
"""Find names of all targets that need to reprocessed, given some triggers.
Returns: A tuple containing a:
* Dictionary from module id to a set of stale targets.
* A set of module ids for unparsed modules with stale targets.
"""
result = {} # type: Dict[str, Set[FineGrainedDeferredNode]]
worklist = triggers
processed = set() # type: Set[str]
stale_protos = set() # type: Set[TypeInfo]
unloaded_files = set() # type: Set[str]
# Find AST nodes corresponding to each target.
#
# TODO: Don't rely on a set, since the items are in an unpredictable order.
while worklist:
processed |= worklist
current = worklist
worklist = set()
for target in current:
if target.startswith('<'):
module_id = module_prefix(graph, trigger_to_target(target))
if module_id:
ensure_deps_loaded(module_id, deps, graph)
worklist |= deps.get(target, set()) - processed
else:
module_id = module_prefix(graph, target)
if module_id is None:
# Deleted module.
continue
if module_id in up_to_date_modules:
# Already processed.
continue
if (module_id not in manager.modules
or manager.modules[module_id].is_cache_skeleton):
# We haven't actually parsed and checked the module, so we don't have
# access to the actual nodes.
# Add it to the queue of files that need to be processed fully.
unloaded_files.add(module_id)
continue
if module_id not in result:
result[module_id] = set()
manager.log_fine_grained('process: %s' % target)
deferred, stale_proto = lookup_target(manager, target)
if stale_proto:
stale_protos.add(stale_proto)
result[module_id].update(deferred)
return result, unloaded_files, stale_protos
def find_targets_recursive(
triggers: Set[str], deps: Dict[str, Set[str]], modules: Dict[str,
MypyFile],
up_to_date_modules: Set[str]) -> Dict[str, Set[DeferredNode]]:
"""Find names of all targets that need to reprocessed, given some triggers.
Returns: Dictionary from module id to a set of stale targets.
"""
result = {} # type: Dict[str, Set[DeferredNode]]
worklist = triggers
processed = set() # type: Set[str]
# Find AST nodes corresponding to each target.
#
# TODO: Don't rely on a set, since the items are in an unpredictable order.
while worklist:
processed |= worklist
current = worklist
worklist = set()
for target in current:
if target.startswith('<'):
worklist |= deps.get(target, set()) - processed
else:
module_id = module_prefix(modules, target)
if module_id in up_to_date_modules:
# Already processed.
continue
if module_id not in result:
result[module_id] = set()
if DEBUG:
print('process', target)
deferred = lookup_target(modules, target)
result[module_id].update(deferred)
return result
def propagate_changes_using_dependencies(
manager: BuildManager, graph: Dict[str, State], deps: Dict[str,
Set[str]],
triggered: Set[str], up_to_date_modules: Set[str],
targets_with_errors: Set[str]) -> List[Tuple[str, str]]:
"""Transitively rechecks targets based on triggers and the dependency map.
Returns a list (module id, path) tuples representing modules that contain
a target that needs to be reprocessed but that has not been parsed yet."""
num_iter = 0
remaining_modules = [] # type: List[Tuple[str, str]]
# Propagate changes until nothing visible has changed during the last
# iteration.
while triggered or targets_with_errors:
num_iter += 1
if num_iter > MAX_ITER:
raise RuntimeError(
'Max number of iterations (%d) reached (endless loop?)' %
MAX_ITER)
todo, unloaded, stale_protos = find_targets_recursive(
manager, graph, triggered, deps, up_to_date_modules)
# TODO: we sort to make it deterministic, but this is *incredibly* ad hoc
remaining_modules.extend(
(id, graph[id].xpath) for id in sorted(unloaded))
# Also process targets that used to have errors, as otherwise some
# errors might be lost.
for target in targets_with_errors:
id = module_prefix(graph, target)
if id is not None and id not in up_to_date_modules:
if id not in todo:
todo[id] = set()
manager.log_fine_grained('process target with error: %s' %
target)
more_nodes, _ = lookup_target(manager, target)
todo[id].update(more_nodes)
triggered = set()
# First invalidate subtype caches in all stale protocols.
# We need to do this to avoid false negatives if the protocol itself is
# unchanged, but was marked stale because its sub- (or super-) type changed.
for info in stale_protos:
TypeState.reset_subtype_caches_for(info)
# Then fully reprocess all targets.
# TODO: Preserve order (set is not optimal)
for id, nodes in sorted(todo.items(), key=lambda x: x[0]):
assert id not in up_to_date_modules
triggered |= reprocess_nodes(manager, graph, id, nodes, deps)
# Changes elsewhere may require us to reprocess modules that were
# previously considered up to date. For example, there may be a
# dependency loop that loops back to an originally processed module.
up_to_date_modules = set()
targets_with_errors = set()
if is_verbose(manager):
manager.log_fine_grained('triggered: %r' % list(triggered))
return remaining_modules
def propagate_changes_using_dependencies(
manager: BuildManager,
graph: Dict[str, State],
deps: Dict[str, Set[str]],
triggered: Set[str],
up_to_date_modules: Set[str],
targets_with_errors: Set[str]) -> List[Tuple[str, str]]:
"""Transitively rechecks targets based on triggers and the dependency map.
Returns a list (module id, path) tuples representing modules that contain
a target that needs to be reprocessed but that has not been parsed yet."""
num_iter = 0
remaining_modules = [] # type: List[Tuple[str, str]]
# Propagate changes until nothing visible has changed during the last
# iteration.
while triggered or targets_with_errors:
num_iter += 1
if num_iter > MAX_ITER:
raise RuntimeError('Max number of iterations (%d) reached (endless loop?)' % MAX_ITER)
todo, unloaded, stale_protos = find_targets_recursive(manager, graph,
triggered, deps, up_to_date_modules)
# TODO: we sort to make it deterministic, but this is *incredibly* ad hoc
remaining_modules.extend((id, graph[id].xpath) for id in sorted(unloaded))
# Also process targets that used to have errors, as otherwise some
# errors might be lost.
for target in targets_with_errors:
id = module_prefix(graph, target)
if id is not None and id not in up_to_date_modules:
if id not in todo:
todo[id] = set()
manager.log_fine_grained('process target with error: %s' % target)
more_nodes, _ = lookup_target(manager, target)
todo[id].update(more_nodes)
triggered = set()
# First invalidate subtype caches in all stale protocols.
# We need to do this to avoid false negatives if the protocol itself is
# unchanged, but was marked stale because its sub- (or super-) type changed.
for info in stale_protos:
TypeState.reset_subtype_caches_for(info)
# Then fully reprocess all targets.
# TODO: Preserve order (set is not optimal)
for id, nodes in sorted(todo.items(), key=lambda x: x[0]):
assert id not in up_to_date_modules
triggered |= reprocess_nodes(manager, graph, id, nodes, deps)
# Changes elsewhere may require us to reprocess modules that were
# previously considered up to date. For example, there may be a
# dependency loop that loops back to an originally processed module.
up_to_date_modules = set()
targets_with_errors = set()
if is_verbose(manager):
manager.log_fine_grained('triggered: %r' % list(triggered))
return remaining_modules
def propagate_changes_using_dependencies(
manager: BuildManager, graph: Dict[str, State], deps: Dict[str,
Set[str]],
triggered: Set[str], up_to_date_modules: Set[str],
targets_with_errors: Set[str]) -> List[Tuple[str, str]]:
"""Transitively rechecks targets based on triggers and the dependency map.
Returns a list (module id, path) tuples representing modules that contain
a target that needs to be reprocessed but that has not been parsed yet."""
# TODO: Multiple type checking passes
num_iter = 0
remaining_modules = []
# Propagate changes until nothing visible has changed during the last
# iteration.
while triggered or targets_with_errors:
num_iter += 1
if num_iter > MAX_ITER:
raise RuntimeError(
'Max number of iterations (%d) reached (endless loop?)' %
MAX_ITER)
todo = find_targets_recursive(manager, triggered, deps,
up_to_date_modules)
# Also process targets that used to have errors, as otherwise some
# errors might be lost.
for target in targets_with_errors:
id = module_prefix(manager.modules, target)
if id is not None and id not in up_to_date_modules:
if id not in todo:
todo[id] = set()
manager.log_fine_grained('process target with error: %s' %
target)
todo[id].update(lookup_target(manager, target))
triggered = set()
# TODO: Preserve order (set is not optimal)
for id, nodes in sorted(todo.items(), key=lambda x: x[0]):
assert id not in up_to_date_modules
if manager.modules[id].is_cache_skeleton:
# We have only loaded the cache for this file, not the actual file,
# so we can't access the nodes to reprocess.
# Add it to the queue of files that need to be processed fully.
remaining_modules.append((id, manager.modules[id].path))
else:
triggered |= reprocess_nodes(manager, graph, id, nodes, deps)
# Changes elsewhere may require us to reprocess modules that were
# previously considered up to date. For example, there may be a
# dependency loop that loops back to an originally processed module.
up_to_date_modules = set()
targets_with_errors = set()
if is_verbose(manager):
manager.log_fine_grained('triggered: %r' % list(triggered))
return remaining_modules
def restore_after(self, target: str) -> Iterator[None]:
"""Context manager that reloads a module after executing the body.
This should undo any damage done to the module state while mucking around.
"""
try:
yield
finally:
module = module_prefix(self.graph, target)
if module:
self.reload(self.graph[module])
def restore_after(self, target: str) -> Iterator[None]:
"""Context manager that reloads a module after executing the body.
This should undo any damage done to the module state while mucking around.
"""
try:
yield
finally:
module = module_prefix(self.graph, target)
if module:
self.reload(self.graph[module])
def propagate_changes_using_dependencies(
manager: BuildManager,
graph: Dict[str, State],
deps: Dict[str, Set[str]],
triggered: Set[str],
up_to_date_modules: Set[str],
targets_with_errors: Set[str]) -> List[Tuple[str, str]]:
"""Transitively rechecks targets based on triggers and the dependency map.
Returns a list (module id, path) tuples representing modules that contain
a target that needs to be reprocessed but that has not been parsed yet."""
num_iter = 0
remaining_modules = []
# Propagate changes until nothing visible has changed during the last
# iteration.
while triggered or targets_with_errors:
num_iter += 1
if num_iter > MAX_ITER:
raise RuntimeError('Max number of iterations (%d) reached (endless loop?)' % MAX_ITER)
todo = find_targets_recursive(manager, triggered, deps, up_to_date_modules)
# Also process targets that used to have errors, as otherwise some
# errors might be lost.
for target in targets_with_errors:
id = module_prefix(manager.modules, target)
if id is not None and id not in up_to_date_modules:
if id not in todo:
todo[id] = set()
manager.log_fine_grained('process target with error: %s' % target)
todo[id].update(lookup_target(manager, target))
triggered = set()
# TODO: Preserve order (set is not optimal)
for id, nodes in sorted(todo.items(), key=lambda x: x[0]):
assert id not in up_to_date_modules
if manager.modules[id].is_cache_skeleton:
# We have only loaded the cache for this file, not the actual file,
# so we can't access the nodes to reprocess.
# Add it to the queue of files that need to be processed fully.
remaining_modules.append((id, manager.modules[id].path))
else:
triggered |= reprocess_nodes(manager, graph, id, nodes, deps)
# Changes elsewhere may require us to reprocess modules that were
# previously considered up to date. For example, there may be a
# dependency loop that loops back to an originally processed module.
up_to_date_modules = set()
targets_with_errors = set()
if is_verbose(manager):
manager.log_fine_grained('triggered: %r' % list(triggered))
return remaining_modules
def propagate_changes_using_dependencies(manager: BuildManager,
graph: Dict[str, State],
deps: Dict[str, Set[str]],
triggered: Set[str],
up_to_date_modules: Set[str],
targets_with_errors: Set[str],
modules: Iterable[str]) -> None:
# TODO: Multiple type checking passes
num_iter = 0
# Propagate changes until nothing visible has changed during the last
# iteration.
while triggered or targets_with_errors:
num_iter += 1
if num_iter > MAX_ITER:
raise RuntimeError(
'Max number of iterations (%d) reached (endless loop?)' %
MAX_ITER)
todo = find_targets_recursive(triggered, deps, manager.modules,
up_to_date_modules)
# Also process targets that used to have errors, as otherwise some
# errors might be lost.
for target in targets_with_errors:
id = module_prefix(modules, target)
if id is not None and id not in up_to_date_modules:
if id not in todo:
todo[id] = set()
if DEBUG:
print('process', target)
todo[id].update(lookup_target(manager.modules, target))
triggered = set()
# TODO: Preserve order (set is not optimal)
for id, nodes in sorted(todo.items(), key=lambda x: x[0]):
assert id not in up_to_date_modules
triggered |= reprocess_nodes(manager, graph, id, nodes, deps)
# Changes elsewhere may require us to reprocess modules that were
# previously considered up to date. For example, there may be a
# dependency loop that loops back to an originally processed module.
up_to_date_modules = set()
targets_with_errors = set()
if DEBUG:
print('triggered:', list(triggered))
def find_targets_recursive(
manager: BuildManager,
triggers: Set[str],
deps: Dict[str, Set[str]],
up_to_date_modules: Set[str]) -> Dict[str, Set[DeferredNode]]:
"""Find names of all targets that need to reprocessed, given some triggers.
Returns: Dictionary from module id to a set of stale targets.
"""
result = {} # type: Dict[str, Set[DeferredNode]]
worklist = triggers
processed = set() # type: Set[str]
# Find AST nodes corresponding to each target.
#
# TODO: Don't rely on a set, since the items are in an unpredictable order.
while worklist:
processed |= worklist
current = worklist
worklist = set()
for target in current:
if target.startswith('<'):
worklist |= deps.get(target, set()) - processed
else:
module_id = module_prefix(manager.modules, target)
if module_id is None:
# Deleted module.
continue
if module_id in up_to_date_modules:
# Already processed.
continue
if module_id not in result:
result[module_id] = set()
manager.log_fine_grained('process: %s' % target)
deferred = lookup_target(manager, target)
result[module_id].update(deferred)
return result
