def find_targets_recursive( manager: BuildManager, graph: Graph, triggers: Set[str], deps: Dict[str, Set[str]], up_to_date_modules: Set[str]) -> Tuple[Dict[str, Set[FineGrainedDeferredNode]], Set[str], Set[TypeInfo]]: """Find names of all targets that need to reprocessed, given some triggers. Returns: A tuple containing a: * Dictionary from module id to a set of stale targets. * A set of module ids for unparsed modules with stale targets. """ result = {} # type: Dict[str, Set[FineGrainedDeferredNode]] worklist = triggers processed = set() # type: Set[str] stale_protos = set() # type: Set[TypeInfo] unloaded_files = set() # type: Set[str] # Find AST nodes corresponding to each target. # # TODO: Don't rely on a set, since the items are in an unpredictable order. while worklist: processed |= worklist current = worklist worklist = set() for target in current: if target.startswith('<'): module_id = module_prefix(graph, trigger_to_target(target)) if module_id: ensure_deps_loaded(module_id, deps, graph) worklist |= deps.get(target, set()) - processed else: module_id = module_prefix(graph, target) if module_id is None: # Deleted module. continue if module_id in up_to_date_modules: # Already processed. continue if (module_id not in manager.modules or manager.modules[module_id].is_cache_skeleton): # We haven't actually parsed and checked the module, so we don't have # access to the actual nodes. # Add it to the queue of files that need to be processed fully. unloaded_files.add(module_id) continue if module_id not in result: result[module_id] = set() manager.log_fine_grained('process: %s' % target) deferred, stale_proto = lookup_target(manager, target) if stale_proto: stale_protos.add(stale_proto) result[module_id].update(deferred) return result, unloaded_files, stale_protos
def find_targets_recursive( manager: BuildManager, graph: Graph, triggers: Set[str], deps: Dict[str, Set[str]], up_to_date_modules: Set[str] ) -> Tuple[Dict[str, Set[FineGrainedDeferredNode]], Set[str], Set[TypeInfo]]: """Find names of all targets that need to reprocessed, given some triggers. Returns: A tuple containing a: * Dictionary from module id to a set of stale targets. * A set of module ids for unparsed modules with stale targets. """ result = {} # type: Dict[str, Set[FineGrainedDeferredNode]] worklist = triggers processed = set() # type: Set[str] stale_protos = set() # type: Set[TypeInfo] unloaded_files = set() # type: Set[str] # Find AST nodes corresponding to each target. # # TODO: Don't rely on a set, since the items are in an unpredictable order. while worklist: processed |= worklist current = worklist worklist = set() for target in current: if target.startswith('<'): module_id = module_prefix(graph, trigger_to_target(target)) if module_id: ensure_deps_loaded(module_id, deps, graph) worklist |= deps.get(target, set()) - processed else: module_id = module_prefix(graph, target) if module_id is None: # Deleted module. continue if module_id in up_to_date_modules: # Already processed. continue if (module_id not in manager.modules or manager.modules[module_id].is_cache_skeleton): # We haven't actually parsed and checked the module, so we don't have # access to the actual nodes. # Add it to the queue of files that need to be processed fully. unloaded_files.add(module_id) continue if module_id not in result: result[module_id] = set() manager.log_fine_grained('process: %s' % target) deferred, stale_proto = lookup_target(manager, target) if stale_proto: stale_protos.add(stale_proto) result[module_id].update(deferred) return result, unloaded_files, stale_protos
def find_targets_recursive( triggers: Set[str], deps: Dict[str, Set[str]], modules: Dict[str, MypyFile], up_to_date_modules: Set[str]) -> Dict[str, Set[DeferredNode]]: """Find names of all targets that need to reprocessed, given some triggers. Returns: Dictionary from module id to a set of stale targets. """ result = {} # type: Dict[str, Set[DeferredNode]] worklist = triggers processed = set() # type: Set[str] # Find AST nodes corresponding to each target. # # TODO: Don't rely on a set, since the items are in an unpredictable order. while worklist: processed |= worklist current = worklist worklist = set() for target in current: if target.startswith('<'): worklist |= deps.get(target, set()) - processed else: module_id = module_prefix(modules, target) if module_id in up_to_date_modules: # Already processed. continue if module_id not in result: result[module_id] = set() if DEBUG: print('process', target) deferred = lookup_target(modules, target) result[module_id].update(deferred) return result
def propagate_changes_using_dependencies( manager: BuildManager, graph: Dict[str, State], deps: Dict[str, Set[str]], triggered: Set[str], up_to_date_modules: Set[str], targets_with_errors: Set[str]) -> List[Tuple[str, str]]: """Transitively rechecks targets based on triggers and the dependency map. Returns a list (module id, path) tuples representing modules that contain a target that needs to be reprocessed but that has not been parsed yet.""" num_iter = 0 remaining_modules = [] # type: List[Tuple[str, str]] # Propagate changes until nothing visible has changed during the last # iteration. while triggered or targets_with_errors: num_iter += 1 if num_iter > MAX_ITER: raise RuntimeError( 'Max number of iterations (%d) reached (endless loop?)' % MAX_ITER) todo, unloaded, stale_protos = find_targets_recursive( manager, graph, triggered, deps, up_to_date_modules) # TODO: we sort to make it deterministic, but this is *incredibly* ad hoc remaining_modules.extend( (id, graph[id].xpath) for id in sorted(unloaded)) # Also process targets that used to have errors, as otherwise some # errors might be lost. for target in targets_with_errors: id = module_prefix(graph, target) if id is not None and id not in up_to_date_modules: if id not in todo: todo[id] = set() manager.log_fine_grained('process target with error: %s' % target) more_nodes, _ = lookup_target(manager, target) todo[id].update(more_nodes) triggered = set() # First invalidate subtype caches in all stale protocols. # We need to do this to avoid false negatives if the protocol itself is # unchanged, but was marked stale because its sub- (or super-) type changed. for info in stale_protos: TypeState.reset_subtype_caches_for(info) # Then fully reprocess all targets. # TODO: Preserve order (set is not optimal) for id, nodes in sorted(todo.items(), key=lambda x: x[0]): assert id not in up_to_date_modules triggered |= reprocess_nodes(manager, graph, id, nodes, deps) # Changes elsewhere may require us to reprocess modules that were # previously considered up to date. For example, there may be a # dependency loop that loops back to an originally processed module. up_to_date_modules = set() targets_with_errors = set() if is_verbose(manager): manager.log_fine_grained('triggered: %r' % list(triggered)) return remaining_modules
def propagate_changes_using_dependencies( manager: BuildManager, graph: Dict[str, State], deps: Dict[str, Set[str]], triggered: Set[str], up_to_date_modules: Set[str], targets_with_errors: Set[str]) -> List[Tuple[str, str]]: """Transitively rechecks targets based on triggers and the dependency map. Returns a list (module id, path) tuples representing modules that contain a target that needs to be reprocessed but that has not been parsed yet.""" num_iter = 0 remaining_modules = [] # type: List[Tuple[str, str]] # Propagate changes until nothing visible has changed during the last # iteration. while triggered or targets_with_errors: num_iter += 1 if num_iter > MAX_ITER: raise RuntimeError('Max number of iterations (%d) reached (endless loop?)' % MAX_ITER) todo, unloaded, stale_protos = find_targets_recursive(manager, graph, triggered, deps, up_to_date_modules) # TODO: we sort to make it deterministic, but this is *incredibly* ad hoc remaining_modules.extend((id, graph[id].xpath) for id in sorted(unloaded)) # Also process targets that used to have errors, as otherwise some # errors might be lost. for target in targets_with_errors: id = module_prefix(graph, target) if id is not None and id not in up_to_date_modules: if id not in todo: todo[id] = set() manager.log_fine_grained('process target with error: %s' % target) more_nodes, _ = lookup_target(manager, target) todo[id].update(more_nodes) triggered = set() # First invalidate subtype caches in all stale protocols. # We need to do this to avoid false negatives if the protocol itself is # unchanged, but was marked stale because its sub- (or super-) type changed. for info in stale_protos: TypeState.reset_subtype_caches_for(info) # Then fully reprocess all targets. # TODO: Preserve order (set is not optimal) for id, nodes in sorted(todo.items(), key=lambda x: x[0]): assert id not in up_to_date_modules triggered |= reprocess_nodes(manager, graph, id, nodes, deps) # Changes elsewhere may require us to reprocess modules that were # previously considered up to date. For example, there may be a # dependency loop that loops back to an originally processed module. up_to_date_modules = set() targets_with_errors = set() if is_verbose(manager): manager.log_fine_grained('triggered: %r' % list(triggered)) return remaining_modules
def propagate_changes_using_dependencies( manager: BuildManager, graph: Dict[str, State], deps: Dict[str, Set[str]], triggered: Set[str], up_to_date_modules: Set[str], targets_with_errors: Set[str]) -> List[Tuple[str, str]]: """Transitively rechecks targets based on triggers and the dependency map. Returns a list (module id, path) tuples representing modules that contain a target that needs to be reprocessed but that has not been parsed yet.""" # TODO: Multiple type checking passes num_iter = 0 remaining_modules = [] # Propagate changes until nothing visible has changed during the last # iteration. while triggered or targets_with_errors: num_iter += 1 if num_iter > MAX_ITER: raise RuntimeError( 'Max number of iterations (%d) reached (endless loop?)' % MAX_ITER) todo = find_targets_recursive(manager, triggered, deps, up_to_date_modules) # Also process targets that used to have errors, as otherwise some # errors might be lost. for target in targets_with_errors: id = module_prefix(manager.modules, target) if id is not None and id not in up_to_date_modules: if id not in todo: todo[id] = set() manager.log_fine_grained('process target with error: %s' % target) todo[id].update(lookup_target(manager, target)) triggered = set() # TODO: Preserve order (set is not optimal) for id, nodes in sorted(todo.items(), key=lambda x: x[0]): assert id not in up_to_date_modules if manager.modules[id].is_cache_skeleton: # We have only loaded the cache for this file, not the actual file, # so we can't access the nodes to reprocess. # Add it to the queue of files that need to be processed fully. remaining_modules.append((id, manager.modules[id].path)) else: triggered |= reprocess_nodes(manager, graph, id, nodes, deps) # Changes elsewhere may require us to reprocess modules that were # previously considered up to date. For example, there may be a # dependency loop that loops back to an originally processed module. up_to_date_modules = set() targets_with_errors = set() if is_verbose(manager): manager.log_fine_grained('triggered: %r' % list(triggered)) return remaining_modules
def restore_after(self, target: str) -> Iterator[None]: """Context manager that reloads a module after executing the body. This should undo any damage done to the module state while mucking around. """ try: yield finally: module = module_prefix(self.graph, target) if module: self.reload(self.graph[module])
def restore_after(self, target: str) -> Iterator[None]: """Context manager that reloads a module after executing the body. This should undo any damage done to the module state while mucking around. """ try: yield finally: module = module_prefix(self.graph, target) if module: self.reload(self.graph[module])
def propagate_changes_using_dependencies( manager: BuildManager, graph: Dict[str, State], deps: Dict[str, Set[str]], triggered: Set[str], up_to_date_modules: Set[str], targets_with_errors: Set[str]) -> List[Tuple[str, str]]: """Transitively rechecks targets based on triggers and the dependency map. Returns a list (module id, path) tuples representing modules that contain a target that needs to be reprocessed but that has not been parsed yet.""" num_iter = 0 remaining_modules = [] # Propagate changes until nothing visible has changed during the last # iteration. while triggered or targets_with_errors: num_iter += 1 if num_iter > MAX_ITER: raise RuntimeError('Max number of iterations (%d) reached (endless loop?)' % MAX_ITER) todo = find_targets_recursive(manager, triggered, deps, up_to_date_modules) # Also process targets that used to have errors, as otherwise some # errors might be lost. for target in targets_with_errors: id = module_prefix(manager.modules, target) if id is not None and id not in up_to_date_modules: if id not in todo: todo[id] = set() manager.log_fine_grained('process target with error: %s' % target) todo[id].update(lookup_target(manager, target)) triggered = set() # TODO: Preserve order (set is not optimal) for id, nodes in sorted(todo.items(), key=lambda x: x[0]): assert id not in up_to_date_modules if manager.modules[id].is_cache_skeleton: # We have only loaded the cache for this file, not the actual file, # so we can't access the nodes to reprocess. # Add it to the queue of files that need to be processed fully. remaining_modules.append((id, manager.modules[id].path)) else: triggered |= reprocess_nodes(manager, graph, id, nodes, deps) # Changes elsewhere may require us to reprocess modules that were # previously considered up to date. For example, there may be a # dependency loop that loops back to an originally processed module. up_to_date_modules = set() targets_with_errors = set() if is_verbose(manager): manager.log_fine_grained('triggered: %r' % list(triggered)) return remaining_modules
def propagate_changes_using_dependencies(manager: BuildManager, graph: Dict[str, State], deps: Dict[str, Set[str]], triggered: Set[str], up_to_date_modules: Set[str], targets_with_errors: Set[str], modules: Iterable[str]) -> None: # TODO: Multiple type checking passes num_iter = 0 # Propagate changes until nothing visible has changed during the last # iteration. while triggered or targets_with_errors: num_iter += 1 if num_iter > MAX_ITER: raise RuntimeError( 'Max number of iterations (%d) reached (endless loop?)' % MAX_ITER) todo = find_targets_recursive(triggered, deps, manager.modules, up_to_date_modules) # Also process targets that used to have errors, as otherwise some # errors might be lost. for target in targets_with_errors: id = module_prefix(modules, target) if id is not None and id not in up_to_date_modules: if id not in todo: todo[id] = set() if DEBUG: print('process', target) todo[id].update(lookup_target(manager.modules, target)) triggered = set() # TODO: Preserve order (set is not optimal) for id, nodes in sorted(todo.items(), key=lambda x: x[0]): assert id not in up_to_date_modules triggered |= reprocess_nodes(manager, graph, id, nodes, deps) # Changes elsewhere may require us to reprocess modules that were # previously considered up to date. For example, there may be a # dependency loop that loops back to an originally processed module. up_to_date_modules = set() targets_with_errors = set() if DEBUG: print('triggered:', list(triggered))
def find_targets_recursive( manager: BuildManager, triggers: Set[str], deps: Dict[str, Set[str]], up_to_date_modules: Set[str]) -> Dict[str, Set[DeferredNode]]: """Find names of all targets that need to reprocessed, given some triggers. Returns: Dictionary from module id to a set of stale targets. """ result = {} # type: Dict[str, Set[DeferredNode]] worklist = triggers processed = set() # type: Set[str] # Find AST nodes corresponding to each target. # # TODO: Don't rely on a set, since the items are in an unpredictable order. while worklist: processed |= worklist current = worklist worklist = set() for target in current: if target.startswith('<'): worklist |= deps.get(target, set()) - processed else: module_id = module_prefix(manager.modules, target) if module_id is None: # Deleted module. continue if module_id in up_to_date_modules: # Already processed. continue if module_id not in result: result[module_id] = set() manager.log_fine_grained('process: %s' % target) deferred = lookup_target(manager, target) result[module_id].update(deferred) return result