def fine_grained_increment(self,
sources: List[BuildSource],
remove: Optional[List[str]] = None,
update: Optional[List[str]] = None,
) -> Dict[str, Any]:
assert self.fine_grained_manager is not None
manager = self.fine_grained_manager.manager
t0 = time.time()
if remove is None and update is None:
# Use the fswatcher to determine which files were changed
# (updated or added) or removed.
self.update_sources(sources)
changed, removed = self.find_changed(sources)
else:
# Use the remove/update lists to update fswatcher.
# This avoids calling stat() for unchanged files.
changed, removed = self.update_changed(sources, remove or [], update or [])
manager.search_paths = compute_search_paths(sources, manager.options, manager.data_dir)
t1 = time.time()
manager.log("fine-grained increment: find_changed: {:.3f}s".format(t1 - t0))
messages = self.fine_grained_manager.update(changed, removed)
t2 = time.time()
manager.log("fine-grained increment: update: {:.3f}s".format(t2 - t1))
status = 1 if messages else 0
self.previous_sources = sources
return {'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status}
def fine_grained_increment(self,
sources: List[BuildSource],
remove: Optional[List[str]] = None,
update: Optional[List[str]] = None,
) -> Dict[str, Any]:
assert self.fine_grained_manager is not None
manager = self.fine_grained_manager.manager
t0 = time.time()
if remove is None and update is None:
# Use the fswatcher to determine which files were changed
# (updated or added) or removed.
self.update_sources(sources)
changed, removed = self.find_changed(sources)
else:
# Use the remove/update lists to update fswatcher.
# This avoids calling stat() for unchanged files.
changed, removed = self.update_changed(sources, remove or [], update or [])
manager.search_paths = compute_search_paths(sources, manager.options, manager.data_dir)
t1 = time.time()
manager.log("fine-grained increment: find_changed: {:.3f}s".format(t1 - t0))
messages = self.fine_grained_manager.update(changed, removed)
t2 = time.time()
manager.log("fine-grained increment: update: {:.3f}s".format(t2 - t1))
manager.add_stats(
find_changes_time=t1 - t0,
fg_update_time=t2 - t1,
files_changed=len(removed) + len(changed))
status = 1 if messages else 0
self.previous_sources = sources
return {'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status}
def fine_grained_increment(self,
sources: List[BuildSource]) -> Dict[str, Any]:
assert self.fine_grained_manager is not None
manager = self.fine_grained_manager.manager
t0 = time.time()
self.update_sources(sources)
changed, removed = self.find_changed(sources)
# TODO: Why create a new FileSystemCache rather than using self.fscache?
manager.search_paths = compute_search_paths(sources, manager.options,
manager.data_dir,
FileSystemCache())
t1 = time.time()
messages = self.fine_grained_manager.update(changed, removed)
t2 = time.time()
manager.log(
"fine-grained increment: find_changed: {:.3f}s, update: {:.3f}s".
format(t1 - t0, t2 - t1))
status = 1 if messages else 0
self.previous_sources = sources
return {
'out': ''.join(s + '\n' for s in messages),
'err': '',
'status': status
}
def main(args: List[str]) -> Iterator[Error]:
if len(args) == 1:
print('must provide at least one module to test')
sys.exit(1)
else:
modules = args[1:]
options = Options()
options.incremental = False
data_dir = default_data_dir()
search_path = compute_search_paths([], options, data_dir)
find_module_cache = FindModuleCache(search_path)
for module in modules:
for error in test_stub(options, find_module_cache, module):
yield error
def fine_grained_increment(
self,
sources: List[BuildSource],
remove: Optional[List[str]] = None,
update: Optional[List[str]] = None,
) -> List[str]:
"""Perform a fine-grained type checking increment.
If remove and update are None, determine changed paths by using
fswatcher. Otherwise, assume that only these files have changes.
Args:
sources: sources passed on the command line
remove: paths of files that have been removed
update: paths of files that have been changed or created
"""
assert self.fine_grained_manager is not None
manager = self.fine_grained_manager.manager
t0 = time.time()
if remove is None and update is None:
# Use the fswatcher to determine which files were changed
# (updated or added) or removed.
self.update_sources(sources)
changed, removed = self.find_changed(sources)
else:
# Use the remove/update lists to update fswatcher.
# This avoids calling stat() for unchanged files.
changed, removed = self.update_changed(sources, remove or [],
update or [])
changed += self.find_added_suppressed(self.fine_grained_manager.graph,
set(), manager.search_paths)
manager.search_paths = compute_search_paths(sources, manager.options,
manager.data_dir)
t1 = time.time()
manager.log("fine-grained increment: find_changed: {:.3f}s".format(t1 -
t0))
messages = self.fine_grained_manager.update(changed, removed)
t2 = time.time()
manager.log("fine-grained increment: update: {:.3f}s".format(t2 - t1))
manager.add_stats(find_changes_time=t1 - t0,
fg_update_time=t2 - t1,
files_changed=len(removed) + len(changed))
self.previous_sources = sources
return messages
def fine_grained_increment_follow_imports(
self, sources: List[BuildSource]) -> List[str]:
"""Like fine_grained_increment, but follow imports."""
t0 = time.time()
# TODO: Support file events
assert self.fine_grained_manager is not None
fine_grained_manager = self.fine_grained_manager
graph = fine_grained_manager.graph
manager = fine_grained_manager.manager
orig_modules = list(graph.keys())
self.update_sources(sources)
changed_paths = self.fswatcher.find_changed()
manager.search_paths = compute_search_paths(sources, manager.options,
manager.data_dir)
t1 = time.time()
manager.log("fine-grained increment: find_changed: {:.3f}s".format(t1 -
t0))
seen = {source.module for source in sources}
# Find changed modules reachable from roots (or in roots) already in graph.
changed, new_files = self.find_reachable_changed_modules(
sources, graph, seen, changed_paths)
sources.extend(new_files)
# Process changes directly reachable from roots.
messages = fine_grained_manager.update(changed, [])
# Follow deps from changed modules (still within graph).
worklist = changed[:]
while worklist:
module = worklist.pop()
if module[0] not in graph:
continue
sources2 = self.direct_imports(module, graph)
# Filter anything already seen before. This prevents
# infinite looping if there are any self edges. (Self
# edges are maybe a bug, but...)
sources2 = [
source for source in sources2 if source.module not in seen
]
changed, new_files = self.find_reachable_changed_modules(
sources2, graph, seen, changed_paths)
self.update_sources(new_files)
messages = fine_grained_manager.update(changed, [])
worklist.extend(changed)
t2 = time.time()
def refresh_file(module: str, path: str) -> List[str]:
return fine_grained_manager.update([(module, path)], [])
for module_id, state in list(graph.items()):
new_messages = refresh_suppressed_submodules(
module_id, state.path, fine_grained_manager.deps, graph,
self.fscache, refresh_file)
if new_messages is not None:
messages = new_messages
t3 = time.time()
# There may be new files that became available, currently treated as
# suppressed imports. Process them.
while True:
new_unsuppressed = self.find_added_suppressed(
graph, seen, manager.search_paths)
if not new_unsuppressed:
break
new_files = [
BuildSource(mod[1], mod[0]) for mod in new_unsuppressed
]
sources.extend(new_files)
self.update_sources(new_files)
messages = fine_grained_manager.update(new_unsuppressed, [])
for module_id, path in new_unsuppressed:
new_messages = refresh_suppressed_submodules(
module_id, path, fine_grained_manager.deps, graph,
self.fscache, refresh_file)
if new_messages is not None:
messages = new_messages
t4 = time.time()
# Find all original modules in graph that were not reached -- they are deleted.
to_delete = []
for module_id in orig_modules:
if module_id not in graph:
continue
if module_id not in seen:
module_path = graph[module_id].path
assert module_path is not None
to_delete.append((module_id, module_path))
if to_delete:
messages = fine_grained_manager.update([], to_delete)
fix_module_deps(graph)
self.previous_sources = find_all_sources_in_build(graph)
self.update_sources(self.previous_sources)
# Store current file state as side effect
self.fswatcher.find_changed()
t5 = time.time()
manager.log("fine-grained increment: update: {:.3f}s".format(t5 - t1))
manager.add_stats(find_changes_time=t1 - t0,
fg_update_time=t2 - t1,
refresh_suppressed_time=t3 - t2,
find_added_supressed_time=t4 - t3,
cleanup_time=t5 - t4)
return messages
