def test_type_inference_with_timeout(self, gen_cache: MagicMock) -> None: timeout = 1 timeout_error = subprocess.TimeoutExpired(cmd="pyre query ...", timeout=timeout) gen_cache.side_effect = timeout_error paths = (str(self.DUMMY_PATH), ) type_caches = get_repo_caches( paths, FullRepoMetadataConfig({TypeInferenceProvider}, timeout)) reports = next( map_paths( operation=self.mock_operation, paths=paths, config={DummyTypeDependentRule}, workers=LintWorkers.USE_CURRENT_THREAD, metadata_caches=type_caches, )) # Expecting a placeholder cache to have been plugged in instead. self.mock_operation.assert_called_with( self.DUMMY_PATH, {DummyTypeDependentRule}, {TypeInferenceProvider: { "types": [] }}, ) self.assertEqual(len(reports), 1)
def test_get_repo_caches_empty_paths(self, gen_cache: MagicMock) -> None: repo_caches: Mapping[str, Dict[ProviderT, object]] = get_repo_caches( [], FullRepoMetadataConfig({TypeInferenceProvider}, 1)) # We expect the call to resolve_cache to be bypassed. gen_cache.assert_not_called() self.assertEqual(repo_caches, {})
def get_metadata_caches( cache_timeout: int, file_paths: Iterable[str] ) -> Mapping[str, Mapping["ProviderT", object]]: """ Returns a metadata cache for each file in ``file_paths``. """ logger: Logger = getLogger("Metadata Caches Logger") handler = MetadataCacheErrorHandler() logger.addHandler(handler) full_repo_metadata_config: FullRepoMetadataConfig = FullRepoMetadataConfig( providers={TypeInferenceProvider}, timeout_seconds=cache_timeout, batch_size=100, logger=logger, ) metadata_caches = get_repo_caches(file_paths, full_repo_metadata_config) # Let user know of any cache retrieval failures. if handler.timeout_paths: print("Unable to get metadata cache for the following paths:\n" + "\n".join(handler.timeout_paths)) print_yellow( "Try increasing the --cache-timeout value or passing fewer files.") for k, v in handler.other_exceptions.items(): print(f"Encountered exception {k} for the following paths:\n" + "\n".join(v)) print_yellow("Running `pyre start` may solve the issue.") return metadata_caches
def test_get_repo_caches_single_path(self, gen_cache: MagicMock) -> None: gen_cache.return_value = {self.DUMMY_PATH: {}} repo_caches = get_repo_caches( (self.DUMMY_PATH, ), FullRepoMetadataConfig({TypeInferenceProvider}, 1)) gen_cache.assert_called_with(Path(""), [self.DUMMY_PATH], 1) assert repo_caches == {self.DUMMY_PATH: {TypeInferenceProvider: {}}}
def test_get_repo_caches_honors_batch_size(self, gen_cache: MagicMock) -> None: paths = [f"{idx}_" + p for idx, p in enumerate([self.DUMMY_PATH] * 5)] gen_cache.return_value = {} get_repo_caches( paths, FullRepoMetadataConfig({TypeInferenceProvider}, 1, batch_size=2)) # A call to `gen_cache.__bool__()` will accompany every regular call to gen_cache() due to # the implementation of cache resolving in `libcst.metadata.FullRepoManager`. calls = [ call(Path(""), paths[:2], 1), call(Path(""), paths[2:4], 1), call(Path(""), [paths[4]], 1), ] all_calls = chain.from_iterable([(call.__bool__(), c) for c in calls]) gen_cache.assert_has_calls(all_calls)
def test_get_repo_caches_with_logger(self, gen_cache: MagicMock) -> None: gen_cache.side_effect = TimeoutExpired("pyre query ...", 1) logger = logging.getLogger("Test") class CustomHandler(logging.Handler): errors = defaultdict(list) def emit(self, record: logging.LogRecord) -> None: exception_type = record.exc_info[0] self.errors[exception_type] += record.__dict__["paths"] hdlr = CustomHandler() logger.addHandler(hdlr) get_repo_caches( [self.DUMMY_PATH], FullRepoMetadataConfig({TypeInferenceProvider}, 1, batch_size=2, logger=logger), ) assert CustomHandler.errors == {TimeoutExpired: [self.DUMMY_PATH]}
def test_get_repo_caches_swallows_exception(self, gen_cache: MagicMock) -> None: gen_cache.side_effect = Exception() repo_caches = get_repo_caches( (self.DUMMY_PATH, ), FullRepoMetadataConfig({TypeInferenceProvider}, 1)) gen_cache.assert_called_with(Path(""), [self.DUMMY_PATH], 1) # Assert the placeholder cache is returned in the event of an error. self.assertEqual( repo_caches, {self.DUMMY_PATH: { TypeInferenceProvider: { "types": [] } }})
def ipc_main(opts: LintOpts) -> IPCResult: """ Given a LintOpts config with lint rules and lint success/failure report formatter, this IPC helper took paths of source files from either a path file (with @paths arg) or a list of paths as args. Results are formed as JSON and delimited by newlines. It uses a multiprocess pool and the results are streamed to stdout as soon as they're available. Returns an IPCResult object. """ parser = argparse.ArgumentParser( description= "Runs Fixit lint rules and print results as console output.", fromfile_prefix_chars="@", parents=[get_multiprocessing_parser()], ) parser.add_argument("paths", nargs="*", help="List of paths to run lint rules on.") parser.add_argument("--prefix", help="A prefix to be added to all paths.") args: argparse.Namespace = parser.parse_args() paths: Generator[str, None, None] = (os.path.join(args.prefix, p) if args.prefix else p for p in args.paths) full_repo_metadata_config = opts.full_repo_metadata_config metadata_caches: Optional[Mapping[str, Mapping["ProviderT", object]]] = None if full_repo_metadata_config is not None: metadata_caches = get_repo_caches(paths, full_repo_metadata_config) results_iter: Iterator[Sequence[str]] = map_paths( get_file_lint_result_json, paths, opts, workers=args.workers, metadata_caches=metadata_caches, ) for results in results_iter: # Use print outside of the executor to avoid multiple processes trying to write # to stdout in parallel, which could cause a corrupted output. for result in results: print(result) return IPCResult(args.paths)
def test_basic_type_inference_multiple_paths(self, gen_cache: MagicMock) -> None: paths = (str(self.DUMMY_PATH), str(self.DUMMY_PATH_2)) gen_cache.return_value = {path: self.fake_pyre_data for path in paths} type_caches: Mapping[str, Dict[ProviderT, object]] = get_repo_caches( paths, FullRepoMetadataConfig({TypeInferenceProvider}, 1)) all_reports = map_paths( operation=self.mock_operation, paths=paths, config={DummyTypeDependentRule}, workers=LintWorkers.USE_CURRENT_THREAD, metadata_caches=type_caches, ) # Reports should be returned in order since we specified `LintWorkers.USE_CURRENT_THREAD`. for i, reports in enumerate(all_reports): self.assertEqual(len(reports), 1) self.assertEqual(reports[0].file_path, Path(paths[i]))
def run_ipc( opts: LintOpts, paths: List[str], prefix: Optional[str] = None, workers: LintWorkers = LintWorkers.CPU_COUNT, ) -> IPCResult: """ Given a LintOpts config with lint rules and lint success/failure report formatter, this IPC helper takes a path of source files (with an optional `prefix` that will be prepended). Results are formed as JSON and delimited by newlines. It uses a multiprocess pool and the results are streamed to stdout as soon as they're available. Returns an IPCResult object. """ paths: Generator[str, None, None] = (os.path.join(prefix, p) if prefix else p for p in paths) full_repo_metadata_config = opts.full_repo_metadata_config metadata_caches: Optional[Mapping[str, Mapping["ProviderT", object]]] = None if full_repo_metadata_config is not None: metadata_caches = get_repo_caches(paths, full_repo_metadata_config) results_iter: Iterator[Sequence[str]] = map_paths( get_file_lint_result_json, paths, opts, workers=workers, metadata_caches=metadata_caches, ) for results in results_iter: # Use print outside of the executor to avoid multiple processes trying to write # to stdout in parallel, which could cause a corrupted output. for result in results: print(result) return IPCResult(list(paths))
def test_basic_type_inference(self, gen_cache: MagicMock) -> None: # We want to intercept any calls that will require the pyre engine to be running. gen_cache.return_value = {str(self.DUMMY_PATH): self.fake_pyre_data} paths = (str(path) for path in [self.DUMMY_PATH]) type_caches: Mapping[str, Dict[ProviderT, object]] = get_repo_caches( paths, FullRepoMetadataConfig({TypeInferenceProvider}, 1)) paths = (path for path in type_caches.keys()) reports = next( map_paths( operation=self.mock_operation, paths=(path for path in type_caches.keys()), config={DummyTypeDependentRule}, workers=LintWorkers.USE_CURRENT_THREAD, metadata_caches=type_caches, )) self.mock_operation.assert_called_with( self.DUMMY_PATH, {DummyTypeDependentRule}, type_caches[str(self.DUMMY_PATH)]) self.assertEqual(len(reports), 1) self.assertEqual(reports[0].file_path, self.DUMMY_PATH)