def test_type_inference_with_timeout(self, gen_cache: MagicMock) -> None:
timeout = 1
timeout_error = subprocess.TimeoutExpired(cmd="pyre query ...",
timeout=timeout)
gen_cache.side_effect = timeout_error
paths = (str(self.DUMMY_PATH), )
type_caches = get_repo_caches(
paths, FullRepoMetadataConfig({TypeInferenceProvider}, timeout))
reports = next(
map_paths(
operation=self.mock_operation,
paths=paths,
config={DummyTypeDependentRule},
workers=LintWorkers.USE_CURRENT_THREAD,
metadata_caches=type_caches,
))
# Expecting a placeholder cache to have been plugged in instead.
self.mock_operation.assert_called_with(
self.DUMMY_PATH,
{DummyTypeDependentRule},
{TypeInferenceProvider: {
"types": []
}},
)
self.assertEqual(len(reports), 1)
def test_get_repo_caches_empty_paths(self, gen_cache: MagicMock) -> None:
repo_caches: Mapping[str, Dict[ProviderT, object]] = get_repo_caches(
[], FullRepoMetadataConfig({TypeInferenceProvider}, 1))
# We expect the call to resolve_cache to be bypassed.
gen_cache.assert_not_called()
self.assertEqual(repo_caches, {})
def get_metadata_caches(
cache_timeout: int, file_paths: Iterable[str]
) -> Mapping[str, Mapping["ProviderT", object]]:
"""
Returns a metadata cache for each file in ``file_paths``.
"""
logger: Logger = getLogger("Metadata Caches Logger")
handler = MetadataCacheErrorHandler()
logger.addHandler(handler)
full_repo_metadata_config: FullRepoMetadataConfig = FullRepoMetadataConfig(
providers={TypeInferenceProvider},
timeout_seconds=cache_timeout,
batch_size=100,
logger=logger,
)
metadata_caches = get_repo_caches(file_paths, full_repo_metadata_config)
# Let user know of any cache retrieval failures.
if handler.timeout_paths:
print("Unable to get metadata cache for the following paths:\n" +
"\n".join(handler.timeout_paths))
print_yellow(
"Try increasing the --cache-timeout value or passing fewer files.")
for k, v in handler.other_exceptions.items():
print(f"Encountered exception {k} for the following paths:\n" +
"\n".join(v))
print_yellow("Running `pyre start` may solve the issue.")
return metadata_caches
def test_get_repo_caches_single_path(self, gen_cache: MagicMock) -> None:
gen_cache.return_value = {self.DUMMY_PATH: {}}
repo_caches = get_repo_caches(
(self.DUMMY_PATH, ),
FullRepoMetadataConfig({TypeInferenceProvider}, 1))
gen_cache.assert_called_with(Path(""), [self.DUMMY_PATH], 1)
assert repo_caches == {self.DUMMY_PATH: {TypeInferenceProvider: {}}}
def test_get_repo_caches_honors_batch_size(self,
gen_cache: MagicMock) -> None:
paths = [f"{idx}_" + p for idx, p in enumerate([self.DUMMY_PATH] * 5)]
gen_cache.return_value = {}
get_repo_caches(
paths,
FullRepoMetadataConfig({TypeInferenceProvider}, 1, batch_size=2))
# A call to `gen_cache.__bool__()` will accompany every regular call to gen_cache() due to
# the implementation of cache resolving in `libcst.metadata.FullRepoManager`.
calls = [
call(Path(""), paths[:2], 1),
call(Path(""), paths[2:4], 1),
call(Path(""), [paths[4]], 1),
]
all_calls = chain.from_iterable([(call.__bool__(), c) for c in calls])
gen_cache.assert_has_calls(all_calls)
def test_get_repo_caches_with_logger(self, gen_cache: MagicMock) -> None:
gen_cache.side_effect = TimeoutExpired("pyre query ...", 1)
logger = logging.getLogger("Test")
class CustomHandler(logging.Handler):
errors = defaultdict(list)
def emit(self, record: logging.LogRecord) -> None:
exception_type = record.exc_info[0]
self.errors[exception_type] += record.__dict__["paths"]
hdlr = CustomHandler()
logger.addHandler(hdlr)
get_repo_caches(
[self.DUMMY_PATH],
FullRepoMetadataConfig({TypeInferenceProvider},
1,
batch_size=2,
logger=logger),
)
assert CustomHandler.errors == {TimeoutExpired: [self.DUMMY_PATH]}
def test_get_repo_caches_swallows_exception(self,
gen_cache: MagicMock) -> None:
gen_cache.side_effect = Exception()
repo_caches = get_repo_caches(
(self.DUMMY_PATH, ),
FullRepoMetadataConfig({TypeInferenceProvider}, 1))
gen_cache.assert_called_with(Path(""), [self.DUMMY_PATH], 1)
# Assert the placeholder cache is returned in the event of an error.
self.assertEqual(
repo_caches,
{self.DUMMY_PATH: {
TypeInferenceProvider: {
"types": []
}
}})
def ipc_main(opts: LintOpts) -> IPCResult:
"""
Given a LintOpts config with lint rules and lint success/failure report formatter,
this IPC helper took paths of source files from either a path file (with @paths arg)
or a list of paths as args. Results are formed as JSON and delimited by newlines.
It uses a multiprocess pool and the results are streamed to stdout as soon
as they're available.
Returns an IPCResult object.
"""
parser = argparse.ArgumentParser(
description=
"Runs Fixit lint rules and print results as console output.",
fromfile_prefix_chars="@",
parents=[get_multiprocessing_parser()],
)
parser.add_argument("paths",
nargs="*",
help="List of paths to run lint rules on.")
parser.add_argument("--prefix", help="A prefix to be added to all paths.")
args: argparse.Namespace = parser.parse_args()
paths: Generator[str, None, None] = (os.path.join(args.prefix, p)
if args.prefix else p
for p in args.paths)
full_repo_metadata_config = opts.full_repo_metadata_config
metadata_caches: Optional[Mapping[str, Mapping["ProviderT",
object]]] = None
if full_repo_metadata_config is not None:
metadata_caches = get_repo_caches(paths, full_repo_metadata_config)
results_iter: Iterator[Sequence[str]] = map_paths(
get_file_lint_result_json,
paths,
opts,
workers=args.workers,
metadata_caches=metadata_caches,
)
for results in results_iter:
# Use print outside of the executor to avoid multiple processes trying to write
# to stdout in parallel, which could cause a corrupted output.
for result in results:
print(result)
return IPCResult(args.paths)
def test_basic_type_inference_multiple_paths(self,
gen_cache: MagicMock) -> None:
paths = (str(self.DUMMY_PATH), str(self.DUMMY_PATH_2))
gen_cache.return_value = {path: self.fake_pyre_data for path in paths}
type_caches: Mapping[str, Dict[ProviderT, object]] = get_repo_caches(
paths, FullRepoMetadataConfig({TypeInferenceProvider}, 1))
all_reports = map_paths(
operation=self.mock_operation,
paths=paths,
config={DummyTypeDependentRule},
workers=LintWorkers.USE_CURRENT_THREAD,
metadata_caches=type_caches,
)
# Reports should be returned in order since we specified `LintWorkers.USE_CURRENT_THREAD`.
for i, reports in enumerate(all_reports):
self.assertEqual(len(reports), 1)
self.assertEqual(reports[0].file_path, Path(paths[i]))
def run_ipc(
opts: LintOpts,
paths: List[str],
prefix: Optional[str] = None,
workers: LintWorkers = LintWorkers.CPU_COUNT,
) -> IPCResult:
"""
Given a LintOpts config with lint rules and lint success/failure report formatter,
this IPC helper takes a path of source files (with an optional `prefix` that will be prepended).
Results are formed as JSON and delimited by newlines.
It uses a multiprocess pool and the results are streamed to stdout as soon
as they're available.
Returns an IPCResult object.
"""
paths: Generator[str, None,
None] = (os.path.join(prefix, p) if prefix else p
for p in paths)
full_repo_metadata_config = opts.full_repo_metadata_config
metadata_caches: Optional[Mapping[str, Mapping["ProviderT",
object]]] = None
if full_repo_metadata_config is not None:
metadata_caches = get_repo_caches(paths, full_repo_metadata_config)
results_iter: Iterator[Sequence[str]] = map_paths(
get_file_lint_result_json,
paths,
opts,
workers=workers,
metadata_caches=metadata_caches,
)
for results in results_iter:
# Use print outside of the executor to avoid multiple processes trying to write
# to stdout in parallel, which could cause a corrupted output.
for result in results:
print(result)
return IPCResult(list(paths))
def test_basic_type_inference(self, gen_cache: MagicMock) -> None:
# We want to intercept any calls that will require the pyre engine to be running.
gen_cache.return_value = {str(self.DUMMY_PATH): self.fake_pyre_data}
paths = (str(path) for path in [self.DUMMY_PATH])
type_caches: Mapping[str, Dict[ProviderT, object]] = get_repo_caches(
paths, FullRepoMetadataConfig({TypeInferenceProvider}, 1))
paths = (path for path in type_caches.keys())
reports = next(
map_paths(
operation=self.mock_operation,
paths=(path for path in type_caches.keys()),
config={DummyTypeDependentRule},
workers=LintWorkers.USE_CURRENT_THREAD,
metadata_caches=type_caches,
))
self.mock_operation.assert_called_with(
self.DUMMY_PATH, {DummyTypeDependentRule},
type_caches[str(self.DUMMY_PATH)])
self.assertEqual(len(reports), 1)
self.assertEqual(reports[0].file_path, self.DUMMY_PATH)