def test_accepts_single_file_snapshot(self): input_files_content = InputFilesContent( (FileContent(path="subdir/a.txt", content=b"test file contents"),) ) snapshot = self.request_single_product(Snapshot, input_files_content) assert SingleFileExecutable(snapshot).exe_filename == "./subdir/a.txt" input_files_content = InputFilesContent( (FileContent(path="some_silly_file_name", content=b"test file contents"),) ) snapshot = self.request_single_product(Snapshot, input_files_content) assert SingleFileExecutable(snapshot).exe_filename == "./some_silly_file_name"
async def generate_pants_ini(console: Console, workspace: Workspace) -> GeneratePantsIni: pants_ini_content = dedent(f"""\ [GLOBAL] pants_version: {pants_version} """) preexisting_snapshot = await Get[Snapshot](PathGlobs(include=('pants.ini',))) if preexisting_snapshot.files: console.print_stderr( "./pants.ini already exists. This goal is only meant to be run the first time you run Pants " "in a project.\n\nTo update config values, please directly modify the file." ) return GeneratePantsIni(exit_code=1) console.print_stdout(dedent(f"""\ Adding sensible defaults to ./pants.ini: * Pinning `pants_version` to `{pants_version}`. """)) digest = await Get[Digest](InputFilesContent([ FileContent(path='pants.ini', content=pants_ini_content.encode()) ])) workspace.materialize_directory(DirectoryToMaterialize(digest)) console.print_stdout( "You may modify these values directly in the file at any time. The ./pants script will detect " "any changes the next time you run it.\n\nYou are now ready to use Pants!" ) return GeneratePantsIni(exit_code=0)
def test_snapshot_subset_globs(self) -> None: ss = SnapshotSubset( directory_digest=self.generate_original_digest(), globs=PathGlobs(("a.txt", "c.txt", "subdir2/**")), ) subset_snapshot = self.request_single_product(Snapshot, ss) assert set(subset_snapshot.files) == { "a.txt", "c.txt", "subdir2/a.txt", "subdir2/nested_subdir/x.txt", } assert set(subset_snapshot.dirs) == {"subdir2/nested_subdir"} content = b"dummy content" subset_input = InputFilesContent( ( FileContent(path="a.txt", content=content), FileContent(path="c.txt", content=content), FileContent(path="subdir2/a.txt", content=content), FileContent(path="subdir2/nested_subdir/x.txt", content=content), ) ) subset_digest = self.request_single_product(Digest, subset_input) assert subset_snapshot.directory_digest == subset_digest
def run_flake8( self, source_files: List[FileContent], *, config: Optional[str] = None, passthrough_args: Optional[Sequence[str]] = None, interpreter_constraints: Optional[Sequence[str]] = None, skip: bool = False, ) -> LintResult: if config is not None: self.create_file(relpath=".flake8", contents=config) input_snapshot = self.request_single_product( Snapshot, InputFilesContent(source_files)) target = Flake8Target( PythonTargetAdaptor( sources=EagerFilesetWithSpec('test', {'globs': []}, snapshot=input_snapshot), address=Address.parse("test:target"), compatibility=interpreter_constraints, )) flake8_subsystem = global_subsystem_instance( Flake8, options={ Flake8.options_scope: { "config": ".flake8" if config else None, "args": passthrough_args or [], "skip": skip, } }) return self.request_single_product( LintResult, Params(target, flake8_subsystem, PythonNativeCode.global_instance(), PythonSetup.global_instance(), SubprocessEnvironment.global_instance()))
def run_black( self, source_files: List[FileContent], *, config: Optional[str] = None, passthrough_args: Optional[str] = None, skip: bool = False, ) -> Tuple[LintResult, FmtResult]: args = ["--backend-packages2=pants.backend.python.lint.black"] if config is not None: self.create_file(relpath="pyproject.toml", contents=config) args.append("--black-config=pyproject.toml") if passthrough_args: args.append(f"--black-args='{passthrough_args}'") if skip: args.append(f"--black-skip") input_snapshot = self.request_single_product(Snapshot, InputFilesContent(source_files)) target_adaptor = TargetAdaptor( sources=EagerFilesetWithSpec('test', {'globs': []}, snapshot=input_snapshot), address=Address.parse("test:target"), ) lint_target = BlackTarget(target_adaptor) fmt_target = BlackTarget(target_adaptor, prior_formatter_result_digest=input_snapshot.directory_digest) options_bootstrapper = create_options_bootstrapper(args=args) lint_result = self.request_single_product(LintResult, Params(lint_target, options_bootstrapper)) fmt_result = self.request_single_product(FmtResult, Params(fmt_target, options_bootstrapper)) return lint_result, fmt_result
def get_coverage_plugin_input() -> InputFilesContent: return InputFilesContent( FilesContent((FileContent( path=f"{COVERAGE_PLUGIN_MODULE_NAME}.py", content=pkg_resources.resource_string(__name__, "coverage_plugin/plugin.py"), ), )))
def test_pex_execution(self) -> None: input_files_content = InputFilesContent( ( FileContent(path="main.py", content=b'print("from main")'), FileContent(path="subdir/sub.py", content=b'print("from sub")'), ) ) input_files = self.request_single_product(Digest, input_files_content) pex_output = self.create_pex_and_get_all_data(entry_point="main", input_files=input_files) pex_files = pex_output["files"] self.assertTrue("pex" not in pex_files) self.assertTrue("main.py" in pex_files) self.assertTrue("subdir/sub.py" in pex_files) init_subsystem(PythonSetup) python_setup = PythonSetup.global_instance() env = {"PATH": create_path_env_var(python_setup.interpreter_search_paths)} req = ExecuteProcessRequest( argv=("python", "test.pex"), env=env, input_files=pex_output["pex"].directory_digest, description="Run the pex and make sure it works", ) result = self.request_single_product(ExecuteProcessResult, req) self.assertEqual(result.stdout, b"from main\n")
def test_workspace_materialize_directories_result(self): #TODO(#8336): at some point, this test should require that Workspace only be invoked from a console_role workspace = Workspace(self.scheduler) input_files_content = InputFilesContent(( FileContent(path='a.txt', content=b'hello'), FileContent(path='subdir/b.txt', content=b'goodbye'), )) digest = self.request_single_product(Digest, input_files_content) path1 = Path('a.txt') path2 = Path('subdir/b.txt') assert not path1.is_file() assert not path2.is_file() output = workspace.materialize_directories( (DirectoryToMaterialize(digest), )) assert type(output) == MaterializeDirectoriesResult materialize_result = output.dependencies[0] assert type(materialize_result) == MaterializeDirectoryResult assert materialize_result.output_paths == tuple( str(Path(self.build_root, p)) for p in [path1, path2])
def test_workspace_materialize_directories_result(self): #TODO(#8336): at some point, this test should require that Workspace only be invoked from a console_role workspace = Workspace(self.scheduler) input_files_content = InputFilesContent(( FileContent(path='a.txt', content=b'hello', is_executable=False), FileContent(path='subdir/b.txt', content=b'goodbye', is_executable=False), )) digest, = self.scheduler.product_request(Digest, [input_files_content]) with temporary_dir() as tmp_dir: path1 = Path(tmp_dir, 'a.txt') path2 = Path(tmp_dir, 'subdir', 'b.txt') self.assertFalse(path1.is_file()) self.assertFalse(path2.is_file()) output = workspace.materialize_directories( (DirectoryToMaterialize(path=tmp_dir, directory_digest=digest), )) self.assertEqual(type(output), MaterializeDirectoriesResult) materialize_result = output.dependencies[0] self.assertEqual(type(materialize_result), MaterializeDirectoryResult) self.assertEqual(materialize_result.output_paths, ( str(Path(tmp_dir, 'a.txt')), str(Path(tmp_dir, 'subdir/b.txt')), ))
def run_fmt_rule(self, *, targets: List[HydratedTarget]) -> Tuple[Fmt, str]: result_digest = self.request_single_product( Digest, InputFilesContent([ FileContent(path=str(self.formatted_file), content=self.formatted_content.encode()) ])) console = MockConsole(use_colors=False) result: Fmt = run_rule( fmt, rule_args=[ console, HydratedTargets(targets), Workspace(self.scheduler), UnionMembership( union_rules={FormatTarget: [PythonTargetAdaptor]}) ], mock_gets=[ MockGet(product_type=AggregatedFmtResults, subject_type=PythonTargetAdaptor, mock=lambda adaptor: AggregatedFmtResults( (FmtResult(digest=result_digest, stdout=f"Formatted `{adaptor.name}`", stderr=""), ), combined_digest=result_digest)), MockGet(product_type=Digest, subject_type=DirectoriesToMerge, mock=lambda _: result_digest), ], ) return result, console.stdout.getvalue()
def test(self): msg = MessageToConsoleRule(input_files_content=InputFilesContent( [FileContent(path='a.txt', content=b'hello')])) output_path = Path(self.build_root, 'a.txt') self.assert_console_output_contains(str(output_path), additional_params=[msg]) assert output_path.read_text() == "hello"
def run_bandit( self, source_files: List[FileContent], *, config: Optional[str] = None, passthrough_args: Optional[str] = None, interpreter_constraints: Optional[str] = None, skip: bool = False, ) -> LintResult: args = ["--backend-packages2=pants.backend.python.lint.bandit"] if config: # TODO: figure out how to get this file to exist... self.create_file(relpath=".bandit", contents=config) args.append("--bandit-config=.bandit") if passthrough_args: args.append(f"--bandit-args={passthrough_args}") if skip: args.append(f"--bandit-skip") input_snapshot = self.request_single_product(Snapshot, InputFilesContent(source_files)) target = BanditTarget( PythonTargetAdaptor( sources=EagerFilesetWithSpec('test', {'globs': []}, snapshot=input_snapshot), address=Address.parse("test:target"), compatibility=[interpreter_constraints] if interpreter_constraints else None, ) ) return self.request_single_product( LintResult, Params(target, create_options_bootstrapper(args=args)), )
def test_generic_pex_creation(self) -> None: input_files_content = InputFilesContent(( FileContent(path='main.py', content=b'print("from main")'), FileContent(path='subdir/sub.py', content=b'print("from sub")'), )) input_files = self.request_single_product(Digest, input_files_content) pex_output = self.create_pex_and_get_all_data(entry_point='main', input_files=input_files) pex_files = pex_output['files'] self.assertTrue('pex' not in pex_files) self.assertTrue('main.py' in pex_files) self.assertTrue('subdir/sub.py' in pex_files) python_setup = PythonSetup.global_instance() env = { "PATH": create_path_env_var(python_setup.interpreter_search_paths) } pex = pex_output['pex'] req = ExecuteProcessRequest( argv=('python', 'test.pex'), env=env, input_files=pex.directory_digest, description="Run the pex and make sure it works") result = self.request_single_product(ExecuteProcessResult, req) self.assertEqual(result.stdout, b"from main\n")
def get_coveragerc_input(coveragerc_content: str) -> InputFilesContent: return InputFilesContent([ FileContent( path='.coveragerc', content=coveragerc_content.encode(), is_executable=False, ), ])
def create_mock_binary(self, program_text: bytes) -> CreatedBinary: input_files_content = InputFilesContent(( FileContent(path='program.py', content=program_text, is_executable=True), )) digest = self.request_single_product(Digest, input_files_content) return CreatedBinary( binary_name='program.py', digest=digest, )
async def create_coverage_config( coverage_config_request: CoverageConfigRequest, source_root_config: SourceRootConfig ) -> CoverageConfig: sources = await Get[SourceFiles]( AllSourceFilesRequest( (tgt.get(Sources) for tgt in coverage_config_request.targets), strip_source_roots=False, ) ) init_injected = await Get[InitInjectedSnapshot](InjectInitRequest(sources.snapshot)) source_roots = source_root_config.get_source_roots() # Generate a map from source root stripped source to its source root. eg: # {'pants/testutil/subsystem/util.py': 'src/python'}. This is so that coverage reports # referencing /chroot/path/pants/testutil/subsystem/util.py can be mapped back to the actual # sources they reference when generating coverage reports. def stripped_file_with_source_root(file_name: str) -> Tuple[str, str]: source_root_object = source_roots.find_by_path(file_name) source_root = source_root_object.path if source_root_object is not None else "" stripped_path = file_name[len(source_root) + 1 :] return stripped_path, source_root stripped_files_to_source_roots = dict( stripped_file_with_source_root(filename) for filename in sorted(init_injected.snapshot.files) ) default_config = dedent( """ [run] branch = True timid = False relative_files = True """ ) config_parser = configparser.ConfigParser() config_parser.read_file(StringIO(default_config)) config_parser.set("run", "plugins", COVERAGE_PLUGIN_MODULE_NAME) config_parser.add_section(COVERAGE_PLUGIN_MODULE_NAME) config_parser.set( COVERAGE_PLUGIN_MODULE_NAME, "source_to_target_base", json.dumps(stripped_files_to_source_roots), ) config_parser.set( COVERAGE_PLUGIN_MODULE_NAME, "test_time", json.dumps(coverage_config_request.is_test_time) ) config_io_stream = StringIO() config_parser.write(config_io_stream) digest = await Get[Digest]( InputFilesContent( [FileContent(".coveragerc", content=config_io_stream.getvalue().encode())] ) ) return CoverageConfig(digest)
async def generate_fortran_from_avro(request: GenerateFortranFromAvroRequest) -> GeneratedSources: protocol_files = request.protocol_sources.files def generate_fortran(fp: str) -> FileContent: parent = str(PurePath(fp).parent).replace("src/avro", "src/fortran") file_name = f"{PurePath(fp).stem}.f95" return FileContent(str(PurePath(parent, file_name)), b"Generated") result = await Get[Snapshot](InputFilesContent([generate_fortran(fp) for fp in protocol_files])) return GeneratedSources(result)
def make_target_with_origin( self, source_files: List[FileContent], *, origin: Optional[OriginSpec] = None, ) -> TargetAdaptorWithOrigin: input_snapshot = self.request_single_product(Snapshot, InputFilesContent(source_files)) adaptor = TargetAdaptor( sources=EagerFilesetWithSpec("test", {"globs": []}, snapshot=input_snapshot), address=Address.parse("test:target"), ) if origin is None: origin = SingleAddress(directory="test", name="target") return TargetAdaptorWithOrigin(adaptor, origin)
def test(self): with temporary_dir() as tmp_dir: input_files_content = InputFilesContent(( FileContent(path='a.txt', content=b'hello', is_executable=False), )) msg = MessageToConsoleRule(tmp_dir=tmp_dir, input_files_content=input_files_content) output_path = str(Path(tmp_dir, 'a.txt')) self.assert_console_output_contains(output_path, additional_params=[msg]) contents = open(output_path).read() self.assertEqual(contents, 'hello')
def test_raises_with_multiple_files(self): input_files_content = InputFilesContent(( FileContent(path='a.txt', content=b'test file contents'), FileContent(path='b.txt', content=b'more test file contents'), )) snapshot = self.request_single_product(Snapshot, input_files_content) with self.assertRaisesWithMessage( SingleFileExecutable.ValidationError, f'snapshot {snapshot} used for {SingleFileExecutable} should have exactly 1 file!'): SingleFileExecutable(snapshot)
def generate_original_digest(self) -> Digest: content = b'dummy content' input_files_content = InputFilesContent(( FileContent(path='a.txt', content=content), FileContent(path='b.txt', content=content), FileContent(path='c.txt', content=content), FileContent(path='subdir/a.txt', content=content), FileContent(path='subdir/b.txt', content=content), FileContent(path='subdir2/a.txt', content=content), FileContent(path='subdir2/nested_subdir/x.txt', content=content), )) return self.request_single_product(Digest, input_files_content)
def make_ipr(self) -> InteractiveProcessRequest: input_files_content = InputFilesContent( (FileContent(path="program.py", content=b"def test(): pass"), )) digest = self.request_single_product(Digest, input_files_content) return InteractiveProcessRequest( argv=( "/usr/bin/python", "program.py", ), run_in_workspace=False, input_digest=digest, )
async def generate_chroot(request: SetupPyChrootRequest) -> SetupPyChroot: if request.py2: # TODO: Implement Python 2 support. This will involve, among other things: merging ancestor # __init__.py files into the chroot, detecting packages based on the presence of __init__.py, # and inspecting all __init__.py files for the namespace package incantation. raise UnsupportedPythonVersion('Running setup.py commands not supported for Python 2.') owned_deps = await Get[OwnedDependencies](DependencyOwner(request.exported_target)) targets = HydratedTargets(od.hydrated_target for od in owned_deps) sources = await Get[SetupPySources](SetupPySourcesRequest(targets)) requirements = await Get[ExportedTargetRequirements](DependencyOwner(request.exported_target)) # Nest the sources under the src/ prefix. src_digest = await Get[Digest](DirectoryWithPrefixToAdd(sources.digest, CHROOT_SOURCE_ROOT)) # Generate the kwargs to the setup() call. setup_kwargs = request.exported_target.hydrated_target.adaptor.provides.setup_py_keywords.copy() setup_kwargs.update({ 'package_dir': {'': CHROOT_SOURCE_ROOT}, 'packages': sources.packages, 'namespace_packages': sources.namespace_packages, 'package_data': dict(sources.package_data), 'install_requires': requirements.requirement_strs }) ht = request.exported_target.hydrated_target key_to_binary_spec = getattr(ht.adaptor.provides, 'binaries', {}) keys = list(key_to_binary_spec.keys()) binaries = await MultiGet(Get[HydratedTarget]( Address, Address.parse(key_to_binary_spec[key], relative_to=ht.address.spec_path)) for key in keys) for key, binary in zip(keys, binaries): if (not isinstance(binary.adaptor, PythonBinaryAdaptor) or getattr(binary.adaptor, 'entry_point', None) is None): raise InvalidEntryPoint( f'The binary {key} exported by {ht.address.reference()} is not a valid entry point.') entry_points = setup_kwargs['entry_points'] = setup_kwargs.get('entry_points', {}) console_scripts = entry_points['console_scripts'] = entry_points.get('console_scripts', []) console_scripts.append(f'{key}={binary.adaptor.entry_point}') # Generate the setup script. setup_py_content = SETUP_BOILERPLATE.format( target_address_spec=ht.address.reference(), setup_kwargs_str=distutils_repr(setup_kwargs) ).encode() extra_files_digest = await Get[Digest]( InputFilesContent([ FileContent('setup.py', setup_py_content), FileContent('MANIFEST.in', 'include *.py'.encode()) # Make sure setup.py is included. ])) chroot_digest = await Get[Digest](DirectoriesToMerge((src_digest, extra_files_digest))) return SetupPyChroot(chroot_digest, json.dumps(setup_kwargs, sort_keys=True))
def test_add_prefix(self) -> None: input_files_content = InputFilesContent(( FileContent(path="main.py", content=b'print("from main")'), FileContent(path="subdir/sub.py", content=b'print("from sub")'), )) digest = self.request_single_product(Digest, input_files_content) output_digest = self.request_single_product( Digest, AddPrefix(digest, "outer_dir")) snapshot = self.request_single_product(Snapshot, output_digest) assert sorted(snapshot.files) == [ "outer_dir/main.py", "outer_dir/subdir/sub.py" ] assert sorted(snapshot.dirs) == ["outer_dir", "outer_dir/subdir"]
def test_not_executable(self): file_name = "echo.sh" file_contents = b'#!/bin/bash -eu\necho "Hello"\n' input_file = InputFilesContent((FileContent(path=file_name, content=file_contents),)) digest = self.request_single_product(Digest, input_file) req = ExecuteProcessRequest( argv=("./echo.sh",), input_files=digest, description="cat the contents of this file", ) with self.assertRaisesWithMessageContaining(ExecutionError, "Permission"): self.request_single_product(ExecuteProcessResult, req)
def make_ipr(self, content: bytes) -> InteractiveProcessRequest: input_files_content = InputFilesContent( (FileContent(path='program.py', content=content, is_executable=True), )) digest = self.request_single_product(Digest, input_files_content) return InteractiveProcessRequest( argv=( "/usr/bin/python", "program.py", ), run_in_workspace=False, input_files=digest, )
async def inject_missing_init_files(request: InjectInitRequest) -> InitInjectedSnapshot: """Ensure that every package has an `__init__.py` file in it. This will preserve any `__init__.py` files already in the input snapshot. """ snapshot = request.snapshot missing_init_files = sorted(identify_missing_init_files(snapshot.files)) if not missing_init_files: return InitInjectedSnapshot(snapshot) generated_inits_digest = await Get[Digest]( InputFilesContent(FileContent(path=fp, content=b"") for fp in missing_init_files) ) result = await Get[Snapshot](MergeDigests((snapshot.digest, generated_inits_digest))) return InitInjectedSnapshot(result)
def test_add_prefix(self): input_files_content = InputFilesContent(( FileContent(path='main.py', content=b'print("from main")', is_executable=False), FileContent(path='subdir/sub.py', content=b'print("from sub")', is_executable=False), )) digest, = self.scheduler.product_request(Digest, [input_files_content]) dpa = DirectoryWithPrefixToAdd(digest, "outer_dir") output_digest, = self.scheduler.product_request(Digest, [dpa]) snapshot, = self.scheduler.product_request(Snapshot, [output_digest]) self.assertEqual(sorted(snapshot.files), ['outer_dir/main.py', 'outer_dir/subdir/sub.py']) self.assertEqual(sorted(snapshot.dirs), ['outer_dir', 'outer_dir/subdir'])
def test_executable(self): file_name = "echo.sh" file_contents = b'#!/bin/bash -eu\necho "Hello"\n' input_file = InputFilesContent( (FileContent(path=file_name, content=file_contents, is_executable=True),) ) digest = self.request_single_product(Digest, input_file) req = ExecuteProcessRequest( argv=("./echo.sh",), input_files=digest, description="cat the contents of this file", ) result = self.request_single_product(ExecuteProcessResult, req) self.assertEqual(result.stdout, b"Hello\n")
def test_file_in_directory_creation(self): path = "somedir/filename" content = b"file contents" input_file = InputFilesContent((FileContent(path=path, content=content),)) digest = self.request_single_product(Digest, input_file) req = ExecuteProcessRequest( argv=("/bin/cat", "somedir/filename"), input_files=digest, description="Cat a file in a directory to make sure that doesn't break", ) result = self.request_single_product(ExecuteProcessResult, req) self.assertEqual(result.stdout, content)