def setUp(self):
super().setUp()
init_subsystems([LibcDev, NativeToolchain], options={"libc": {"enable_libc_search": True}})
self.platform = Platform.current
self.toolchain = global_subsystem_instance(NativeToolchain)
gcc_subsystem = global_subsystem_instance(GCC)
self.gcc_version = gcc_subsystem.version()
llvm_subsystem = global_subsystem_instance(LLVM)
self.llvm_version = llvm_subsystem.version()
def _jar_dependency_management(self, **flags):
Subsystem.reset()
options = {
JarDependencyManagement.options_scope: flags,
}
return global_subsystem_instance(JarDependencyManagement,
options=options)
def run_flake8(
self,
source_files: List[FileContent],
*,
config: Optional[str] = None,
passthrough_args: Optional[Sequence[str]] = None,
interpreter_constraints: Optional[Sequence[str]] = None,
skip: bool = False,
) -> LintResult:
if config is not None:
self.create_file(relpath=".flake8", contents=config)
input_snapshot = self.request_single_product(
Snapshot, InputFilesContent(source_files))
target = Flake8Target(
PythonTargetAdaptor(
sources=EagerFilesetWithSpec('test', {'globs': []},
snapshot=input_snapshot),
address=Address.parse("test:target"),
compatibility=interpreter_constraints,
))
flake8_subsystem = global_subsystem_instance(
Flake8,
options={
Flake8.options_scope: {
"config": ".flake8" if config else None,
"args": passthrough_args or [],
"skip": skip,
}
})
return self.request_single_product(
LintResult,
Params(target, flake8_subsystem,
PythonNativeCode.global_instance(),
PythonSetup.global_instance(),
SubprocessEnvironment.global_instance()))
def _init_manager(self, **jar_dependency_management_options):
options = {
JarDependencyManagement.options_scope:
jar_dependency_management_options
}
return global_subsystem_instance(JarDependencyManagement,
options=options)
def setUp(self):
init_subsystems(
[LibcDev],
options={"libc": {"enable_libc_search": False, "libc_dir": "/does/not/exist",},},
)
self.libc = global_subsystem_instance(LibcDev)
self.platform = Platform.current
def setUp(self):
super().setUp()
init_subsystems([LibcDev, NativeToolchain],
options={
'libc': {
'enable_libc_search': True,
},
})
self.platform = Platform.current
self.toolchain = global_subsystem_instance(NativeToolchain)
self.rules = native_backend_rules() + create_platform_rules()
gcc_subsystem = global_subsystem_instance(GCC)
self.gcc_version = gcc_subsystem.version()
llvm_subsystem = global_subsystem_instance(LLVM)
self.llvm_version = llvm_subsystem.version()
def setUp(self):
init_subsystems([LibcDev], options={
'libc': {
'enable_libc_search': False,
'libc_dir': '/does/not/exist',
},
})
self.libc = global_subsystem_instance(LibcDev)
self.platform = Platform.current
def setUp(self):
init_subsystems([LibcDev], options={
'libc': {
'enable_libc_search': True,
'host_compiler': 'this_executable_does_not_exist',
},
})
self.libc = global_subsystem_instance(LibcDev)
self.platform = Platform.current
def test_parse_proxy_string(self):
ivy_subsystem = global_subsystem_instance(IvySubsystem)
self.assertEqual(
('example.com', 1234),
ivy_subsystem._parse_proxy_string('http://example.com:1234'))
self.assertEqual(
('secure-example.com', 999),
ivy_subsystem._parse_proxy_string('http://secure-example.com:999'))
# trailing slash is ok
self.assertEqual(
('example.com', 1234),
ivy_subsystem._parse_proxy_string('http://example.com:1234/'))
def setUp(self):
init_subsystems(
[LibcDev],
options={
"libc": {
"enable_libc_search": True,
"host_compiler": "this_executable_does_not_exist",
},
},
)
self.libc = global_subsystem_instance(LibcDev)
self.platform = Platform.current
def test_parse_proxy_string(self):
ivy_subsystem = global_subsystem_instance(IvySubsystem)
self.assertEqual(
("example.com", 1234),
ivy_subsystem._parse_proxy_string("http://example.com:1234"))
self.assertEqual(
("secure-example.com", 999),
ivy_subsystem._parse_proxy_string("http://secure-example.com:999"),
)
# trailing slash is ok
self.assertEqual(
("example.com", 1234),
ivy_subsystem._parse_proxy_string("http://example.com:1234/"))
def test_go_compile_simple(self):
with self.temporary_workdir() as workdir:
args = ['compile',
'contrib/go/examples/src/go/libA']
pants_run = self.run_pants_with_workdir(args, workdir)
self.assert_success(pants_run)
go_dist = global_subsystem_instance(GoDistribution)
goos = go_dist.create_go_cmd('env', args=['GOOS']).check_output().decode().strip()
goarch = go_dist.create_go_cmd('env', args=['GOARCH']).check_output().decode().strip()
expected_files = set('contrib.go.examples.src.go.{libname}.{libname}/'
'pkg/{goos}_{goarch}/{libname}.a'
.format(libname=libname, goos=goos, goarch=goarch)
for libname in ('libA', 'libB', 'libC', 'libD', 'libE'))
self.assertTrue(contains_exact_files(os.path.join(workdir, 'compile', 'go'),
expected_files, ignore_links=True))
def test_validate_live(self):
with self.assertRaises(Distribution.Error):
Distribution(bin_path=os.path.dirname(self.JAVA), minimum_version="999.9.9").validate()
with self.assertRaises(Distribution.Error):
Distribution(bin_path=os.path.dirname(self.JAVA), maximum_version="0.0.1").validate()
Distribution(bin_path=os.path.dirname(self.JAVA)).validate()
Distribution(bin_path=os.path.dirname(self.JAVA), minimum_version="1.3.1").validate()
Distribution(bin_path=os.path.dirname(self.JAVA), maximum_version="999.999.999").validate()
Distribution(
bin_path=os.path.dirname(self.JAVA),
minimum_version="1.3.1",
maximum_version="999.999.999",
).validate()
locator = global_subsystem_instance(DistributionLocator)
locator.cached(jdk=False)
def test_proxy_from_env(self):
ivy_subsystem = global_subsystem_instance(IvySubsystem)
self.assertIsNone(ivy_subsystem.http_proxy())
self.assertIsNone(ivy_subsystem.https_proxy())
with environment_as(
HTTP_PROXY='http://proxy.example.com:456',
HTTPS_PROXY='https://secure-proxy.example.com:789'):
self.assertEqual('http://proxy.example.com:456',
ivy_subsystem.http_proxy())
self.assertEqual('https://secure-proxy.example.com:789',
ivy_subsystem.https_proxy())
self.assertEqual([
'-Dhttp.proxyHost=proxy.example.com',
'-Dhttp.proxyPort=456',
'-Dhttps.proxyHost=secure-proxy.example.com',
'-Dhttps.proxyPort=789',
], ivy_subsystem.extra_jvm_options())
def test_go_compile_simple(self):
with self.temporary_workdir() as workdir:
args = ["compile", "contrib/go/examples/src/go/libA"]
pants_run = self.run_pants_with_workdir(args, workdir)
self.assert_success(pants_run)
go_dist = global_subsystem_instance(GoDistribution)
goos = go_dist.create_go_cmd(
"env", args=["GOOS"]).check_output().decode().strip()
goarch = go_dist.create_go_cmd(
"env", args=["GOARCH"]).check_output().decode().strip()
expected_files = set(
"contrib.go.examples.src.go.{libname}.{libname}/"
"pkg/{goos}_{goarch}/{libname}.a".format(
libname=libname, goos=goos, goarch=goarch)
for libname in ("libA", "libB", "libC", "libD", "libE"))
self.assertTrue(
contains_exact_files(os.path.join(workdir, "compile", "go"),
expected_files,
ignore_links=True))
def assert_timeout_calculated(self,
*,
field_value: Optional[int],
expected: Optional[int],
global_default: Optional[int] = None,
global_max: Optional[int] = None,
timeouts_enabled: bool = True) -> None:
pytest = global_subsystem_instance(
PyTest,
options={
PyTest.options_scope: {
"timeouts": timeouts_enabled,
"timeout_default": global_default,
"timeout_maximum": global_max,
}
},
)
field = PythonTestsTimeout(field_value,
address=Address.parse(":tests"))
assert field.calculate_from_global_options(pytest) == expected
def run_black(
self,
source_files: List[FileContent],
*,
config: Optional[str] = None,
passthrough_args: Optional[Sequence[str]] = None,
) -> Tuple[LintResult, FmtResult]:
if config is not None:
self.create_file(relpath="pyproject.toml", contents=config)
input_snapshot = self.request_single_product(
Snapshot, InputFilesContent(source_files))
target = FormattablePythonTarget(
TargetAdaptor(
sources=EagerFilesetWithSpec('test', {'globs': []},
snapshot=input_snapshot),
address=Address.parse("test:target"),
))
black_subsystem = global_subsystem_instance(
Black,
options={
Black.options_scope: {
"config": "pyproject.toml" if config else None,
"args": passthrough_args or [],
}
})
black_setup = self.request_single_product(
BlackSetup,
Params(
black_subsystem,
PythonNativeCode.global_instance(),
PythonSetup.global_instance(),
SubprocessEnvironment.global_instance(),
))
fmt_and_lint_params = Params(target, black_setup,
PythonSetup.global_instance(),
SubprocessEnvironment.global_instance())
lint_result = self.request_single_product(LintResult,
fmt_and_lint_params)
fmt_result = self.request_single_product(FmtResult,
fmt_and_lint_params)
return lint_result, fmt_result
def test_ignore_dir(self) -> None:
"""Verify that passing --repro-ignore option ignores the directory"""
# Buildroot is is based on your cwd so we need to step into a fresh
# directory for repro to look at.
root_instance = BuildRoot()
with temporary_dir() as build_root, \
root_instance.temporary(build_root), \
pushd(build_root), \
temporary_dir() as capture_dir:
add_file = partial(self.add_file, build_root)
add_file('pants.ini')
add_file('.git/foo', content='foo')
add_file('dist/bar', content='bar')
add_file('foo/bar', content='baz')
add_file('src/test1', content='test1')
add_file('src/test2', content='test1')
repro_file = Path(capture_dir, 'repro.tar.gz')
options = {
Reproducer.options_scope:
dict(
capture=str(repro_file),
ignore=['src'],
)
}
repro_sub = global_subsystem_instance(Reproducer, options=options)
repro = repro_sub.create_repro(
) # This is normally called in pants_exe.
repro.capture(run_info_dict={})
extract_loc = Path(capture_dir, 'extract')
TGZ.extract(str(repro_file), str(extract_loc))
self.assert_file(extract_loc, 'foo/bar', expected_content='baz')
assert_not_exists = partial(self.assert_not_exists, extract_loc)
assert_not_exists('.git')
assert_not_exists('src')
def run_isort(
self,
source_files: List[FileContent],
*,
config: Optional[str] = None,
passthrough_args: Optional[Sequence[str]] = None,
skip: bool = False,
) -> Tuple[LintResult, FmtResult]:
if config is not None:
self.create_file(relpath=".isort.cfg", contents=config)
input_snapshot = self.request_single_product(Snapshot, InputFilesContent(source_files))
target_adaptor = TargetAdaptor(
sources=EagerFilesetWithSpec('test', {'globs': []}, snapshot=input_snapshot),
address=Address.parse("test:target"),
)
lint_target = IsortTarget(target_adaptor)
fmt_target = IsortTarget(target_adaptor, prior_formatter_result_digest=input_snapshot.directory_digest)
isort_subsystem = global_subsystem_instance(
Isort, options={Isort.options_scope: {
"config": [".isort.cfg"] if config else None,
"args": passthrough_args or [],
"skip": skip,
}}
)
python_subsystems = [
PythonNativeCode.global_instance(),
PythonSetup.global_instance(),
SubprocessEnvironment.global_instance(),
]
isort_setup = self.request_single_product(
IsortSetup, Params(isort_subsystem, *python_subsystems)
)
lint_result = self.request_single_product(
LintResult, Params(lint_target, isort_setup, *python_subsystems)
)
fmt_result = self.request_single_product(
FmtResult, Params(fmt_target, isort_setup, *python_subsystems)
)
return lint_result, fmt_result
def test_ignore_dir(self) -> None:
"""Verify that passing --repro-ignore option ignores the directory."""
# Buildroot is is based on your cwd so we need to step into a fresh
# directory for repro to look at.
root_instance = BuildRoot()
with temporary_dir() as build_root, root_instance.temporary(
build_root), pushd(build_root), temporary_dir() as capture_dir:
add_file = partial(self.add_file, build_root)
add_file("pants.toml")
add_file(".git/foo", content="foo")
add_file("dist/bar", content="bar")
add_file("foo/bar", content="baz")
add_file("src/test1", content="test1")
add_file("src/test2", content="test1")
repro_file = Path(capture_dir, "repro.tar.gz")
options = {
Reproducer.options_scope:
dict(
capture=str(repro_file),
ignore=["src"],
)
}
repro_sub = global_subsystem_instance(Reproducer, options=options)
repro = repro_sub.create_repro(
) # This is normally called in pants_exe.
repro.capture(run_info_dict={})
extract_loc = Path(capture_dir, "extract")
TGZ.extract(str(repro_file), str(extract_loc))
self.assert_file(extract_loc, "foo/bar", expected_content="baz")
assert_not_exists = partial(self.assert_not_exists, extract_loc)
assert_not_exists(".git")
assert_not_exists("src")
def test_list_backends() -> None:
# NB: Here, we assume that the code to find all the `register.py`s is valid. Instead, the focus
# is on us being able to correctly extract all the relevant information from those
# `register.py` files and then to format the information.
all_register_pys = FilesContent([
FileContent(
"src/python/pants/backend/fortran/register.py",
dedent('''\
"""Support for Fortran 98."""
# V1 entry-point
def register_goals():
pass
# This naively looks like a V2 entry-point, but it's not!
def rules(x: int):
pass
''').encode(),
),
FileContent(
"contrib/elixir/src/python/pants/contrib/elixir/register.py",
dedent("""\
# V1 entry-point
def register_goals():
pass
# V2 entry-point
def rules():
pass
""").encode(),
),
FileContent(
"src/python/pants/core/register.py",
dedent('''\
"""Core V2 rules.
These are always activated.
"""
def rules():
pass
''').encode(),
),
])
console = MockConsole(use_colors=False)
run_rule(
list_backends,
rule_args=[
create_goal_subsystem(BackendsOptions, sep="\\n",
output_file=None),
global_subsystem_instance(SourceRootConfig),
global_subsystem_instance(GlobalOptions),
console,
],
mock_gets=[
MockGet(product_type=Snapshot,
subject_type=PathGlobs,
mock=lambda _: EMPTY_SNAPSHOT),
MockGet(product_type=FilesContent,
subject_type=Digest,
mock=lambda _: all_register_pys),
],
)
assert console.stdout.getvalue() == dedent("""\
V1 backends
-----------
To enable V1 backends, add the backend to `backend_packages.add` in your
`pants.toml`, like this:
[GLOBAL]
backend_packages.add = ["pants.backend.python"]
In the below list, all activated backends end with `*`.
pants.backend.fortran Support for Fortran 98.
pants.contrib.elixir <no description>
V2 backends
-----------
To enable V2 backends, add the backend to `backend_packages2.add` in your
`pants.toml`, like this:
[GLOBAL]
backend_packages2.add = ["pants.backend.python"]
In the below list, all activated backends end with `*`.
pants.contrib.elixir <no description>
pants.core* Core V2 rules. These are always activated.
""")
def setUp(self):
self.distribution = global_subsystem_instance(NodeDistribution)
def get_thrift_version(apache_thrift_gen):
thrift = global_subsystem_instance(Thrift).scoped_instance(apache_thrift_gen)
return thrift.get_options().version
def test_validate_live_jdk(self):
Distribution(bin_path=os.path.dirname(self.JAVAC), jdk=True).validate()
Distribution(bin_path=os.path.dirname(self.JAVAC),
jdk=True).binary("javap")
locator = global_subsystem_instance(DistributionLocator)
locator.cached(jdk=True)
def subprocess(self):
return global_subsystem_instance(Subprocess.Factory).create()
def distribution(self):
return global_subsystem_instance(GoDistribution)
def fetcher(self, import_path):
fetcher_factory = global_subsystem_instance(FetcherFactory)
return fetcher_factory.get_fetcher(import_path)
def _execute_junit_runner(self,
list_of_filename_content_tuples,
create_some_resources=True,
target_name=None):
# Create the temporary base test directory
test_rel_path = "tests/java/org/pantsbuild/foo"
# Create the temporary classes directory under work dir
test_classes_abs_path = self.create_workdir_dir(test_rel_path)
test_java_file_abs_paths = []
# Generate the temporary java test source code.
for filename, content in list_of_filename_content_tuples:
test_java_file_rel_path = os.path.join(test_rel_path, filename)
test_java_file_abs_path = self.create_file(test_java_file_rel_path,
content)
test_java_file_abs_paths.append(test_java_file_abs_path)
distribution = DistributionLocator.cached(jdk=True)
executor = SubprocessExecutor(distribution=distribution)
with temporary_file() as f:
res = executor.execute(
classpath=[
global_subsystem_instance(CoursierSubsystem).select(
self.context())
],
main="coursier.cli.Coursier",
args=[
"fetch",
"junit:junit:4.12",
"-r",
# This is needed to get around the maven blacklisting RBE.
"https://maven-central.storage-download.googleapis.com/maven2",
],
stdout=f,
)
self.assertEqual(0, res, "Coursier resolve failed.")
f.seek(0)
classpath = ":".join(f.read().decode().split())
# Now directly invoke javac to compile the test java code into classfiles that we can later
# inject into a product mapping for JUnitRun to execute against.
javac = distribution.binary("javac")
subprocess.check_call(
[javac, "-d", test_classes_abs_path, "-cp", classpath] +
test_java_file_abs_paths)
# If a target_name is specified create a target with it, otherwise create a junit_tests target.
if target_name:
target = self.target(target_name)
else:
target = self.create_library(
path=test_rel_path,
target_type="junit_tests",
name="foo_test",
sources=["FooTest.java"],
)
target_roots = []
if create_some_resources:
# Create a synthetic resource target.
target_roots.append(self.make_target("some_resources", Resources))
target_roots.append(target)
# Set the context with the two targets, one junit_tests target and
# one synthetic resources target.
# The synthetic resources target is to make sure we won't regress
# in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note
# in that bug, the resources target must be the first one in the list.
context = self.context(target_roots=target_roots)
# Before we run the task, we need to inject the "runtime_classpath" with
# the compiled test java classes that JUnitRun will know which test
# classes to execute. In a normal run, this "runtime_classpath" will be
# populated by java compilation step.
self.populate_runtime_classpath(context=context,
classpath=[test_classes_abs_path])
# Finally execute the task.
self.execute(context)
def _distribution_locator(distribution_locator_options=None):
options = {DistributionLocator.options_scope: distribution_locator_options or {}}
yield global_subsystem_instance(DistributionLocator, options=options)
def _execute_junit_runner(self,
list_of_filename_content_tuples,
create_some_resources=True,
target_name=None):
# Create the temporary base test directory
test_rel_path = 'tests/java/org/pantsbuild/foo'
test_abs_path = self.create_dir(test_rel_path)
# Create the temporary classes directory under work dir
test_classes_abs_path = self.create_workdir_dir(test_rel_path)
test_java_file_abs_paths = []
# Generate the temporary java test source code.
for filename, content in list_of_filename_content_tuples:
test_java_file_rel_path = os.path.join(test_rel_path, filename)
test_java_file_abs_path = self.create_file(test_java_file_rel_path,
content)
test_java_file_abs_paths.append(test_java_file_abs_path)
# Invoke ivy to resolve classpath for junit.
classpath_file_abs_path = os.path.join(test_abs_path,
'junit.classpath')
ivy_subsystem = global_subsystem_instance(IvySubsystem)
distribution = DistributionLocator.cached(jdk=True)
ivy = Bootstrapper(ivy_subsystem=ivy_subsystem).ivy()
ivy.execute(args=[
'-cachepath', classpath_file_abs_path, '-dependency', 'junit',
'junit-dep', '4.10'
],
executor=SubprocessExecutor(distribution=distribution))
with open(classpath_file_abs_path, 'r') as fp:
classpath = fp.read()
# Now directly invoke javac to compile the test java code into classfiles that we can later
# inject into a product mapping for JUnitRun to execute against.
javac = distribution.binary('javac')
subprocess.check_call(
[javac, '-d', test_classes_abs_path, '-cp', classpath] +
test_java_file_abs_paths)
# If a target_name is specified create a target with it, otherwise create a junit_tests target.
if target_name:
target = self.target(target_name)
else:
target = self.create_library(test_rel_path, 'junit_tests',
'foo_test', ['FooTest.java'])
target_roots = []
if create_some_resources:
# Create a synthetic resource target.
target_roots.append(self.make_target('some_resources', Resources))
target_roots.append(target)
# Set the context with the two targets, one junit_tests target and
# one synthetic resources target.
# The synthetic resources target is to make sure we won't regress
# in the future with bug like https://github.com/pantsbuild/pants/issues/508. Note
# in that bug, the resources target must be the first one in the list.
context = self.context(target_roots=target_roots)
# Before we run the task, we need to inject the "runtime_classpath" with
# the compiled test java classes that JUnitRun will know which test
# classes to execute. In a normal run, this "runtime_classpath" will be
# populated by java compilation step.
self.populate_runtime_classpath(context=context,
classpath=[test_classes_abs_path])
# Finally execute the task.
self.execute(context)
