def test_dependees_type(self):
SourceRoot.register('tests', PythonTests)
self.assert_console_output(
'tests/d:d',
args=['--test-type=python_tests'],
targets=[self.target('common/d')]
)
def test_exported_antlr(self):
SourceRoot.register('src/antlr', PythonThriftLibrary)
self.create_file(relpath='src/antlr/exported/exported.g', contents=dedent("""
grammar exported;
options {
language = Python;
}
WORD: ('a'..'z'|'A'..'Z'|'0'..'9'|'-'|'_')+;
static: WORD;
"""))
target = self.make_target(spec='src/antlr/exported',
target_type=PythonAntlrLibrary,
antlr_version='3.1.3',
sources=['exported.g'],
module='exported',
provides=PythonArtifact(name='test.exported', version='0.0.0.'))
# TODO(John Sirois): Find a way to get easy access to pants option defaults.
self.set_options_for_scope('ivy',
ivy_profile='2.4.0',
ivy_settings=None,
cache_dir=os.path.expanduser('~/.ivy2/pants'),
http_proxy=None,
https_proxy=None)
with self.run_execute(target) as created:
self.assertEqual([target], created)
def _sibling_is_test(source_set):
"""Determine if a SourceSet represents a test path.
Non test targets that otherwise live in test target roots (say a java_library), must
be marked as test for IDEA to correctly link the targets with the test code that uses
them. Therefore we check to see if the source root registered to the path or any of its sibling
source roots are defined with a test type.
:param source_set: SourceSet to analyze
:returns: True if the SourceSet represents a path containing tests
"""
def has_test_type(types):
for target_type in types:
# TODO(Eric Ayers) Find a way for a target to identify itself instead of a hard coded list
if target_type in (JavaTests, PythonTests):
return True
return False
if source_set.path:
path = os.path.join(source_set.source_base, source_set.path)
else:
path = source_set.source_base
sibling_paths = SourceRoot.find_siblings_by_path(path)
for sibling_path in sibling_paths:
if has_test_type(SourceRoot.types(sibling_path)):
return True
return False
def test_exported_antlr(self):
SourceRoot.register('src/antlr', PythonThriftLibrary)
self.create_file(relpath='src/antlr/exported/exported.g', contents=dedent("""
grammar exported;
options {
language = Python;
}
WORD: ('a'..'z'|'A'..'Z'|'0'..'9'|'-'|'_')+;
static: WORD;
"""))
target = self.make_target(spec='src/antlr/exported',
target_type=PythonAntlrLibrary,
antlr_version='3.1.3',
sources=['exported.g'],
module='exported',
provides=PythonArtifact(name='test.exported', version='0.0.0'))
# TODO(John Sirois): This hacks around a direct but undeclared dependency
# `pants.java.distribution.distribution.Distribution` gained in
# https://rbcommons.com/s/twitter/r/2657
# Remove this once proper Subsystem dependency chains are re-established.
with subsystem_instance(JVM):
with self.run_execute(target) as created:
self.assertEqual([target], created.keys())
def test_resolve_and_inject_failure(self):
go_fetch = self.create_task(self.context())
SourceRoot.register(os.path.join(self.build_root, '3rdparty'), Target)
r1 = self.make_target(spec='3rdparty/r1', target_type=Target)
with self.assertRaises(go_fetch.UndeclaredRemoteLibError) as cm:
go_fetch._resolve_and_inject(r1, 'r2')
self.assertEqual(cm.exception.spec_path, '3rdparty/r2')
def test_resolve_simple(self):
SourceRoot.register('3rdparty/node', NodeRemoteModule)
typ = self.make_target(spec='3rdparty/node:typ', target_type=NodeRemoteModule, version='0.6.3')
SourceRoot.register('src/node', NodeModule)
self.create_file('src/node/util/util.js', contents=dedent("""
var typ = require('typ');
console.log("type of boolean is: " + typ.BOOLEAN);
"""))
target = self.make_target(spec='src/node/util',
target_type=NodeModule,
sources=['util.js'],
dependencies=[typ])
context = self.context(target_roots=[target])
task = self.create_task(context)
task.execute()
node_paths = context.products.get_data(NodePaths)
node_path = node_paths.node_path(target)
self.assertIsNotNone(node_path)
script_path = os.path.join(node_path, 'util.js')
out = task.node_distribution.node_command(args=[script_path]).check_output()
self.assertIn('type of boolean is: boolean', out)
def test_protos_extracted_under_build_root(self):
"""This testcase shows that you can put sources for protos outside the directory where the
BUILD file is defined. This will be the case for .proto files that have been extracted
under .pants.d.
"""
# place a .proto file in a place outside of where the BUILD file is defined
extracted_source_path = os.path.join(self.build_root, 'extracted-source')
SourceRoot.register(extracted_source_path, JavaProtobufLibrary)
safe_mkdir(os.path.join(extracted_source_path, 'sample-package'))
sample_proto_path = os.path.join(extracted_source_path, 'sample-package', 'sample.proto')
with open(sample_proto_path, 'w') as sample_proto:
sample_proto.write(dedent('''
package com.example;
message sample {}
'''))
self.add_to_build_file('sample', dedent('''
java_protobuf_library(name='sample',
sources=['{sample_proto_path}'],
)''').format(sample_proto_path=sample_proto_path))
target = self.target("sample:sample")
context = self.context(target_roots=[target])
task = self.create_task(context=context)
sources_by_base = task._calculate_sources([target])
self.assertEquals(['extracted-source'], sources_by_base.keys())
self.assertEquals(OrderedSet([sample_proto_path]), sources_by_base['extracted-source'])
def add_new_target(self, address, target_type, target_base=None, dependencies=None,
derived_from=None, **kwargs):
"""Creates a new target, adds it to the context and returns it.
This method ensures the target resolves files against the given target_base, creating the
directory if needed and registering a source root.
"""
target_base = os.path.join(get_buildroot(), target_base or address.spec_path)
if not os.path.exists(target_base):
os.makedirs(target_base)
if not SourceRoot.find_by_path(target_base):
SourceRoot.register(target_base)
if dependencies:
dependencies = [dep.address for dep in dependencies]
self.build_graph.inject_synthetic_target(address=address,
target_type=target_type,
dependencies=dependencies,
derived_from=derived_from,
**kwargs)
new_target = self.build_graph.get_target(address)
if derived_from:
self._synthetic_targets[derived_from].append(new_target)
return new_target
def test_dependees_type(self):
SourceRoot.register('tests', PythonTests)
self.assert_console_output(
'tests/d:d',
targets=[self.target('common/d')],
options={'type': ['python_tests']}
)
def test_multiple_local_roots_failure(self):
SourceRoot.register('src/go', GoBinary)
SourceRoot.register('src2/go', GoLibrary)
context = self.context(target_roots=[self.make_target('src/go/fred', GoBinary)])
task = self.create_task(context)
with self.assertRaises(task.InvalidLocalRootsError):
task.execute()
def test_noop_applicable_targets_complete_graph(self):
SourceRoot.register('src/go', GoBinary, GoLibrary)
self.create_file(relpath='src/go/jane/bar.go', contents=dedent("""
package jane
var PublicConstant = 42
"""))
jane = self.make_target('src/go/jane', GoLibrary)
self.create_file(relpath='src/go/fred/foo.go', contents=dedent("""
package main
import (
"fmt"
"jane"
)
func main() {
fmt.Printf("Hello %s!", jane.PublicConstant)
}
"""))
fred = self.make_target('src/go/fred', GoBinary, dependencies=[jane])
context = self.context(target_roots=[fred])
expected = context.targets()
task = self.create_task(context)
task.execute()
self.assertEqual(expected, context.targets())
def test_sources_generated_by_target(self):
root_path = os.path.join('project', 'src', 'main', 'wire')
wire_path = os.path.join(root_path, 'wire-lib')
file_path = os.path.join(wire_path, 'org', 'pantsbuild', 'example', 'foo.proto')
SourceRoot.register(root_path)
self.add_to_build_file(wire_path, dedent('''
java_wire_library(name='wire-target',
sources=['{0}'],
)
'''.format(os.path.relpath(file_path, wire_path))))
self.create_dir(os.path.dirname(file_path))
self.create_file(file_path, dedent('''
package org.pantsbuild.example;
message Foo {
optional string bar = 1;
optional string foobar = 2;
}
'''))
target = self.target('project/src/main/wire/wire-lib:wire-target')
context = self.context(target_roots=[target])
task = self.create_task(context)
previous_working_directory = os.path.abspath('.')
os.chdir(os.path.abspath(self.build_root))
result = task.sources_generated_by_target(target)
os.chdir(previous_working_directory)
self.assertEquals(OrderedSet(['org/pantsbuild/example/Foo.java']), OrderedSet(result))
def stitch_deps_local(self):
SourceRoot.register('src/go', GoBinary, GoLibrary)
self.create_file(relpath='src/go/jane/bar.go', contents=dedent("""
package jane
var PublicConstant = 42
"""))
self.create_file(relpath='src/go/fred/foo.go', contents=dedent("""
package main
import (
"fmt"
"jane"
)
func main() {
fmt.Printf("Hello %s!", jane.PublicConstant)
}
"""))
fred = self.make_target('src/go/fred', GoBinary)
context = self.context(target_roots=[fred])
self.assertEqual([fred], context.target_roots)
pre_execute_files = self.buildroot_files()
task = self.create_task(context)
task.execute()
jane = self.target('src/go/jane')
self.assertIsNotNone(jane)
self.assertEqual([jane], fred.dependencies)
self.assertEqual({jane, fred}, set(context.targets()))
return pre_execute_files
def test_symlink_remote_lib(self):
with pushd(self.build_root):
with temporary_dir() as d:
SourceRoot.register('3rdparty')
spec = '3rdparty/github.com/user/lib'
remote_lib_src_dir = os.path.join(d, spec)
self.create_file(os.path.join(remote_lib_src_dir, 'file.go'))
go_remote_lib = self.make_target(spec=spec, target_type=GoRemoteLibrary)
context = self.context()
go_remote_lib_src = context.products.get_data('go_remote_lib_src',
init_func=lambda: defaultdict(str))
go_remote_lib_src[go_remote_lib] = remote_lib_src_dir
ws_task = self.create_task(context)
gopath = ws_task.get_gopath(go_remote_lib)
ws_task._symlink_remote_lib(gopath, go_remote_lib, set())
workspace_dir = os.path.join(gopath, 'src/github.com/user/lib')
self.assertTrue(os.path.isdir(workspace_dir))
link = os.path.join(workspace_dir, 'file.go')
self.assertEqual(os.readlink(link), os.path.join(remote_lib_src_dir, 'file.go'))
def setUp(self):
super(PythonReplTest, self).setUp()
SourceRoot.register("3rdparty", PythonRequirementLibrary)
SourceRoot.register("src", PythonBinary, PythonLibrary)
self.six = self.create_python_requirement_library("3rdparty/six", "six", requirements=["six==1.9.0"])
self.requests = self.create_python_requirement_library(
"3rdparty/requests", "requests", requirements=["requests==2.6.0"]
)
self.library = self.create_python_library(
"src/lib",
"lib",
{
"lib.py": dedent(
"""
import six
def go():
six.print_('go', 'go', 'go!', sep='')
"""
)
},
dependencies=["//3rdparty/six"],
)
self.binary = self.create_python_binary("src/bin", "bin", "lib.go", dependencies=["//src/lib"])
self.non_python_target = self.create_non_python_target("src/java", "java")
def test_multiple_remote_roots_failure(self):
SourceRoot.register('3rdparty/go', GoRemoteLibrary)
SourceRoot.register('src/go', GoLibrary, GoRemoteLibrary)
context = self.context(target_roots=[self.make_target('src/go/fred', GoLibrary)])
task = self.create_task(context)
with self.assertRaises(task.InvalidRemoteRootsError):
task.execute()
def test_resolve_and_inject_explicit_failure(self):
SourceRoot.register(os.path.join(self.build_root, '3rdparty'), GoRemoteLibrary)
r1 = self.make_target(spec='3rdparty/r1', target_type=GoRemoteLibrary)
go_fetch = self.create_task(self.context())
with self.assertRaises(go_fetch.UndeclaredRemoteLibError) as cm:
go_fetch._resolve(r1, self.address('3rdparty/r2'), 'r2', implict_ok=False)
self.assertEqual(cm.exception.address, self.address('3rdparty/r2'))
def test_resolve_and_inject_implicit(self):
SourceRoot.register(os.path.join(self.build_root, '3rdparty'), GoRemoteLibrary)
r1 = self.make_target(spec='3rdparty/r1', target_type=GoRemoteLibrary)
go_fetch = self.create_task(self.context())
r2 = go_fetch._resolve(r1, self.address('3rdparty/r2'), 'r2', implict_ok=True)
self.assertEqual(self.address('3rdparty/r2'), r2.address)
self.assertIsInstance(r2, GoRemoteLibrary)
def test_exported_thrift_issues_2005(self):
# Issue #2005 highlighted the fact the PythonThriftBuilder was building both a given
# PythonThriftLibrary's thrift files as well as its transitive dependencies thrift files.
# We test here to ensure that independently published PythonThriftLibraries each only own their
# own thrift stubs and the proper dependency links exist between the distributions.
SourceRoot.register('src/thrift', PythonThriftLibrary)
self.create_file(relpath='src/thrift/exported/exported.thrift', contents=dedent("""
namespace py exported
const set<string> VALID_IDENTIFIERS = ["Hello", "World", "!"]
"""))
target1 = self.make_target(spec='src/thrift/exported',
target_type=PythonThriftLibrary,
sources=['exported.thrift'],
provides=PythonArtifact(name='test.exported', version='0.0.0'))
self.create_file(relpath='src/thrift/exported_dependee/exported_dependee.thrift',
contents=dedent("""
namespace py exported_dependee
include "exported/exported.thrift"
const set<string> ALIASED_IDENTIFIERS = exported.VALID_IDENTIFIERS
"""))
target2 = self.make_target(spec='src/thrift/exported_dependee',
target_type=PythonThriftLibrary,
sources=['exported_dependee.thrift'],
dependencies=[target1],
provides=PythonArtifact(name='test.exported_dependee',
version='0.0.0'))
with self.run_execute(target2, recursive=True) as created:
self.assertEqual({target2, target1}, set(created.keys()))
with self.extracted_sdist(sdist=created[target1],
expected_prefix='test.exported-0.0.0') as (py_files, path):
self.assertEqual({path('setup.py'),
path('src/__init__.py'),
path('src/exported/__init__.py'),
path('src/exported/constants.py'),
path('src/exported/ttypes.py')},
py_files)
self.assertFalse(os.path.exists(path('src/test.exported.egg-info/requires.txt')))
with self.extracted_sdist(sdist=created[target2],
expected_prefix='test.exported_dependee-0.0.0') as (py_files, path):
self.assertEqual({path('setup.py'),
path('src/__init__.py'),
path('src/exported_dependee/__init__.py'),
path('src/exported_dependee/constants.py'),
path('src/exported_dependee/ttypes.py')},
py_files)
requirements = path('src/test.exported_dependee.egg-info/requires.txt')
self.assertTrue(os.path.exists(requirements))
with open(requirements) as fp:
self.assertEqual('test.exported==0.0.0', fp.read().strip())
def setUp(self):
super(PythonEvalTest, self).setUp()
SourceRoot.register('src', PythonBinary, PythonLibrary)
self.a_library = self.create_python_library('src/a', 'a', {'a.py': dedent("""
import inspect
def compile_time_check_decorator(cls):
if not inspect.isclass(cls):
raise TypeError('This decorator can only be applied to classes, given {}'.format(cls))
return cls
""")})
self.b_library = self.create_python_library('src/b', 'b', {'b.py': dedent("""
from a.a import compile_time_check_decorator
@compile_time_check_decorator
class BarB(object):
pass
""")})
self.b_library = self.create_python_library('src/c', 'c', {'c.py': dedent("""
from a.a import compile_time_check_decorator
@compile_time_check_decorator
def baz_c():
pass
""")}, dependencies=['//src/a'])
def fix_c_source():
self.create_file('src/c/c.py', contents=dedent("""
from a.a import compile_time_check_decorator
# Change from decorated function baz_c to decorated class BazC.
@compile_time_check_decorator
class BazC(object):
pass
"""))
self.fix_c_source = fix_c_source
self.d_library = self.create_python_library('src/d', 'd', { 'd.py': dedent("""
from a.a import compile_time_check_decorator
@compile_time_check_decorator
class BazD(object):
pass
""")}, dependencies=['//src/a'])
self.e_binary = self.create_python_binary('src/e', 'e', 'a.a', dependencies=['//src/a'])
self.f_binary = self.create_python_binary('src/f', 'f', 'a.a:compile_time_check_decorator',
dependencies=['//src/a'])
self.g_binary = self.create_python_binary('src/g', 'g', 'a.a:does_not_exist',
dependencies=['//src/a'])
self.h_binary = self.create_python_binary('src/h', 'h', 'a.a')
def test_reset(self):
self._assert_source_root_empty()
SourceRoot.register("tests", TestTarget)
self.assertEquals({"tests": OrderedSet([TestTarget])}, SourceRoot.all_roots())
SourceRoot.reset()
self._assert_source_root_empty()
def test_cannot_sources(self):
SourceRoot.register('3rdparty/go', GoRemoteLibrary)
self.add_to_build_file('3rdparty/go/github.com/foo/bar', dedent("""
go_remote_library(dependencies=[])
"""))
with self.assertRaises(AddressLookupError):
self.target('3rdparty/go/github.com/foo/bar')
def test_resolve_and_inject_explicit(self):
SourceRoot.register(os.path.join(self.build_root, "3rdparty"), GoRemoteLibrary)
r1 = self.make_target(spec="3rdparty/r1", target_type=GoRemoteLibrary)
r2 = self.make_target(spec="3rdparty/r2", target_type=GoRemoteLibrary)
go_fetch = self.create_task(self.context())
resolved = go_fetch._resolve(r1, self.address("3rdparty/r2"), "r2", implict_ok=False)
self.assertEqual(r2, resolved)
def test_here(self):
target = TestTarget("//mock/foo/bar:baz")
proxy = AddressableCallProxy(addressable_type=target.get_addressable_type(),
build_file=None,
registration_callback=None)
self.assertEqual("mock/foo/bar", SourceRoot.find(target))
SourceRoot("mock/foo").here(proxy)
self.assertEqual("mock/foo", SourceRoot.find(target))
def register_sourceroot():
try:
yield SourceRoot.register
except (ValueError, IndexError) as e:
print('SourceRoot Registration Failed.')
raise e
finally:
SourceRoot.reset()
def test_register(self):
self._assert_source_root_empty()
SourceRoot.register("tests", TestTarget)
self.assertEquals({"tests": OrderedSet([TestTarget])}, SourceRoot.all_roots())
self.assertEquals(OrderedSet([TestTarget]), SourceRoot.types("tests"))
self.assertEquals(OrderedSet(["tests"]), SourceRoot.roots(TestTarget))
def test_cannot_name(self):
SourceRoot.register('src/go', self.target_type)
self.add_to_build_file('src/go/src/foo', dedent("""
{target_alias}(name='bob')
""".format(target_alias=self.target_type.alias())))
with self.assertRaises(AddressLookupError):
self.target('src/go/src/foo')
def test_antlr(self):
SourceRoot.register('src/antlr', PythonThriftLibrary)
self.create_file(relpath='src/antlr/word/word.g', contents=dedent("""
grammar word;
options {
language=Python;
output=AST;
}
WORD: ('a'..'z'|'A'..'Z'|'!')+;
word_up: WORD (' ' WORD)*;
"""))
antlr_target = self.make_target(spec='src/antlr/word',
target_type=PythonAntlrLibrary,
antlr_version='3.1.3',
sources=['word.g'],
module='word')
SourceRoot.register('src/python', PythonBinary)
antlr3 = self.make_target(spec='3rdparty/python:antlr3',
target_type=PythonRequirementLibrary,
requirements=[PythonRequirement('antlr_python_runtime==3.1.3')])
self.create_file(relpath='src/python/test/main.py', contents=dedent("""
import antlr3
from word import wordLexer, wordParser
def word_up():
input = 'Hello World!'
char_stream = antlr3.ANTLRStringStream(input)
lexer = wordLexer.wordLexer(char_stream)
tokens = antlr3.CommonTokenStream(lexer)
parser = wordParser.wordParser(tokens)
def print_node(node):
print(node.text)
visitor = antlr3.tree.TreeVisitor()
visitor.visit(parser.word_up().tree, pre_action=print_node)
"""))
binary = self.make_target(spec='src/python/test',
target_type=PythonBinary,
source='main.py',
dependencies=[antlr_target, antlr3])
with self.dumped_chroot([binary]) as (pex_builder, python_chroot):
pex_builder.set_entry_point('test.main:word_up')
pex_builder.freeze()
pex = python_chroot.pex()
process = pex.run(blocking=False, stdout=subprocess.PIPE)
stdout, _ = process.communicate()
self.assertEqual(0, process.returncode)
self.assertEqual(['Hello', ' ', 'World!'], stdout.splitlines())
def setUp(self):
super(TestPythonThriftBuilder, self).setUp()
SourceRoot.register(os.path.realpath(os.path.join(self.build_root, 'test_thrift_replacement')),
PythonThriftLibrary)
self.add_to_build_file('test_thrift_replacement', dedent('''
python_thrift_library(name='one',
sources=['thrift/keyword.thrift'],
)
'''))
def test_resolve_and_inject(self):
go_fetch = self.create_task(self.context())
SourceRoot.register(os.path.join(self.build_root, '3rdparty'), Target)
r1 = self.make_target(spec='3rdparty/r1', target_type=Target)
self.add_to_build_file('3rdparty/r2',
'target(name="{}")'.format('r2'))
r2 = go_fetch._resolve_and_inject(r1, 'r2')
self.assertEqual(r2.name, 'r2')
self.assertItemsEqual(r1.dependencies, [r2])
def test_transitive_download_remote_libs_simple(self):
with temporary_dir() as src:
with temporary_dir() as zipdir:
SourceRoot.register('3rdparty', GoRemoteLibrary)
dep_graph = {'r1': ['r2'], 'r2': ['r3'], 'r3': []}
self._init_dep_graph_files(src, zipdir, dep_graph)
r1 = self.target('3rdparty/localzip/r1')
context = self._create_fetch_context(zipdir)
go_fetch = self.create_task(context)
undeclared_deps = go_fetch._transitive_download_remote_libs(
{r1})
self.assertEqual(undeclared_deps, {})
self._assert_dependency_graph(r1, dep_graph)
def test_default_name_and_sources(self):
SourceRoot.register('src/go', self.target_type)
self.create_file('src/go/src/foo/jake.go')
self.create_file('src/go/src/foo/sub/jane.go')
self.add_to_build_file(
'src/go/src/foo',
dedent("""
{target_alias}()
""".format(target_alias=self.target_type.alias())))
go_local_source_target = self.target('src/go/src/foo')
self.assertIsNotNone(go_local_source_target)
self.assertEqual('src/foo', go_local_source_target.import_path)
self.assertEqual(
['src/foo/jake.go'],
list(go_local_source_target.sources_relative_to_source_root()))
def test_globs_resources(self):
SourceRoot.register('src/go', self.target_type)
# We shouldn't grab these - no BUILDs, no dirents, no subdir files.
self.create_file('src/go/src/foo/BUILD')
self.create_file('src/go/src/foo/subpackage/jane.go')
self.create_file('src/go/src/foo/subpackage/jane.png')
# We should grab all of these though.
self.create_file('src/go/src/foo/jake.go')
self.create_file('src/go/src/foo/jake.png')
target = self.make_target(spec='src/go/src/foo',
target_type=self.target_type)
self.assertEqual(sorted(['src/foo/jake.go', 'src/foo/jake.png']),
sorted(target.sources_relative_to_source_root()))
def _calculate_sources(self, targets):
gentargets = OrderedSet()
def add_to_gentargets(target):
if self.is_gentarget(target):
gentargets.add(target)
self.context.build_graph.walk_transitive_dependency_graph(
[target.address for target in targets],
add_to_gentargets,
postorder=True)
sources_by_base = OrderedDict()
# TODO(Eric Ayers) Extract this logic for general use? When using unpacked_jars it is needed
# to get the correct source root for paths outside the current BUILD tree.
for target in gentargets:
for source in target.sources_relative_to_buildroot():
base = SourceRoot.find_by_path(source)
if not base:
base, _ = target.target_base, target.sources_relative_to_buildroot(
)
self.context.log.debug(
'Could not find source root for {source}.'
' Missing call to SourceRoot.register()? Fell back to {base}.'
.format(source=source, base=base))
if base not in sources_by_base:
sources_by_base[base] = OrderedSet()
sources_by_base[base].add(source)
return sources_by_base
def collect_proto_paths(dep):
if not dep.has_sources():
return
for source in dep.sources_relative_to_buildroot():
if source.endswith('.proto'):
root = SourceRoot.find_by_path(source)
if root:
proto_paths.add(os.path.join(get_buildroot(), root))
def setUp(self):
super(AntlrGenTest, self).setUp()
SourceRoot.register('{srcroot}'.format(**self.PARTS), JavaAntlrLibrary)
for ver in self.VERSIONS:
self.create_file(
relpath='{srcroot}/{dir}/{prefix}.g{version}'.format(
version=ver, **self.PARTS),
contents=dedent("""
grammar {prefix};
////////////////////
start : letter EOF ;
letter : LETTER ;
////////////////////
fragment LETTER : 'a'..'z' | 'A'..'Z' ;
""".format(**self.PARTS)))
def test_calculate_sources_with_source_root(self):
SourceRoot.register('project/src/main/wire')
self.add_to_build_file(
'project/src/main/wire/wire-lib',
dedent('''
java_wire_library(name='wire-target',
sources=['foo.proto'],
)
'''))
target = self.target('project/src/main/wire/wire-lib:wire-target')
context = self.context(target_roots=[target])
task = self.create_task(context)
result = task._calculate_sources([target])
self.assertEquals(1, len(result.keys()))
self.assertEquals(
OrderedSet(['project/src/main/wire/wire-lib/foo.proto']),
result['project/src/main/wire'])
def test_compiler_args_wirev1(self):
self._create_fake_wire_tool()
SourceRoot.register('wire-src')
wire_targetv1 = self.make_target(
'wire-src:wire-targetv1',
JavaWireLibrary,
sources=['bar.proto'],
service_writer='org.pantsbuild.DummyServiceWriter',
service_writer_options=['opt1', 'opt2'])
task = self.create_task(self.context(target_roots=[wire_targetv1]))
self.assertEquals([
'--java_out={}/{}/wire-src.wire-targetv1'.format(
self.build_root, self.EXPECTED_TASK_PATH),
'--service_writer=org.pantsbuild.DummyServiceWriter',
'--service_writer_opt', 'opt1', '--service_writer_opt', 'opt2',
'--proto_path={}/wire-src'.format(self.build_root), 'bar.proto'
], task.format_args_for_target(wire_targetv1))
def console_output(self, _):
buildfiles = OrderedSet()
if self._dependees_type:
base_paths = OrderedSet()
for dependees_type in self._dependees_type:
target_aliases = self.context.build_file_parser.registered_aliases(
).targets
if dependees_type not in target_aliases:
raise TaskError('Invalid type name: %s' % dependees_type)
target_type = target_aliases[dependees_type]
# Try to find the SourceRoot for the given input type
try:
roots = SourceRoot.roots(target_type)
base_paths.update(roots)
except KeyError:
pass
if not base_paths:
raise TaskError(
'No SourceRoot set for any target type in %s.' %
self._dependees_type +
'\nPlease define a source root in BUILD file as:' +
'\n\tsource_root(\'<src-folder>\', %s)' %
', '.join(self._dependees_type))
for base_path in base_paths:
buildfiles.update(
BuildFile.scan_buildfiles(
get_buildroot(),
os.path.join(get_buildroot(), base_path),
spec_excludes=self._spec_excludes))
else:
buildfiles = BuildFile.scan_buildfiles(
get_buildroot(), spec_excludes=self._spec_excludes)
build_graph = self.context.build_graph
build_file_parser = self.context.build_file_parser
dependees_by_target = defaultdict(set)
for build_file in buildfiles:
address_map = build_file_parser.parse_build_file(build_file)
for address in address_map.keys():
build_graph.inject_address_closure(address)
for address in address_map.keys():
target = build_graph.get_target(address)
# TODO(John Sirois): tighten up the notion of targets written down in a BUILD by a
# user vs. targets created by pants at runtime.
target = self.get_concrete_target(target)
for dependency in target.dependencies:
dependency = self.get_concrete_target(dependency)
dependees_by_target[dependency].add(target)
roots = set(self.context.target_roots)
if self._closed:
for root in roots:
yield root.address.spec
for dependant in self.get_dependants(dependees_by_target, roots):
yield dependant.address.spec
def test_compiler_args_wirev2(self):
self._create_fake_wire_tool(version='2.0.0')
SourceRoot.register('wire-src')
wire_targetv2 = self.make_target(
'wire-src:wire-targetv2',
JavaWireLibrary,
sources=['baz.proto'],
service_factory='org.pantsbuild.DummyServiceFactory',
service_factory_options=['v2opt1', 'v2opt2'])
task = self.create_task(self.context(target_roots=[wire_targetv2]))
self.assertEquals([
'--java_out={}/{}/wire-src.wire-targetv2'.format(
self.build_root, self.EXPECTED_TASK_PATH),
'--service_factory=org.pantsbuild.DummyServiceFactory',
'--service_factory_opt', 'v2opt1', '--service_factory_opt',
'v2opt2', '--proto_path={}/wire-src'.format(
self.build_root), 'baz.proto'
], task.format_args_for_target(wire_targetv2))
def add_new_target(self, address, target_type, dependencies=None, **kwargs):
"""Creates a new target, adds it to the context and returns it.
This method ensures the target resolves files against the given target_base, creating the
directory if needed and registering a source root.
"""
target_base = os.path.join(get_buildroot(), address.spec_path)
if not os.path.exists(target_base):
os.makedirs(target_base)
SourceRoot.register(address.spec_path)
if dependencies:
dependencies = [dep.address for dep in dependencies]
self.build_graph.inject_synthetic_target(address=address,
target_type=target_type,
dependencies=dependencies,
**kwargs)
return self.build_graph.get_target(address)
def test_resolve_remote(self):
SourceRoot.register('3rdparty/node', NodeRemoteModule)
typ = self.make_target(spec='3rdparty/node:typ',
target_type=NodeRemoteModule,
version='0.6.3')
context = self.context(target_roots=[typ])
task = self.create_task(context)
task.execute()
node_paths = context.products.get_data(NodePaths)
node_path = node_paths.node_path(typ)
self.assertIsNotNone(node_path)
script = 'var typ = require("typ"); console.log("type of boolean is: " + typ.BOOLEAN)'
out = task.node_distribution.node_command(
args=['--eval', script]).check_output(cwd=node_path)
self.assertIn('type of boolean is: boolean', out)
def test_noop_applicable_targets_simple(self):
SourceRoot.register('src/go', GoBinary)
self.create_file(relpath='src/go/fred/foo.go',
contents=dedent("""
package main
import "fmt"
func main() {
fmt.Printf("Hello World!")
}
"""))
context = self.context(
target_roots=[self.make_target('src/go/fred', GoBinary)])
expected = context.targets()
task = self.create_task(context)
task.execute()
self.assertEqual(expected, context.targets())
def generate_targets(self, local_go_targets):
# TODO(John Sirois): support multiple source roots like GOPATH does?
# The GOPATH's 1st element is read-write, the rest are read-only; ie: their sources build to
# the 1st element's pkg/ and bin/ dirs.
all_rooted_types = set()
for types in SourceRoot.all_roots().values():
all_rooted_types.update(types)
def safe_get_source_roots(target_type):
return set(SourceRoot.roots(
target_type)) if target_type in all_rooted_types else set()
local_roots = safe_get_source_roots(GoBinary) | safe_get_source_roots(
GoLibrary)
if not local_roots:
raise self.NoLocalRootsError(
'Can only BUILD gen if a Go local sources source root is'
'defined.')
if len(local_roots) > 1:
raise self.InvalidLocalRootsError(
'Can only BUILD gen for a single Go local sources source '
'root, found:\n\t{}'.format('\n\t'.join(sorted(local_roots))))
local_root = local_roots.pop()
unrooted_locals = {
t
for t in local_go_targets if t.target_base != local_root
}
if unrooted_locals:
raise self.UnrootedLocalSourceError(
'Cannot BUILD gen until the following targets are '
'relocated to the build root at {}:\n\t{}'.format(
local_root, '\n\t'.join(
sorted(t.address.reference()
for t in unrooted_locals))))
remote_roots = set(safe_get_source_roots(GoRemoteLibrary))
if len(remote_roots) > 1:
raise self.InvalidRemoteRootsError(
'Can only BUILD gen for a single Go remote library source '
'root, found:\n\t{}'.format('\n\t'.join(sorted(remote_roots))))
remote_root = remote_roots.pop() if remote_roots else None
generator = GoTargetGenerator(
self.context.new_workunit,
self.go_dist,
self.context.build_graph,
local_root,
Fetchers.global_instance(),
generate_remotes=self.get_options().remote,
remote_root=remote_root)
with self.context.new_workunit('go.buildgen',
labels=[WorkUnitLabel.MULTITOOL]):
try:
return generator.generate(local_go_targets)
except generator.GenerationError as e:
raise self.GenerationError(e)
def test_transitive_download_remote_libs_undeclared_deps(self):
with temporary_dir() as src:
with temporary_dir() as zipdir:
SourceRoot.register(os.path.join(self.build_root, '3rdparty'),
GoRemoteLibrary)
dep_graph = {'r1': ['r2', 'r3'], 'r2': ['r4']}
self._init_dep_graph_files(src, zipdir, dep_graph)
r1 = self.target('3rdparty/r1')
context = self._create_fetch_context(zipdir)
go_fetch = self.create_task(context)
undeclared_deps = go_fetch._transitive_download_remote_libs(
set([r1]))
expected = defaultdict(set)
expected['r1'] = set([('r3', '3rdparty/r3')])
expected['r2'] = set([('r4', '3rdparty/r4')])
self.assertEqual(undeclared_deps, expected)
def test_existing_targets_wrong_type(self):
SourceRoot.register('src/go', GoBinary, GoLibrary)
self.create_file(relpath='src/go/fred/foo.go',
contents=dedent("""
package main
import "fmt"
func main() {
fmt.Printf("Hello World!")
}
"""))
context = self.context(
target_roots=[self.make_target('src/go/fred', GoLibrary)])
task = self.create_task(context)
with self.assertRaises(task.GenerationError) as exc:
task.execute()
self.assertEqual(GoTargetGenerator.WrongLocalSourceTargetTypeError,
type(exc.exception.cause))
def test_exported_thrift(self):
SourceRoot.register('src/thrift', PythonThriftLibrary)
self.create_file(relpath='src/thrift/exported/exported.thrift',
contents=dedent("""
namespace py pants.constants_only
const set<string> VALID_IDENTIFIERS = ["Hello", "World", "!"]
"""))
target = self.make_target(spec='src/thrift/exported',
target_type=PythonThriftLibrary,
sources=['exported.thrift'],
provides=PythonArtifact(name='test.exported',
version='0.0.0.'))
# TODO(John Sirois): Find a way to get easy access to pants option defaults.
self.set_options_for_scope(
'binaries',
baseurls=['https://dl.bintray.com/pantsbuild/bin/build-support'],
fetch_timeout_secs=30)
with self.run_execute(target) as created:
self.assertEqual([target], created)
def test_find_by_path(self):
# No source_root is registered yet
query = "tests/foo/bar:baz"
self.assertIsNone(SourceRoot.find_by_path(query),
msg="Query {query} Failed for tree: {dump}"
.format(query=query, dump=SourceRoot._dump()))
SourceRoot.register("tests/foo", TestTarget)
self.assertEquals("tests/foo", SourceRoot.find_by_path(query),
msg="Query {query} Failed for tree: {dump}"
.format(query=query, dump=SourceRoot._dump()))
self.assertIsNone(SourceRoot.find_by_path("tests/bar/foobar:qux"),
msg="Failed for tree: {dump}"
.format(dump=SourceRoot._dump()))
def test_compiler_args_proto_paths(self):
self._create_fake_wire_tool(version='2.0.0')
SourceRoot.register('wire-src')
SourceRoot.register('wire-other-src')
parent_target = self.make_target('wire-other-src:parent-target',
JavaWireLibrary,
sources=['bar.proto'])
simple_wire_target = self.make_target('wire-src:simple-wire-target',
JavaWireLibrary,
sources=['foo.proto'],
dependencies=[parent_target])
context = self.context(
target_roots=[parent_target, simple_wire_target])
task = self.create_task(context)
self.assertEquals([
'--java_out={}/{}/wire-src.simple-wire-target'.format(
self.build_root, self.EXPECTED_TASK_PATH),
'--proto_path={}/wire-src'.format(
self.build_root), '--proto_path={}/wire-other-src'.format(
self.build_root), 'foo.proto'
], task.format_args_for_target(simple_wire_target))
def test_collapse_source_root(self):
source_set_list = []
self.assertEquals([], Project._collapse_by_source_root(source_set_list))
SourceRoot.register("src/java", JavaLibrary)
SourceRoot.register("tests/java", JavaTests)
source_sets = [
SourceSet("/repo-root", "src/java", "org/pantsbuild/app", False),
SourceSet("/repo-root", "tests/java", "org/pantsbuild/app", True),
SourceSet("/repo-root", "some/other", "path", False),
]
results = Project._collapse_by_source_root(source_sets)
self.assertEquals(SourceSet("/repo-root", "src/java", "", False), results[0])
self.assertFalse(results[0].is_test)
self.assertEquals(SourceSet("/repo-root", "tests/java", "", True), results[1])
self.assertTrue(results[1].is_test)
# If there is no registered source root, the SourceSet should be returned unmodified
self.assertEquals(source_sets[2], results[2])
self.assertFalse(results[2].is_test)
def test_transitive_download_remote_libs_undeclared_deps(self):
with temporary_dir() as src:
with temporary_dir() as zipdir:
SourceRoot.register('3rdparty', GoRemoteLibrary)
dep_graph = {'r1': ['r2', 'r3'], 'r2': ['r4']}
self._init_dep_graph_files(src, zipdir, dep_graph)
r1 = self.target('3rdparty/localzip/r1')
r2 = self.target('3rdparty/localzip/r2')
context = self._create_fetch_context(zipdir)
go_fetch = self.create_task(context)
undeclared_deps = go_fetch._transitive_download_remote_libs(
{r1})
expected = defaultdict(set)
expected[r1] = {('localzip/r3',
self.address('3rdparty/localzip/r3'))}
expected[r2] = {('localzip/r4',
self.address('3rdparty/localzip/r4'))}
self.assertEqual(undeclared_deps, expected)
def setUp(self):
super(PythonTaskTest, self).setUp()
SourceRoot.register('3rdparty', PythonRequirementLibrary)
SourceRoot.register('src', PythonBinary, PythonLibrary)
self.requests = self.create_python_requirement_library('3rdparty/requests', 'requests',
requirements=['requests==2.6.0'])
self.six = self.create_python_requirement_library('3rdparty/six', 'six',
requirements=['six==1.9.0'])
self.library = self.create_python_library('src/lib', 'lib', {'lib.py': dedent("""
import six
def go():
six.print_('go', 'go', 'go!', sep='')
""")}, dependencies=['//3rdparty/six'])
self.binary = self.create_python_binary('src/bin', 'bin', 'lib.lib:go',
dependencies=['//src/lib'])
def test_symlink_local_src(self):
with pushd(self.build_root):
SourceRoot.register('src/main/go')
spec = 'src/main/go/foo/bar/mylib'
sources = ['x.go', 'y.go', 'z.go', 'z.c', 'z.h', 'w.png']
for src in sources:
self.create_file(os.path.join(spec, src))
go_lib = self.make_target(spec=spec, target_type=GoLibrary)
ws_task = self.create_task(self.context())
gopath = ws_task.get_gopath(go_lib)
def assert_is_linked(src):
link = os.path.join(gopath, 'src/foo/bar/mylib', src)
self.assertTrue(os.path.islink(link))
self.assertEqual(os.readlink(link),
os.path.join(self.build_root, spec, src))
ws_task._symlink_local_src(gopath, go_lib, set())
for src in sources:
assert_is_linked(src)
# Sleep so that first round of linking has 1.5 second earlier mtime than future links.
time.sleep(1.5)
# Add source file and re-make library.
self.create_file(os.path.join(spec, 'w.go'))
self.reset_build_graph()
go_lib = self.make_target(spec=spec, target_type=GoLibrary)
ws_task._symlink_local_src(gopath, go_lib, set())
for src in chain(sources, ['w.go']):
assert_is_linked(src)
mtime = lambda src: os.lstat(
os.path.join(gopath, 'src/foo/bar/mylib', src)).st_mtime
for src in sources:
# Ensure none of the old links were overwritten.
self.assertLessEqual(mtime(src), mtime('w.go') - 1)
def setUp(self):
super(JarCreateExecuteTest, self).setUp()
def test_path(path):
return os.path.join(self.__class__.__name__, path)
def get_source_root_fs_path(path):
return os.path.realpath(
os.path.join(self.build_root, test_path(path)))
SourceRoot.register(get_source_root_fs_path('src/resources'),
Resources)
SourceRoot.register(get_source_root_fs_path('src/java'), JavaLibrary,
JvmBinary)
SourceRoot.register(get_source_root_fs_path('src/scala'), ScalaLibrary)
SourceRoot.register(get_source_root_fs_path('src/thrift'),
JavaThriftLibrary)
# This is hacky: Creating a scala_library below requires Subsystem initialization,
# which we must force by this call to the superclass context() (note that can't call our own
# context() because it expects the scala_library to already exist). The test methods themselves
# then call self.context() again to get hold of a context, which will superfluously (but
# harmlessly) re-initialize Subsystem.
# TODO: Fix this hack by separating option setup from context setup in the test sequence.
super(JarCreateExecuteTest, self).context()
self.res = self.create_resources(
test_path('src/resources/com/twitter'), 'spam', 'r.txt')
self.jl = self.java_library(
test_path('src/java/com/twitter'),
'foo', ['a.java'],
resources=test_path('src/resources/com/twitter:spam'))
self.sl = self.scala_library(test_path('src/scala/com/twitter'), 'bar',
['c.scala'])
self.jtl = self.java_thrift_library(
test_path('src/thrift/com/twitter'), 'baz', 'd.thrift')
self.java_lib_foo = self.java_library(
test_path('src/java/com/twitter/foo'), 'java_foo',
['java_foo.java'])
self.scala_lib = self.scala_library(
test_path('src/scala/com/twitter/foo'),
'scala_foo', ['scala_foo.scala'],
java_sources=[test_path('src/java/com/twitter/foo:java_foo')])
self.binary = self.jvm_binary(
test_path('src/java/com/twitter/baz'),
'baz',
source='b.java',
resources=test_path('src/resources/com/twitter:spam'))
self.empty_sl = self.scala_library(test_path('src/scala/com/foo'),
'foo', ['dupe.scala'])
def test_multiple_packages(self):
SourceRoot.register('3rdparty/go', GoRemoteLibrary)
self.add_to_build_file('3rdparty/go/github.com/foo/bar', dedent("""
go_remote_libraries(
rev='v42',
packages=[
'',
'baz',
'baz/bip',
'bee/bop'
])
"""))
default = self.target('3rdparty/go/github.com/foo/bar')
self.assertIsInstance(default, GoRemoteLibrary)
self.assertEqual('v42', default.rev)
self.assertEqual('github.com/foo/bar', default.import_path)
self.assertEqual('github.com/foo/bar', default.remote_root)
self.assertEqual('', default.pkg)
baz = self.target('3rdparty/go/github.com/foo/bar:baz')
self.assertIsInstance(baz, GoRemoteLibrary)
self.assertEqual('v42', baz.rev)
self.assertEqual('github.com/foo/bar/baz', baz.import_path)
self.assertEqual('github.com/foo/bar', baz.remote_root)
self.assertEqual('baz', baz.pkg)
baz_bip = self.target('3rdparty/go/github.com/foo/bar:baz/bip')
self.assertIsInstance(baz_bip, GoRemoteLibrary)
self.assertEqual('v42', baz_bip.rev)
self.assertEqual('github.com/foo/bar/baz/bip', baz_bip.import_path)
self.assertEqual('github.com/foo/bar', baz_bip.remote_root)
self.assertEqual('baz/bip', baz_bip.pkg)
bee_bop = self.target('3rdparty/go/github.com/foo/bar:bee/bop')
self.assertIsInstance(bee_bop, GoRemoteLibrary)
self.assertEqual('v42', bee_bop.rev)
self.assertEqual('github.com/foo/bar/bee/bop', bee_bop.import_path)
self.assertEqual('github.com/foo/bar', bee_bop.remote_root)
self.assertEqual('bee/bop', bee_bop.pkg)
def test_thrift_issues_1858(self):
# Confirm a synthetic target for our python_thrift_library from some upstream task does not
# trample the PythonChroot/PythonThriftBuilder generated code.
# In https://github.com/pantsbuild/pants/issues/1858 the ApacheThriftGen task in the 'gen'
# phase upstream of the 'binary' goal was injecting a synthetic python_library target owning
# thrift generated code _and_ that code was a subset of all the code generated by thrift; ie:
# there was a synthetic python_library being added directly to the chroot missing some thrift
# codegened '.py' files, leading to import of those files (and errors) instead of the
# PythonChroot/PythonThriftBuilder generated files (packaged as deps in the PythonChroot).
with self.do_test_thrift() as (binary, thrift_target):
SourceRoot.register('.synthetic', PythonLibrary)
self.create_file(relpath='.synthetic/test/__init__.py')
self.create_file(relpath='.synthetic/test/constants.py',
contents=dedent("""
VALID_IDENTIFIERS = ['generated', 'by', 'upstream', 'and', 'different!']
"""))
synthetic_pythrift_codegen_target = self.make_target(
spec='.synthetic/test:constants',
target_type=PythonLibrary,
sources=['__init__.py', 'constants.py'],
derived_from=thrift_target)
binary.inject_dependency(synthetic_pythrift_codegen_target.address)
def test_find(self):
# When no source_root is registered, it should just return the path from the address
self.assertEqual("tests/foo/bar", SourceRoot.find(TestTarget("//tests/foo/bar:baz")))
SourceRoot.register("tests/foo", TestTarget)
# After the source root is registered, you should get the source root
self.assertEquals("tests/foo", SourceRoot.find(TestTarget("//tests/foo/bar:baz")))
with self.assertRaises(TargetDefinitionException):
SourceRoot.find(NotTestTarget("//tests/foo/foobar:qux"))
def test_register_none(self):
self._assert_source_root_empty()
SourceRoot.register("tests", )
self.assertEquals({"tests": OrderedSet()}, SourceRoot.all_roots())
self.assertEquals(OrderedSet(), SourceRoot.types("tests"))
self.assertEquals("tests", SourceRoot.find(TestTarget("//tests/foo/bar:baz")))
self.assertEquals("tests", SourceRoot.find_by_path("tests/foo/bar"))
def do_test_thrift(self):
SourceRoot.register('src/thrift', PythonThriftLibrary)
self.create_file(relpath='src/thrift/test/const.thrift',
contents=dedent("""
namespace py test
const list<string> VALID_IDENTIFIERS = ["Hello", "World!"]
"""))
thrift_target = self.make_target(spec='src/thrift/test',
target_type=PythonThriftLibrary,
sources=['const.thrift'])
SourceRoot.register('src/python', PythonBinary)
self.create_file(relpath='src/python/test/main.py',
contents=dedent("""
from test.constants import VALID_IDENTIFIERS
def say_hello():
print(' '.join(VALID_IDENTIFIERS))
"""))
binary = self.make_target(spec='src/python/test',
target_type=PythonBinary,
source='main.py',
dependencies=[thrift_target])
yield binary, thrift_target
with self.dumped_chroot([binary]) as (pex_builder, python_chroot):
pex_builder.set_entry_point('test.main:say_hello')
pex_builder.freeze()
pex = python_chroot.pex()
process = pex.run(blocking=False, stdout=subprocess.PIPE)
stdout, _ = process.communicate()
self.assertEqual(0, process.returncode)
self.assertEqual('Hello World!', stdout.strip())
def test_compiler_args_all(self):
self._create_fake_wire_tool(version='2.0.0')
SourceRoot.register('wire-src')
kitchen_sink = self.make_target(
'wire-src:kitchen-sink',
JavaWireLibrary,
sources=['foo.proto', 'bar.proto', 'baz.proto'],
registry_class='org.pantsbuild.Registry',
service_factory='org.pantsbuild.DummyServiceFactory',
no_options=True,
roots=['root1', 'root2', 'root3'],
enum_options=['enum1', 'enum2', 'enum3'],
)
task = self.create_task(self.context(target_roots=[kitchen_sink]))
self.assertEquals([
'--java_out={}/{}/wire-src.kitchen-sink'.format(
self.build_root, self.EXPECTED_TASK_PATH), '--no_options',
'--service_factory=org.pantsbuild.DummyServiceFactory',
'--registry_class=org.pantsbuild.Registry',
'--roots=root1,root2,root3', '--enum_options=enum1,enum2,enum3',
'--proto_path={}/wire-src'.format(
self.build_root), 'foo.proto', 'bar.proto', 'baz.proto'
], task.format_args_for_target(kitchen_sink))
