class UnionProductsTest(BaseTest):
def setUp(self):
super(UnionProductsTest, self).setUp()
self.products = UnionProducts()
def test_get_for_target(self):
c = self.make_target('c')
b = self.make_target('b', dependencies=[c])
a = self.make_target('a', dependencies=[b, c])
self.products.add_for_target(a, [1])
self.products.add_for_target(b, [2])
self.products.add_for_target(c, [3])
self.assertEquals(self.products.get_for_target(a), OrderedSet([1, 2, 3]))
self.assertEquals(self.products.get_for_target(b), OrderedSet([2, 3]))
self.assertEquals(self.products.get_for_target(c), OrderedSet([3]))
def test_empty_products(self):
c = self.make_target('c')
self.assertFalse(self.products.get_for_target(c))
def test_non_empty_products(self):
c = self.make_target('c')
self.products.add_for_target(c, [3])
self.assertTrue(self.products.get_for_target(c))
def test_single_classpath_element_no_excludes(self):
a = self.make_target('a', JvmTarget)
classpath_product = UnionProducts()
path = os.path.join(self.build_root, 'jar/path')
classpath_product.add_for_target(a, [('default', path)])
classpath = ClasspathUtil.compute_classpath([a], classpath_product, [], ['default'])
self.assertEqual([path], classpath)
def test_single_classpath_element_no_excludes(self):
a = self.make_target('a', JvmTarget)
classpath_product = UnionProducts()
path = os.path.join(self.build_root, 'jar/path')
classpath_product.add_for_target(a, [('default', path)])
classpath = ClasspathUtil.compute_classpath([a], classpath_product, [],
['default'])
self.assertEqual([path], classpath)
def test_relies_on_product_to_validate_paths_outside_buildroot(self):
a = self.make_target('a', JvmTarget)
classpath_product = UnionProducts()
classpath_product.add_for_target(a, [('default', '/dev/null')])
classpath = ClasspathUtil.compute_classpath([a], classpath_product, [], ['default'])
self.assertEqual(['/dev/null'], classpath)
def test_excluded_classpath_element(self):
a = self.make_target('a', JvmTarget, excludes=[Exclude('com.example', 'lib')])
classpath_product = UnionProducts()
example_jar_path = os.path.join(self.build_root, 'ivy/jars/com.example/lib/123.4.jar')
classpath_product.add_for_target(a, [('default', example_jar_path)])
classpath = ClasspathUtil.compute_classpath([a], classpath_product, [], ['default'])
self.assertEqual([], classpath)
def test_relies_on_product_to_validate_paths_outside_buildroot(self):
a = self.make_target('a', JvmTarget)
classpath_product = UnionProducts()
classpath_product.add_for_target(a, [('default', '/dev/null')])
classpath = ClasspathUtil.compute_classpath_for_target(
a, classpath_product, extra_classpath_tuples=[], confs=['default'])
self.assertEqual(['/dev/null'], classpath)
def test_path_with_differing_conf_ignored(self):
a = self.make_target('a', JvmTarget)
classpath_product = UnionProducts()
path = os.path.join(self.build_root, 'jar/path')
classpath_product.add_for_target(a, [('default', path)])
classpath = ClasspathUtil.compute_classpath([a], classpath_product, [], ['not-default'])
self.assertEqual([], classpath)
def test_parent_excludes_ignored_for_resolving_child_target(self):
b = self.make_target('b', JvmTarget)
self.make_target('a', JvmTarget, dependencies=[b], excludes=[Exclude('com.example', 'lib')])
classpath_product = UnionProducts()
example_jar_path = os.path.join(self.build_root, 'ivy/jars/com.example/lib/123.4.jar')
classpath_product.add_for_target(b, [('default', example_jar_path)])
classpath = ClasspathUtil.compute_classpath([b], classpath_product, [], ['default'])
self.assertEqual([example_jar_path], classpath)
def test_fails_on_paths_outside_buildroot(self):
a = self.make_target('a', JvmTarget)
classpath_product = UnionProducts()
classpath_product.add_for_target(a, [('default', '/dev/null')])
with self.assertRaises(TaskError) as cm:
ClasspathUtil.compute_classpath([a], classpath_product, [], ['default'])
self.assertEqual(
str('Classpath entry /dev/null for target a:a is located outside the buildroot.'),
str(cm.exception))
def test_path_with_overlapped_conf_added(self):
a = self.make_target('a', JvmTarget)
classpath_product = UnionProducts()
path = os.path.join(self.build_root, 'jar/path')
classpath_product.add_for_target(a, [('default', path)])
classpath = ClasspathUtil.compute_classpath([a], classpath_product, [],
['not-default', 'default'])
self.assertEqual([path], classpath)
def test_fails_on_paths_outside_buildroot(self):
a = self.make_target('a', JvmTarget)
classpath_product = UnionProducts()
classpath_product.add_for_target(a, [('default', '/dev/null')])
with self.assertRaises(TaskError) as cm:
ClasspathUtil.compute_classpath([a], classpath_product, [],
['default'])
self.assertEqual(
str('Classpath entry /dev/null for target a:a is located outside the buildroot.'
), str(cm.exception))
def test_extra_path_added(self):
a = self.make_target('a', JvmTarget)
classpath_product = UnionProducts()
path = os.path.join(self.build_root, 'jar/path')
classpath_product.add_for_target(a, [('default', path)])
extra_path = 'new-path'
classpath = ClasspathUtil.compute_classpath([a], classpath_product,
[('default', extra_path)], ['default'])
self.assertEqual([path, extra_path], classpath)
def test_exclude_leaves_other_jars_unaffected(self):
b = self.make_target('b', JvmTarget, excludes=[Exclude('com.example', 'lib')])
a = self.make_target('a', JvmTarget, dependencies=[b])
classpath_product = UnionProducts()
com_example_jar_path = os.path.join(self.build_root, 'ivy/jars/com.example/lib/123.4.jar')
org_example_jar_path = os.path.join(self.build_root, 'ivy/jars/org.example/lib/123.4.jar')
classpath_product.add_for_target(a, [('default', com_example_jar_path),
('default', org_example_jar_path)])
classpath = ClasspathUtil.compute_classpath([a], classpath_product, [], ['default'])
self.assertEqual([org_example_jar_path], classpath)
def test_excluded_classpath_element(self):
a = self.make_target('a',
JvmTarget,
excludes=[Exclude('com.example', 'lib')])
classpath_product = UnionProducts()
example_jar_path = os.path.join(self.build_root,
'ivy/jars/com.example/lib/123.4.jar')
classpath_product.add_for_target(a, [('default', example_jar_path)])
classpath = ClasspathUtil.compute_classpath([a], classpath_product, [],
['default'])
self.assertEqual([], classpath)
def test_extra_path_added(self):
a = self.make_target('a', JvmTarget)
classpath_product = UnionProducts()
path = os.path.join(self.build_root, 'jar/path')
classpath_product.add_for_target(a, [('default', path)])
extra_path = 'new-path'
classpath = ClasspathUtil.compute_classpath([a], classpath_product,
[('default', extra_path)],
['default'])
self.assertEqual([path, extra_path], classpath)
def test_path_with_overlapped_conf_added(self):
a = self.make_target('a', JvmTarget)
classpath_product = UnionProducts()
path = os.path.join(self.build_root, 'jar/path')
classpath_product.add_for_target(a, [('default', path)])
classpath = ClasspathUtil.compute_classpath([a],
classpath_product,
extra_classpath_tuples=[],
confs=['not-default', 'default'])
self.assertEqual([path], classpath)
def test_path_with_differing_conf_ignored(self):
a = self.make_target('a', JvmTarget)
classpath_product = UnionProducts()
path = os.path.join(self.build_root, 'jar/path')
classpath_product.add_for_target(a, [('default', path)])
classpath = ClasspathUtil.compute_classpath([a],
classpath_product,
extra_classpath_tuples=[],
confs=['not-default'])
self.assertEqual([], classpath)
def test_parent_excludes_ignored_for_resolving_child_target(self):
b = self.make_target('b', JvmTarget)
self.make_target('a',
JvmTarget,
dependencies=[b],
excludes=[Exclude('com.example', 'lib')])
classpath_product = UnionProducts()
example_jar_path = os.path.join(self.build_root,
'ivy/jars/com.example/lib/123.4.jar')
classpath_product.add_for_target(b, [('default', example_jar_path)])
classpath = ClasspathUtil.compute_classpath([b], classpath_product, [],
['default'])
self.assertEqual([example_jar_path], classpath)
def execute(self):
"""Resolves the specified confs for the configured targets and returns an iterator over
tuples of (conf, jar path).
"""
executor = self.create_java_executor()
targets = self.context.targets()
self.context.products.safe_create_data('ivy_cache_dir',
lambda: self._cachedir)
compile_classpath = self.context.products.get_data(
'compile_classpath', lambda: UnionProducts())
# After running ivy, we parse the resulting report, and record the dependencies for
# all relevant targets (ie: those that have direct dependencies).
_, relevant_targets = self.ivy_resolve(
targets,
executor=executor,
workunit_name='ivy-resolve',
confs=self.confs,
custom_args=self._args,
)
# Record the ordered subset of jars that each jar_library/leaf depends on using
# stable symlinks within the working copy.
ivy_jar_products = self._generate_ivy_jar_products(relevant_targets)
symlink_map = self.context.products.get_data('ivy_resolve_symlink_map')
for conf in self.confs:
ivy_jar_memo = {}
ivy_info_list = ivy_jar_products[conf]
if not ivy_info_list:
continue
# TODO: refactor ivy_jar_products to remove list
assert len(ivy_info_list) == 1, (
'The values in ivy_jar_products should always be length 1,'
' since we no longer have exclusives groups.')
ivy_info = ivy_info_list[0]
for target in relevant_targets:
if not isinstance(target, JarLibrary):
continue
# Add the artifacts from each dependency module.
artifact_paths = []
for artifact in ivy_info.get_artifacts_for_jar_library(
target, memo=ivy_jar_memo):
artifact_paths.append(symlink_map[artifact.path])
compile_classpath.add_for_target(target,
[(conf, entry)
for entry in artifact_paths])
if self._report:
self._generate_ivy_report(relevant_targets)
if self.context.products.is_required_data('ivy_jar_products'):
self._populate_ivy_jar_products(relevant_targets, ivy_jar_products)
create_jardeps_for = self.context.products.isrequired(
'jar_dependencies')
if create_jardeps_for:
genmap = self.context.products.get('jar_dependencies')
for target in filter(create_jardeps_for, targets):
self.mapjars(genmap, target, executor=executor)
def test_exclude_leaves_other_jars_unaffected(self):
b = self.make_target('b',
JvmTarget,
excludes=[Exclude('com.example', 'lib')])
a = self.make_target('a', JvmTarget, dependencies=[b])
classpath_product = UnionProducts()
com_example_jar_path = os.path.join(
self.build_root, 'ivy/jars/com.example/lib/123.4.jar')
org_example_jar_path = os.path.join(
self.build_root, 'ivy/jars/org.example/lib/123.4.jar')
classpath_product.add_for_target(a,
[('default', com_example_jar_path),
('default', org_example_jar_path)])
classpath = ClasspathUtil.compute_classpath([a], classpath_product, [],
['default'])
self.assertEqual([org_example_jar_path], classpath)
def _collect_external_libs(self, vts):
"""
Sets the relevant properties of the task product (`NativeExternalLibraryFiles`) object.
"""
product = UnionProducts()
for vt in vts:
lib_dir = os.path.join(vt.results_dir, 'lib')
include_dir = os.path.join(vt.results_dir, 'include')
lib_names = []
if os.path.isdir(lib_dir):
for filename in os.listdir(lib_dir):
lib_name = self._parse_lib_name_from_library_filename(
filename)
if lib_name:
lib_names.append(lib_name)
nelf = NativeExternalLibraryFiles(include_dir=include_dir,
lib_dir=lib_dir,
lib_names=tuple(lib_names))
product.add_for_target(vt.target, [nelf])
return product
def populate_compile_classpath(self, context, classpath=None):
"""
Helps actual test cases to populate the 'compile_classpath' products data mapping
in the context, which holds the classpath value for targets.
:param context: The execution context where the products data mapping lives.
:param classpath: a list of classpath strings. If not specified, ['none'] will be used.
"""
compile_classpaths = context.products.get_data('compile_classpath',
lambda: UnionProducts())
compile_classpaths.add_for_targets(
context.targets(),
[('default', entry) for entry in classpath or ['none']])
class ClasspathProducts(object):
def __init__(self):
self._classpaths = UnionProducts()
self._exclude_patterns = UnionProducts()
self._buildroot = get_buildroot()
def add_for_targets(self, targets, classpath_elements):
"""Adds classpath elements to the products of all the provided targets."""
for target in targets:
self.add_for_target(target, classpath_elements)
def add_for_target(self, target, classpath_elements):
"""Adds classpath elements to the products of the provided target."""
self._validate_classpath_tuples(classpath_elements, target)
self._classpaths.add_for_target(target, classpath_elements)
def add_excludes_for_targets(self, targets):
"""Add excludes from the provided targets. Does not look up transitive excludes."""
for target in targets:
self._add_excludes_for_target(target)
def remove_for_target(self, target, classpath_elements):
"""Removes the given entries for the target"""
self._classpaths.remove_for_target(target, classpath_elements)
def get_for_target(self, target):
"""Gets the transitive classpath products for the given target, in order, respecting target
excludes."""
return self.get_for_targets([target])
def get_for_targets(self, targets):
"""Gets the transitive classpath products for the given targets, in order, respecting target
excludes."""
classpath_tuples = self._classpaths.get_for_targets(targets)
filtered_classpath_tuples = self._filter_by_excludes(
classpath_tuples, targets)
return filtered_classpath_tuples
def _filter_by_excludes(self, classpath_tuples, root_targets):
exclude_patterns = self._exclude_patterns.get_for_targets(root_targets)
filtered_classpath_tuples = filter(
_not_excluded_filter(exclude_patterns), classpath_tuples)
return filtered_classpath_tuples
def _add_excludes_for_target(self, target):
# TODO(nhoward): replace specific ivy based exclude filterings in the jar object refactor
# creates strings from excludes that will match classpath entries generated by ivy
# eg exclude(org='org.example', name='lib') => 'jars/org.example/lib'
# exclude(org='org.example') => 'jars/org.example/'
# The empty string was added to the end of the os.path.join list, so that the exclude pattern
# always ends with a path separator. It's a short term fix so we don't match the following
# 'jars/com.twitter.common/text' in '.../jars/com.twitter.common/text-lang-model/jars/text...'
if target.is_exported:
self._exclude_patterns.add_for_target(target, [
os.path.join('jars', target.provides.org, target.provides.name,
'')
])
if isinstance(target, JvmTarget) and target.excludes:
self._exclude_patterns.add_for_target(target, [
os.path.join('jars', e.org, e.name or '', '')
for e in target.excludes
])
def _validate_classpath_tuples(self, classpath, target):
"""Validates that all files are located within the working copy, to simplify relativization."""
for classpath_tuple in classpath:
self._validate_path_in_buildroot(classpath_tuple, target)
def _validate_path_in_buildroot(self, classpath_tuple, target):
conf, path = classpath_tuple
if os.path.relpath(path, self._buildroot).startswith(os.pardir):
raise TaskError(
'Classpath entry {} for target {} is located outside the buildroot.'
.format(path, target.address.spec))
def __init__(self, pants_workdir, classpaths=None, excludes=None): self._classpaths = classpaths or UnionProducts() self._excludes = excludes or UnionProducts() self._pants_workdir = pants_workdir
def setUp(self): super(UnionProductsTest, self).setUp() self.products = UnionProducts()
class UnionProductsTest(BaseTest):
def setUp(self):
super(UnionProductsTest, self).setUp()
self.products = UnionProducts()
def test_get_for_target(self):
c = self.make_target('c')
b = self.make_target('b', dependencies=[c])
a = self.make_target('a', dependencies=[b, c])
self.products.add_for_target(a, [1])
self.products.add_for_target(b, [2])
self.products.add_for_target(c, [3])
self.assertEquals(self.products.get_for_target(a), OrderedSet([1, 2, 3]))
self.assertEquals(self.products.get_for_target(b), OrderedSet([2, 3]))
self.assertEquals(self.products.get_for_target(c), OrderedSet([3]))
self.assertEquals(self.products.get_for_target(a, transitive=False), OrderedSet([1]))
self.assertEquals(self.products.get_for_target(b, transitive=False), OrderedSet([2]))
self.assertEquals(self.products.get_for_target(c, transitive=False), OrderedSet([3]))
def test_copy(self):
c = self.make_target('c')
b = self.make_target('b', dependencies=[c])
a = self.make_target('a', dependencies=[b, c])
self.products.add_for_target(a, [1])
self.products.add_for_target(b, [2])
copied = self.products.copy()
self.assertEquals(self.products.get_for_target(a), OrderedSet([1, 2]))
self.assertEquals(self.products.get_for_target(b), OrderedSet([2]))
self.assertEquals(copied.get_for_target(a), OrderedSet([1, 2]))
self.assertEquals(copied.get_for_target(b), OrderedSet([2]))
copied.add_for_target(c, [3])
self.assertEquals(self.products.get_for_target(a), OrderedSet([1, 2]))
self.assertEquals(self.products.get_for_target(b), OrderedSet([2]))
self.assertEquals(self.products.get_for_target(c), OrderedSet())
self.assertEquals(copied.get_for_target(a), OrderedSet([1, 2, 3]))
self.assertEquals(copied.get_for_target(b), OrderedSet([2, 3]))
self.assertEquals(copied.get_for_target(c), OrderedSet([3]))
def test_remove_for_target(self):
c = self.make_target('c')
b = self.make_target('b', dependencies=[c])
a = self.make_target('a', dependencies=[b, c])
self.products.add_for_target(a, [1])
self.products.add_for_target(b, [2])
self.products.add_for_target(c, [3])
self.products.remove_for_target(a, [1])
self.assertEquals(self.products.get_for_target(a), OrderedSet([2, 3]))
self.assertEquals(self.products.get_for_target(b), OrderedSet([2, 3]))
self.assertEquals(self.products.get_for_target(c), OrderedSet([3]))
def test_empty_products(self):
c = self.make_target('c')
self.assertFalse(self.products.get_for_target(c))
def test_non_empty_products(self):
c = self.make_target('c')
self.products.add_for_target(c, [3])
self.assertTrue(self.products.get_for_target(c))
def test_target_for_product_existing_product(self):
c = self.make_target('c')
self.products.add_for_target(c, [3])
found_target = self.products.target_for_product(3)
self.assertEqual(c, found_target)
def test_target_for_product_nonexistent_product(self):
c = self.make_target('c')
self.products.add_for_target(c, [3])
found_target = self.products.target_for_product(1000)
self.assertIsNone(found_target)
def setUp(self): super(UnionProductsTest, self).setUp() self.products = UnionProducts()
class UnionProductsTest(BaseTest):
def setUp(self):
super(UnionProductsTest, self).setUp()
self.products = UnionProducts()
def test_get_for_target(self):
c = self.make_target('c')
b = self.make_target('b', dependencies=[c])
a = self.make_target('a', dependencies=[b, c])
self.products.add_for_target(a, [1])
self.products.add_for_target(b, [2])
self.products.add_for_target(c, [3])
self.assertEquals(self.products.get_for_targets(a.closure(bfs=True)), OrderedSet([1, 2, 3]))
self.assertEquals(self.products.get_for_targets(b.closure(bfs=True)), OrderedSet([2, 3]))
self.assertEquals(self.products.get_for_targets(c.closure(bfs=True)), OrderedSet([3]))
self.assertEquals(self.products.get_for_target(a), OrderedSet([1]))
self.assertEquals(self.products.get_for_target(b), OrderedSet([2]))
self.assertEquals(self.products.get_for_target(c), OrderedSet([3]))
def test_get_product_target_mappings_for_targets(self):
b = self.make_target('b')
a = self.make_target('a', dependencies=[b])
self.products.add_for_target(a, [1, 3])
self.products.add_for_target(b, [2, 3])
self.assertEquals(self.products.get_for_targets(a.closure(bfs=True)), OrderedSet([1, 3, 2]))
self.assertEquals(self.products.get_for_targets(b.closure(bfs=True)), OrderedSet([2, 3]))
self.assertEquals(self.products.get_product_target_mappings_for_targets(a.closure(bfs=True)),
[(1, a), (3, a), (2, b), (3, b)])
def test_copy(self):
c = self.make_target('c')
b = self.make_target('b', dependencies=[c])
a = self.make_target('a', dependencies=[b, c])
self.products.add_for_target(a, [1])
self.products.add_for_target(b, [2])
copied = self.products.copy()
self.assertEquals(self.products.get_for_targets(a.closure(bfs=True)), OrderedSet([1, 2]))
self.assertEquals(self.products.get_for_targets(b.closure(bfs=True)), OrderedSet([2]))
self.assertEquals(copied.get_for_targets(a.closure(bfs=True)), OrderedSet([1, 2]))
self.assertEquals(copied.get_for_targets(b.closure(bfs=True)), OrderedSet([2]))
copied.add_for_target(c, [3])
self.assertEquals(self.products.get_for_targets(a.closure(bfs=True)), OrderedSet([1, 2]))
self.assertEquals(self.products.get_for_targets(b.closure(bfs=True)), OrderedSet([2]))
self.assertEquals(self.products.get_for_targets(c.closure(bfs=True)), OrderedSet())
self.assertEquals(copied.get_for_targets(a.closure(bfs=True)), OrderedSet([1, 2, 3]))
self.assertEquals(copied.get_for_targets(b.closure(bfs=True)), OrderedSet([2, 3]))
self.assertEquals(copied.get_for_targets(c.closure(bfs=True)), OrderedSet([3]))
def test_remove_for_target(self):
c = self.make_target('c')
b = self.make_target('b', dependencies=[c])
a = self.make_target('a', dependencies=[b, c])
self.products.add_for_target(a, [1])
self.products.add_for_target(b, [2])
self.products.add_for_target(c, [3])
self.products.remove_for_target(a, [1])
self.assertEquals(self.products.get_for_targets(a.closure(bfs=True)), OrderedSet([2, 3]))
self.assertEquals(self.products.get_for_targets(b.closure(bfs=True)), OrderedSet([2, 3]))
self.assertEquals(self.products.get_for_targets(c.closure(bfs=True)), OrderedSet([3]))
def test_empty_products(self):
c = self.make_target('c')
self.assertFalse(self.products.get_for_target(c))
def test_non_empty_products(self):
c = self.make_target('c')
self.products.add_for_target(c, [3])
self.assertTrue(self.products.get_for_target(c))
def test_target_for_product_existing_product(self):
c = self.make_target('c')
self.products.add_for_target(c, [3])
found_target = self.products.target_for_product(3)
self.assertEqual(c, found_target)
def test_target_for_product_nonexistent_product(self):
c = self.make_target('c')
self.products.add_for_target(c, [3])
found_target = self.products.target_for_product(1000)
self.assertIsNone(found_target)
def setUp(self):
super().setUp()
self.products = UnionProducts()
class UnionProductsTest(TestBase):
def setUp(self):
super(UnionProductsTest, self).setUp()
self.products = UnionProducts()
def test_get_for_target(self):
c = self.make_target('c')
b = self.make_target('b', dependencies=[c])
a = self.make_target('a', dependencies=[b, c])
self.products.add_for_target(a, [1])
self.products.add_for_target(b, [2])
self.products.add_for_target(c, [3])
self.assertEqual(self.products.get_for_targets(a.closure(bfs=True)), OrderedSet([1, 2, 3]))
self.assertEqual(self.products.get_for_targets(b.closure(bfs=True)), OrderedSet([2, 3]))
self.assertEqual(self.products.get_for_targets(c.closure(bfs=True)), OrderedSet([3]))
self.assertEqual(self.products.get_for_target(a), OrderedSet([1]))
self.assertEqual(self.products.get_for_target(b), OrderedSet([2]))
self.assertEqual(self.products.get_for_target(c), OrderedSet([3]))
def test_get_product_target_mappings_for_targets(self):
b = self.make_target('b')
a = self.make_target('a', dependencies=[b])
self.products.add_for_target(a, [1, 3])
self.products.add_for_target(b, [2, 3])
self.assertEqual(self.products.get_for_targets(a.closure(bfs=True)), OrderedSet([1, 3, 2]))
self.assertEqual(self.products.get_for_targets(b.closure(bfs=True)), OrderedSet([2, 3]))
self.assertEqual(self.products.get_product_target_mappings_for_targets(a.closure(bfs=True)),
[(1, a), (3, a), (2, b), (3, b)])
def test_copy(self):
c = self.make_target('c')
b = self.make_target('b', dependencies=[c])
a = self.make_target('a', dependencies=[b, c])
self.products.add_for_target(a, [1])
self.products.add_for_target(b, [2])
copied = self.products.copy()
self.assertEqual(self.products.get_for_targets(a.closure(bfs=True)), OrderedSet([1, 2]))
self.assertEqual(self.products.get_for_targets(b.closure(bfs=True)), OrderedSet([2]))
self.assertEqual(copied.get_for_targets(a.closure(bfs=True)), OrderedSet([1, 2]))
self.assertEqual(copied.get_for_targets(b.closure(bfs=True)), OrderedSet([2]))
copied.add_for_target(c, [3])
self.assertEqual(self.products.get_for_targets(a.closure(bfs=True)), OrderedSet([1, 2]))
self.assertEqual(self.products.get_for_targets(b.closure(bfs=True)), OrderedSet([2]))
self.assertEqual(self.products.get_for_targets(c.closure(bfs=True)), OrderedSet())
self.assertEqual(copied.get_for_targets(a.closure(bfs=True)), OrderedSet([1, 2, 3]))
self.assertEqual(copied.get_for_targets(b.closure(bfs=True)), OrderedSet([2, 3]))
self.assertEqual(copied.get_for_targets(c.closure(bfs=True)), OrderedSet([3]))
def test_remove_for_target(self):
c = self.make_target('c')
b = self.make_target('b', dependencies=[c])
a = self.make_target('a', dependencies=[b, c])
self.products.add_for_target(a, [1])
self.products.add_for_target(b, [2])
self.products.add_for_target(c, [3])
self.products.remove_for_target(a, [1])
self.assertEqual(self.products.get_for_targets(a.closure(bfs=True)), OrderedSet([2, 3]))
self.assertEqual(self.products.get_for_targets(b.closure(bfs=True)), OrderedSet([2, 3]))
self.assertEqual(self.products.get_for_targets(c.closure(bfs=True)), OrderedSet([3]))
def test_empty_products(self):
c = self.make_target('c')
self.assertFalse(self.products.get_for_target(c))
def test_non_empty_products(self):
c = self.make_target('c')
self.products.add_for_target(c, [3])
self.assertTrue(self.products.get_for_target(c))
def test_target_for_product_existing_product(self):
c = self.make_target('c')
self.products.add_for_target(c, [3])
found_target = self.products.target_for_product(3)
self.assertEqual(c, found_target)
def test_target_for_product_nonexistent_product(self):
c = self.make_target('c')
self.products.add_for_target(c, [3])
found_target = self.products.target_for_product(1000)
self.assertIsNone(found_target)
class ClasspathProducts(object):
def __init__(self):
self._classpaths = UnionProducts()
self._exclude_patterns = UnionProducts()
self._buildroot = get_buildroot()
def add_for_targets(self, targets, classpath_elements):
"""Adds classpath elements to the products of all the provided targets."""
for target in targets:
self.add_for_target(target, classpath_elements)
def add_for_target(self, target, classpath_elements):
"""Adds classpath elements to the products of the provided target."""
self._validate_classpath_tuples(classpath_elements, target)
self._classpaths.add_for_target(target, classpath_elements)
def add_excludes_for_targets(self, targets):
"""Add excludes from the provided targets. Does not look up transitive excludes."""
for target in targets:
self._add_excludes_for_target(target)
def remove_for_target(self, target, classpath_elements):
"""Removes the given entries for the target"""
self._classpaths.remove_for_target(target, classpath_elements)
def get_for_target(self, target):
"""Gets the transitive classpath products for the given target, in order, respecting target
excludes."""
return self.get_for_targets([target])
def get_for_targets(self, targets):
"""Gets the transitive classpath products for the given targets, in order, respecting target
excludes."""
classpath_tuples = self._classpaths.get_for_targets(targets)
filtered_classpath_tuples = self._filter_by_excludes(classpath_tuples, targets)
return filtered_classpath_tuples
def _filter_by_excludes(self, classpath_tuples, root_targets):
exclude_patterns = self._exclude_patterns.get_for_targets(root_targets)
filtered_classpath_tuples = filter(_not_excluded_filter(exclude_patterns),
classpath_tuples)
return filtered_classpath_tuples
def _add_excludes_for_target(self, target):
# TODO(nhoward): replace specific ivy based exclude filterings in the jar object refactor
# creates strings from excludes that will match classpath entries generated by ivy
# eg exclude(org='org.example', name='lib') => 'jars/org.example/lib'
# exclude(org='org.example') => 'jars/org.example/'
# The empty string was added to the end of the os.path.join list, so that the exclude pattern
# always ends with a path separator. It's a short term fix so we don't match the following
# 'jars/com.twitter.common/text' in '.../jars/com.twitter.common/text-lang-model/jars/text...'
if target.is_exported:
self._exclude_patterns.add_for_target(target,
[os.path.join('jars',
target.provides.org,
target.provides.name,
'')])
if isinstance(target, JvmTarget) and target.excludes:
self._exclude_patterns.add_for_target(target,
[os.path.join('jars', e.org, e.name or '', '')
for e in target.excludes])
def _validate_classpath_tuples(self, classpath, target):
"""Validates that all files are located within the working copy, to simplify relativization."""
for classpath_tuple in classpath:
self._validate_path_in_buildroot(classpath_tuple, target)
def _validate_path_in_buildroot(self, classpath_tuple, target):
conf, path = classpath_tuple
if os.path.relpath(path, self._buildroot).startswith(os.pardir):
raise TaskError(
'Classpath entry {} for target {} is located outside the buildroot.'
.format(path, target.address.spec))
def __init__(self):
self._classpaths = UnionProducts()
self._exclude_patterns = UnionProducts()
self._buildroot = get_buildroot()
def __init__(self, classpaths=None, excludes=None):
self._classpaths = classpaths or UnionProducts()
self._excludes = excludes or UnionProducts()
self._buildroot = get_buildroot()
class UnionProductsTest(BaseTest):
def setUp(self):
super(UnionProductsTest, self).setUp()
self.products = UnionProducts()
def test_get_for_target(self):
c = self.make_target('c')
b = self.make_target('b', dependencies=[c])
a = self.make_target('a', dependencies=[b, c])
self.products.add_for_target(a, [1])
self.products.add_for_target(b, [2])
self.products.add_for_target(c, [3])
self.assertEquals(self.products.get_for_target(a), OrderedSet([1, 2, 3]))
self.assertEquals(self.products.get_for_target(b), OrderedSet([2, 3]))
self.assertEquals(self.products.get_for_target(c), OrderedSet([3]))
self.assertEquals(self.products.get_for_target(a, transitive=False), OrderedSet([1]))
self.assertEquals(self.products.get_for_target(b, transitive=False), OrderedSet([2]))
self.assertEquals(self.products.get_for_target(c, transitive=False), OrderedSet([3]))
def test_remove_for_target(self):
c = self.make_target('c')
b = self.make_target('b', dependencies=[c])
a = self.make_target('a', dependencies=[b, c])
self.products.add_for_target(a, [1])
self.products.add_for_target(b, [2])
self.products.add_for_target(c, [3])
self.products.remove_for_target(a, [1])
self.assertEquals(self.products.get_for_target(a), OrderedSet([2, 3]))
self.assertEquals(self.products.get_for_target(b), OrderedSet([2, 3]))
self.assertEquals(self.products.get_for_target(c), OrderedSet([3]))
def test_empty_products(self):
c = self.make_target('c')
self.assertFalse(self.products.get_for_target(c))
def test_non_empty_products(self):
c = self.make_target('c')
self.products.add_for_target(c, [3])
self.assertTrue(self.products.get_for_target(c))
def test_target_for_product_existing_product(self):
c = self.make_target('c')
self.products.add_for_target(c, [3])
found_target = self.products.target_for_product(3)
self.assertEqual(c, found_target)
def test_target_for_product_nonexistent_product(self):
c = self.make_target('c')
self.products.add_for_target(c, [3])
found_target = self.products.target_for_product(1000)
self.assertIsNone(found_target)
class ClasspathProducts(object):
def __init__(self):
self._classpaths = UnionProducts()
self._excludes = UnionProducts()
self._buildroot = get_buildroot()
def add_for_targets(self, targets, classpath_elements):
"""Adds classpath path elements to the products of all the provided targets."""
for target in targets:
self.add_for_target(target, classpath_elements)
def add_for_target(self, target, classpath_elements):
"""Adds classpath path elements to the products of the provided target."""
self._add_elements_for_target(
target, self._wrap_path_elements(classpath_elements))
def add_jars_for_targets(self, targets, conf, resolved_jars):
"""Adds jar classpath elements to the products of the provided targets in a way that works with
excludes.
"""
classpath_entries = []
for jar in resolved_jars:
if not jar.pants_path:
raise TaskError("Jar: {!s} has no specified path.".format(
jar.coordinate))
classpath_entries.append(
(conf, ArtifactClasspathEntry(jar.coordinate, jar.pants_path)))
for target in targets:
self._add_elements_for_target(target, classpath_entries)
def add_excludes_for_targets(self, targets):
"""Add excludes from the provided targets. Does not look up transitive excludes."""
for target in targets:
self._add_excludes_for_target(target)
def remove_for_target(self, target, classpath_elements):
"""Removes the given entries for the target"""
self._classpaths.remove_for_target(
target, self._wrap_path_elements(classpath_elements))
def get_for_target(self, target, transitive=True):
"""Gets the transitive classpath products for the given target, in order, respecting target
excludes."""
return self.get_for_targets([target], transitive=transitive)
def get_for_targets(self, targets, transitive=True):
"""Gets the transitive classpath products for the given targets, in order, respecting target
excludes."""
classpath_tuples = self._classpaths.get_for_targets(
targets, transitive)
filtered_classpath_tuples = self._filter_by_excludes(
classpath_tuples, targets, transitive)
return [(classpath_tuple[0], classpath_tuple[1].path)
for classpath_tuple in filtered_classpath_tuples]
def _filter_by_excludes(self, classpath_tuples, root_targets, transitive):
excludes = self._excludes.get_for_targets(root_targets,
transitive=transitive)
return filter(_not_excluded_filter(excludes), classpath_tuples)
def _add_excludes_for_target(self, target):
if target.is_exported:
self._excludes.add_for_target(
target, [Exclude(target.provides.org, target.provides.name)])
if isinstance(target, JvmTarget) and target.excludes:
self._excludes.add_for_target(target, target.excludes)
def _wrap_path_elements(self, classpath_elements):
return [(element[0], ClasspathEntry(element[1]))
for element in classpath_elements]
def _add_elements_for_target(self, target, elements):
self._validate_classpath_tuples(elements, target)
self._classpaths.add_for_target(target, elements)
def _validate_classpath_tuples(self, classpath, target):
"""Validates that all files are located within the working copy, to simplify relativization.
:param classpath: The list of classpath tuples. Each tuple is a 2-tuple of ivy_conf and ClasspathEntry
:param target: The target that the classpath tuple is being registered for
:raises: `TaskError` when the path is outside the build root
"""
for classpath_tuple in classpath:
conf, classpath_entry = classpath_tuple
path = classpath_entry.path
if os.path.relpath(path, self._buildroot).startswith(os.pardir):
raise TaskError(
'Classpath entry {} for target {} is located outside the buildroot.'
.format(path, target.address.spec))
def __init__(self): self._classpaths = UnionProducts() self._excludes = UnionProducts() self._buildroot = get_buildroot()
class ClasspathProducts(object):
def __init__(self):
self._classpaths = UnionProducts()
self._excludes = UnionProducts()
self._buildroot = get_buildroot()
def add_for_targets(self, targets, classpath_elements):
"""Adds classpath path elements to the products of all the provided targets."""
for target in targets:
self.add_for_target(target, classpath_elements)
def add_for_target(self, target, classpath_elements):
"""Adds classpath path elements to the products of the provided target."""
self._add_elements_for_target(target, self._wrap_path_elements(classpath_elements))
def add_jars_for_targets(self, targets, conf, resolved_jars):
"""Adds jar classpath elements to the products of the provided targets in a way that works with
excludes.
"""
classpath_entries = []
for jar in resolved_jars:
if not jar.pants_path:
raise TaskError("Jar: {!s} has no specified path.".format(jar.coordinate))
classpath_entries.append((conf, ArtifactClasspathEntry(jar.coordinate, jar.pants_path)))
for target in targets:
self._add_elements_for_target(target, classpath_entries)
def add_excludes_for_targets(self, targets):
"""Add excludes from the provided targets. Does not look up transitive excludes."""
for target in targets:
self._add_excludes_for_target(target)
def remove_for_target(self, target, classpath_elements):
"""Removes the given entries for the target"""
self._classpaths.remove_for_target(target, self._wrap_path_elements(classpath_elements))
def get_for_target(self, target, transitive=True):
"""Gets the transitive classpath products for the given target, in order, respecting target
excludes."""
return self.get_for_targets([target], transitive=transitive)
def get_for_targets(self, targets, transitive=True):
"""Gets the transitive classpath products for the given targets, in order, respecting target
excludes."""
classpath_tuples = self._classpaths.get_for_targets(targets, transitive)
filtered_classpath_tuples = self._filter_by_excludes(classpath_tuples, targets, transitive)
return [(classpath_tuple[0], classpath_tuple[1].path) for classpath_tuple in filtered_classpath_tuples]
def _filter_by_excludes(self, classpath_tuples, root_targets, transitive):
excludes = self._excludes.get_for_targets(root_targets, transitive=transitive)
return filter(_not_excluded_filter(excludes),
classpath_tuples)
def _add_excludes_for_target(self, target):
if target.is_exported:
self._excludes.add_for_target(target, [Exclude(target.provides.org,
target.provides.name)])
if isinstance(target, JvmTarget) and target.excludes:
self._excludes.add_for_target(target, target.excludes)
def _wrap_path_elements(self, classpath_elements):
return [(element[0], ClasspathEntry(element[1])) for element in classpath_elements]
def _add_elements_for_target(self, target, elements):
self._validate_classpath_tuples(elements, target)
self._classpaths.add_for_target(target, elements)
def _validate_classpath_tuples(self, classpath, target):
"""Validates that all files are located within the working copy, to simplify relativization.
:param classpath: The list of classpath tuples. Each tuple is a 2-tuple of ivy_conf and ClasspathEntry
:param target: The target that the classpath tuple is being registered for
:raises: `TaskError` when the path is outside the build root
"""
for classpath_tuple in classpath:
conf, classpath_entry = classpath_tuple
path = classpath_entry.path
if os.path.relpath(path, self._buildroot).startswith(os.pardir):
raise TaskError(
'Classpath entry {} for target {} is located outside the buildroot.'
.format(path, target.address.spec))
