def test_does_not_visit_diamond_dep_twice(self):
ivy_info = self.parse_ivy_report(
"ivy_utils_resources/report_with_diamond.xml")
ref = IvyModuleRef("toplevel", "toplevelmodule", "latest")
seen = set()
def collector(r):
self.assertNotIn(r, seen)
seen.add(r)
return {r}
result = ivy_info.traverse_dependency_graph(ref, collector)
self.assertEqual(
{
IvyModuleRef("toplevel", "toplevelmodule", "latest"),
IvyModuleRef(
org="org1", name="name1", rev="0.0.1", classifier="tests"),
IvyModuleRef(org="org2", name="name2", rev="0.0.1"),
IvyModuleRef(
org="org3", name="name3", rev="0.0.1", ext="tar.gz"),
},
result,
)
def test_collects_classifiers(self):
ivy_info = self.parse_ivy_report(
"ivy_utils_resources/report_with_multiple_classifiers.xml")
ref = IvyModuleRef("toplevel", "toplevelmodule", "latest")
def collector(r):
x = ivy_info.modules_by_ref.get(r)
if x:
return {x}
else:
return set()
result = ivy_info.traverse_dependency_graph(ref, collector, dict())
self.assertEqual(
{
IvyModule(
ref=IvyModuleRef(org="org1",
name="name1",
rev="0.0.1",
classifier=None,
ext="jar"),
artifact="ivy2cache_path/org1/name1.jar",
callers=(IvyModuleRef(
org="toplevel",
name="toplevelmodule",
rev="latest",
classifier=None,
ext="jar",
), ),
),
IvyModule(
ref=IvyModuleRef(org="org1",
name="name1",
rev="0.0.1",
classifier="wut",
ext="jar"),
artifact="ivy2cache_path/org1/name1-wut.jar",
callers=(IvyModuleRef(
org="toplevel",
name="toplevelmodule",
rev="latest",
classifier=None,
ext="jar",
), ),
),
},
result,
)
def test_does_not_visit_diamond_dep_twice(self):
ivy_info = self.parse_ivy_report('ivy_utils_resources/report_with_diamond.xml')
ref = IvyModuleRef("toplevel", "toplevelmodule", "latest")
seen = set()
def collector(r):
self.assertNotIn(r, seen)
seen.add(r)
return {r}
result = ivy_info.traverse_dependency_graph(ref, collector)
self.assertEqual({IvyModuleRef("toplevel", "toplevelmodule", "latest"),
IvyModuleRef(org='org1', name='name1', rev='0.0.1', classifier='tests'),
IvyModuleRef(org='org2', name='name2', rev='0.0.1'),
IvyModuleRef(org='org3', name='name3', rev='0.0.1', ext='tar.gz')},
result)
def test_memo_reused_across_calls(self):
ivy_info = self.parse_ivy_report('ivy_utils_resources/report_with_diamond.xml')
ref = IvyModuleRef(org='org1', name='name1', rev='0.0.1')
def collector(r):
return {r}
memo = dict()
result1 = ivy_info.traverse_dependency_graph(ref, collector, memo=memo)
result2 = ivy_info.traverse_dependency_graph(ref, collector, memo=memo)
self.assertIs(result1, result2)
self.assertEqual(
{
IvyModuleRef(org='org1', name='name1', rev='0.0.1'),
IvyModuleRef(org='org2', name='name2', rev='0.0.1'),
IvyModuleRef(org='org3', name='name3', rev='0.0.1', ext='tar.gz')
},
result1)
def test_memo_reused_across_calls(self):
ivy_info = self.parse_ivy_report("ivy_utils_resources/report_with_diamond.xml")
ref = IvyModuleRef(org="org1", name="name1", rev="0.0.1")
def collector(r):
return {r}
memo = dict()
result1 = ivy_info.traverse_dependency_graph(ref, collector, memo=memo)
result2 = ivy_info.traverse_dependency_graph(ref, collector, memo=memo)
self.assertIs(result1, result2)
self.assertEqual(
{
IvyModuleRef(org="org1", name="name1", rev="0.0.1"),
IvyModuleRef(org="org2", name="name2", rev="0.0.1"),
IvyModuleRef(org="org3", name="name3", rev="0.0.1", ext="tar.gz"),
},
result1,
)
def test_collects_classifiers(self):
ivy_info = self.parse_ivy_report(
'ivy_utils_resources/report_with_multiple_classifiers.xml')
ref = IvyModuleRef("toplevel", "toplevelmodule", "latest")
def collector(r):
x = ivy_info.modules_by_ref.get(r)
if x:
return {x}
else:
return set()
result = ivy_info.traverse_dependency_graph(ref, collector, dict())
self.assertEqual(
{
IvyModule(ref=IvyModuleRef(org='org1',
name='name1',
rev='0.0.1',
classifier=None,
ext=u'jar'),
artifact='ivy2cache_path/org1/name1.jar',
callers=(IvyModuleRef(org='toplevel',
name='toplevelmodule',
rev='latest',
classifier=None,
ext=u'jar'), )),
IvyModule(ref=IvyModuleRef(org='org1',
name='name1',
rev='0.0.1',
classifier='wut',
ext=u'jar'),
artifact='ivy2cache_path/org1/name1-wut.jar',
callers=(IvyModuleRef(org='toplevel',
name='toplevelmodule',
rev='latest',
classifier=None,
ext=u'jar'), ))
}, result)
def test_does_not_follow_cycle(self):
ivy_info = self.parse_ivy_report(
'tests/python/pants_test/tasks/ivy_utils_resources/report_with_cycle.xml'
)
ref = IvyModuleRef("toplevel", "toplevelmodule", "latest")
seen = set()
def collector(r):
self.assertNotIn(r, seen)
seen.add(r)
return set([r])
result = ivy_info.traverse_dependency_graph(ref, collector)
self.assertEqual(
{
IvyModuleRef("toplevel", "toplevelmodule", "latest"),
IvyModuleRef(org='org1', name='name1', rev='0.0.1'),
IvyModuleRef(org='org2', name='name2', rev='0.0.1'),
IvyModuleRef(org='org3', name='name3', rev='0.0.1')
}, result)
def get_jar_paths(self, ivy_info, target, conf):
jar_paths = OrderedSet()
if target.is_jar_library:
# Jar library proxies jar dependencies or jvm targets, so the jars are just those of the
# dependencies.
for paths in [
self.get_jar_paths(ivy_info, dep, conf)
for dep in target.dependencies
]:
jar_paths.update(paths)
elif isinstance(target, JarDependency):
ref = IvyModuleRef(target.org, target.name, target.rev, conf)
jar_paths.update(self.get_jar_paths_for_ivy_module(ivy_info, ref))
elif target.is_jvm:
for basedir, jars in self.context.products.get('jars').get(
target).items():
jar_paths.update([os.path.join(basedir, jar) for jar in jars])
if self.transitive:
for dep in target.dependencies:
jar_paths.update(self.get_jar_paths(ivy_info, dep, conf))
return jar_paths
def mock_generate_ivy_jar_products(cache_key_ignored):
ivy_products = defaultdict(list)
ivy_info = IvyInfo()
# Guava 16.0 would be evicted by Guava 16.0.1. But in a real
# resolve, it's possible that before it was evicted, it would
# generate some resolution data.
artifact_1 = artifact_path('bogus0')
unused_artifact = artifact_path('unused')
# Because guava 16.0 was evicted, it has no artifacts
guava_0 = IvyModule(
IvyModuleRef('com.google.guava', 'guava', '16.0'), None, [])
guava_1 = IvyModule(
IvyModuleRef('com.google.guava', 'guava', '16.0.1'),
artifact_1, [])
ivy_info.add_module(guava_0)
ivy_info.add_module(guava_1)
artifact_dep_1 = artifact_path('bogus1')
# Because fake#dep 16.0 was evicted before it was resolved,
# its deps are never examined, so we don't call add_module.
guava_dep_0 = IvyModule(
IvyModuleRef('com.google.fake', 'dep', '16.0.0'), None,
[guava_0.ref])
guava_dep_1 = IvyModule(
IvyModuleRef('com.google.fake', 'dep', '16.0.1'),
artifact_dep_1, [guava_1.ref])
ivy_info.add_module(guava_dep_0)
ivy_info.add_module(guava_dep_1)
# Add an unrelated module to ensure that it's not returned
unrelated_parent = IvyModuleRef('com.google.other', 'parent',
'1.0')
unrelated = IvyModule(
IvyModuleRef('com.google.unrelated', 'unrelated', '1.0'),
unused_artifact, [unrelated_parent])
ivy_info.add_module(unrelated)
ivy_products['default'] = [ivy_info]
return ivy_products
def mock_ivy_info_for(conf):
ivy_info = IvyInfo(conf)
# Guava 16.0 would be evicted by Guava 16.0.1. But in a real
# resolve, it's possible that before it was evicted, it would
# generate some resolution data.
artifact_1 = artifact_path("bogus0")
unused_artifact = artifact_path("unused")
# Because guava 16.0 was evicted, it has no artifacts.
guava_0 = IvyModule(IvyModuleRef("com.google.guava", "guava", "16.0"), None, [])
guava_1 = IvyModule(IvyModuleRef("com.google.guava", "guava", "16.0.1"), artifact_1, [])
ivy_info.add_module(guava_0)
ivy_info.add_module(guava_1)
artifact_dep_1 = artifact_path("bogus1")
# Because fake#dep 16.0 was evicted before it was resolved,
# its deps are never examined, so we don't call add_module.
guava_dep_0 = IvyModule(
IvyModuleRef("com.google.fake", "dep", "16.0.0"), None, [guava_0.ref]
)
guava_dep_1 = IvyModule(
IvyModuleRef("com.google.fake", "dep", "16.0.1"), artifact_dep_1, [guava_1.ref]
)
ivy_info.add_module(guava_dep_0)
ivy_info.add_module(guava_dep_1)
# Add an unrelated module to ensure that it's not returned.
unrelated_parent = IvyModuleRef("com.google.other", "parent", "1.0")
unrelated = IvyModule(
IvyModuleRef("com.google.unrelated", "unrelated", "1.0"),
unused_artifact,
[unrelated_parent],
)
ivy_info.add_module(unrelated)
return ivy_info
def test_ivy_module_ref_cmp(self):
self.assertEquals(IvyModuleRef('foo', 'bar', '1.2.3'),
IvyModuleRef('foo', 'bar', '1.2.3'))
self.assertTrue(
IvyModuleRef('foo1', 'bar', '1.2.3') < IvyModuleRef(
'foo2', 'bar', '1.2.3'))
self.assertTrue(
IvyModuleRef('foo2', 'bar', '1.2.3') > IvyModuleRef(
'foo1', 'bar', '1.2.3'))
self.assertTrue(
IvyModuleRef('foo', 'bar1', '1.2.3') < IvyModuleRef(
'foo', 'bar2', '1.2.3'))
self.assertTrue(
IvyModuleRef('foo', 'bar2', '1.2.3') > IvyModuleRef(
'foo', 'bar1', '1.2.3'))
self.assertTrue(
IvyModuleRef('foo', 'bar', '1.2.3') < IvyModuleRef(
'foo', 'bar', '1.2.4'))
self.assertTrue(
IvyModuleRef('foo', 'bar', '1.2.4') > IvyModuleRef(
'foo', 'bar', '1.2.3'))
self.assertTrue(
IvyModuleRef('foo', 'bar', '1.2.3', ext='jar') < IvyModuleRef(
'foo', 'bar', '1.2.3', ext='tgz'))
self.assertTrue(
IvyModuleRef('foo', 'bar', '1.2.3', ext='tgz') > IvyModuleRef(
'foo', 'bar', '1.2.3', ext='jar'))
self.assertTrue(
IvyModuleRef(
'foo', 'bar', '1.2.3', ext='jar', classifier='javadoc') <
IvyModuleRef(
'foo', 'bar', '1.2.3', ext='jar', classifier='sources'))
self.assertTrue(
IvyModuleRef(
'foo', 'bar', '1.2.3', ext='tgz', classifier='sources') >
IvyModuleRef(
'foo', 'bar', '1.2.3', ext='jar', classifier='javadoc'))
# make sure rev is sorted last
self.assertTrue(
IvyModuleRef('foo', 'bar', '1.2.4', classifier='javadoc') <
IvyModuleRef('foo', 'bar', '1.2.3', classifier='sources'))
self.assertTrue(
IvyModuleRef('foo', 'bar', '1.2.3', classifier='sources') >
IvyModuleRef('foo', 'bar', '1.2.4', classifier='javadoc'))
self.assertTrue(
IvyModuleRef('foo', 'bar', '1.2.4', ext='jar') < IvyModuleRef(
'foo', 'bar', '1.2.3', ext='tgz'))
self.assertTrue(
IvyModuleRef('foo', 'bar', '1.2.3', ext='tgz') > IvyModuleRef(
'foo', 'bar', '1.2.4', ext='jar'))
def make_ref(org, name):
return IvyModuleRef(org=org, name=name, rev='1.0')
def test_module_ref_str_minus_classifier(self):
module_ref = IvyModuleRef(org='org', name='name', rev='rev')
self.assertEquals("IvyModuleRef(org:name:rev::jar)", str(module_ref))
def process_target(current_target):
"""
:type current_target:pants.base.target.Target
"""
def get_target_type(target):
if target.is_test:
return Export.SourceRootTypes.TEST
else:
if (isinstance(target, Resources) and
target in resource_target_map and
resource_target_map[target].is_test):
return Export.SourceRootTypes.TEST_RESOURCE
elif isinstance(target, Resources):
return Export.SourceRootTypes.RESOURCE
else:
return Export.SourceRootTypes.SOURCE
def get_transitive_jars(jar_lib):
"""
:type jar_lib: pants.backend.jvm.targets.jar_library.JarLibrary
:rtype: twitter.common.collections.orderedset.OrderedSet
"""
if not ivy_info or not self.get_options().libraries:
return OrderedSet()
transitive_jars = OrderedSet()
for jar in jar_lib.jar_dependencies:
transitive_jars.update(ivy_info.get_jars_for_ivy_module(jar, memo=ivy_jar_memo))
return transitive_jars
info = {
'targets': [],
'libraries': [],
'roots': [],
'target_type': get_target_type(current_target),
'is_code_gen': current_target.is_codegen,
'pants_target_type': self._get_pants_target_alias(type(current_target))
}
if not current_target.is_synthetic:
info['globs'] = current_target.globs_relative_to_buildroot()
if self.get_options().sources:
info['sources'] = list(current_target.sources_relative_to_buildroot())
target_libraries = set()
if isinstance(current_target, JarLibrary):
target_libraries = get_transitive_jars(current_target)
for dep in current_target.dependencies:
info['targets'].append(self._address(dep.address))
if isinstance(dep, JarLibrary):
for jar in dep.jar_dependencies:
target_libraries.add(IvyModuleRef(jar.org, jar.name, jar.rev))
# Add all the jars pulled in by this jar_library
target_libraries.update(get_transitive_jars(dep))
if isinstance(dep, Resources):
resource_target_map[dep] = current_target
if isinstance(current_target, ScalaLibrary):
for dep in current_target.java_sources:
info['targets'].append(self._address(dep.address))
process_target(dep)
if isinstance(current_target, JvmTarget):
info['excludes'] = [self._exclude_id(exclude) for exclude in current_target.excludes]
info['roots'] = map(lambda (source_root, package_prefix): {
'source_root': source_root,
'package_prefix': package_prefix
}, self._source_roots_for_target(current_target))
if self.get_options().libraries:
info['libraries'] = [self._jar_id(lib) for lib in target_libraries]
targets_map[self._address(current_target.address)] = info
def test_module_ref_str_minus_classifier(self):
module_ref = IvyModuleRef(org="org", name="name", rev="rev")
self.assertEqual("IvyModuleRef(org:name:rev::jar)", str(module_ref))
def test_ivy_module_ref_cmp(self):
self.assertEqual(IvyModuleRef("foo", "bar", "1.2.3"), IvyModuleRef("foo", "bar", "1.2.3"))
self.assertTrue(IvyModuleRef("foo1", "bar", "1.2.3") < IvyModuleRef("foo2", "bar", "1.2.3"))
self.assertTrue(IvyModuleRef("foo2", "bar", "1.2.3") > IvyModuleRef("foo1", "bar", "1.2.3"))
self.assertTrue(IvyModuleRef("foo", "bar1", "1.2.3") < IvyModuleRef("foo", "bar2", "1.2.3"))
self.assertTrue(IvyModuleRef("foo", "bar2", "1.2.3") > IvyModuleRef("foo", "bar1", "1.2.3"))
self.assertTrue(IvyModuleRef("foo", "bar", "1.2.3") < IvyModuleRef("foo", "bar", "1.2.4"))
self.assertTrue(IvyModuleRef("foo", "bar", "1.2.4") > IvyModuleRef("foo", "bar", "1.2.3"))
self.assertTrue(
IvyModuleRef("foo", "bar", "1.2.3", ext="jar")
< IvyModuleRef("foo", "bar", "1.2.3", ext="tgz")
)
self.assertTrue(
IvyModuleRef("foo", "bar", "1.2.3", ext="tgz")
> IvyModuleRef("foo", "bar", "1.2.3", ext="jar")
)
self.assertTrue(
IvyModuleRef("foo", "bar", "1.2.3", ext="jar", classifier="javadoc")
< IvyModuleRef("foo", "bar", "1.2.3", ext="jar", classifier="sources")
)
self.assertTrue(
IvyModuleRef("foo", "bar", "1.2.3", ext="tgz", classifier="sources")
> IvyModuleRef("foo", "bar", "1.2.3", ext="jar", classifier="javadoc")
)
# make sure rev is sorted last
self.assertTrue(
IvyModuleRef("foo", "bar", "1.2.4", classifier="javadoc")
< IvyModuleRef("foo", "bar", "1.2.3", classifier="sources")
)
self.assertTrue(
IvyModuleRef("foo", "bar", "1.2.3", classifier="sources")
> IvyModuleRef("foo", "bar", "1.2.4", classifier="javadoc")
)
self.assertTrue(
IvyModuleRef("foo", "bar", "1.2.4", ext="jar")
< IvyModuleRef("foo", "bar", "1.2.3", ext="tgz")
)
self.assertTrue(
IvyModuleRef("foo", "bar", "1.2.3", ext="tgz")
> IvyModuleRef("foo", "bar", "1.2.4", ext="jar")
)
def process_target(current_target):
"""
:type current_target:pants.base.target.Target
"""
def get_target_type(target):
if target.is_test:
return Export.SourceRootTypes.TEST
else:
if (isinstance(target, Resources)
and target in resource_target_map
and resource_target_map[target].is_test):
return Export.SourceRootTypes.TEST_RESOURCE
elif isinstance(target, Resources):
return Export.SourceRootTypes.RESOURCE
else:
return Export.SourceRootTypes.SOURCE
def get_transitive_jars(jar_lib):
"""
:type jar_lib: pants.backend.jvm.targets.jar_library.JarLibrary
:rtype: twitter.common.collections.orderedset.OrderedSet
"""
if not ivy_info or not self.get_options().libraries:
return OrderedSet()
transitive_jars = OrderedSet()
for jar in jar_lib.jar_dependencies:
transitive_jars.update(
ivy_info.get_jars_for_ivy_module(jar,
memo=ivy_jar_memo))
return transitive_jars
info = {
'targets': [],
'libraries': [],
'roots': [],
'target_type':
get_target_type(current_target),
'is_code_gen':
current_target.is_codegen,
'pants_target_type':
self._get_pants_target_alias(type(current_target))
}
if not current_target.is_synthetic:
info['globs'] = current_target.globs_relative_to_buildroot()
if self.get_options().sources:
info['sources'] = list(
current_target.sources_relative_to_buildroot())
if isinstance(current_target, PythonRequirementLibrary):
reqs = current_target.payload.get_field_value(
'requirements', set())
""":type : set[pants.backend.python.python_requirement.PythonRequirement]"""
info['requirements'] = [req.key for req in reqs]
if isinstance(current_target, PythonTarget):
interpreter_for_target = self.select_interpreter_for_targets(
[current_target])
if interpreter_for_target is None:
raise TaskError(
'Unable to find suitable interpreter for {}'.format(
current_target.address))
python_interpreter_targets_mapping[
interpreter_for_target].append(current_target)
info['python_interpreter'] = str(
interpreter_for_target.identity)
target_libraries = OrderedSet()
if isinstance(current_target, JarLibrary):
target_libraries = get_transitive_jars(current_target)
for dep in current_target.dependencies:
info['targets'].append(dep.address.spec)
if isinstance(dep, JarLibrary):
for jar in dep.jar_dependencies:
target_libraries.add(
IvyModuleRef(jar.org, jar.name, jar.rev))
# Add all the jars pulled in by this jar_library
target_libraries.update(get_transitive_jars(dep))
if isinstance(dep, Resources):
resource_target_map[dep] = current_target
if isinstance(current_target, ScalaLibrary):
for dep in current_target.java_sources:
info['targets'].append(dep.address.spec)
process_target(dep)
if isinstance(current_target, JvmTarget):
info['excludes'] = [
self._exclude_id(exclude)
for exclude in current_target.excludes
]
info['platform'] = current_target.platform.name
info['roots'] = map(
lambda (source_root, package_prefix): {
'source_root': source_root,
'package_prefix': package_prefix
}, self._source_roots_for_target(current_target))
if self.get_options().libraries:
info['libraries'] = [
self._jar_id(lib) for lib in target_libraries
]
targets_map[current_target.address.spec] = info