def execute(self):
safe_mkdir(self.workdir)
targets = self.context.targets()
for conf in self.confs:
outpath = os.path.join(self.workdir,
'{0}.{1}.provides'.format(IvyUtils.identify(targets)[1], conf))
if self.transitive:
outpath += '.transitive'
ivyinfo = IvyUtils.parse_xml_report(self.context.target_roots, conf)
jar_paths = OrderedSet()
for root in self.target_roots:
jar_paths.update(self.get_jar_paths(ivyinfo, root, conf))
with open(outpath, 'w') as outfile:
def do_write(s):
outfile.write(s)
if self.also_write_to_stdout:
sys.stdout.write(s)
for jar in jar_paths:
do_write('# from jar %s\n' % jar)
for line in self.list_jar(jar):
if line.endswith('.class'):
class_name = line[:-6].replace('/', '.')
do_write(class_name)
do_write('\n')
self.context.log.info('Wrote provides information to %s' % outpath)
def execute(self):
targets = self.context.targets()
jars, global_excludes = IvyUtils.calculate_classpath(targets)
filtered_jars = [jar for jar in jars if self._is_update_coordinate(jar.coordinate)]
sorted_jars = sorted((jar for jar in filtered_jars), key=lambda x: (x.org, x.name, x.rev, x.classifier))
ivyxml = os.path.join(self.workdir, 'ivy.xml')
IvyUtils.generate_ivy(targets, jars=sorted_jars, excludes=global_excludes, ivyxml=ivyxml, confs=['default'])
args = [
'-settings', IvySubsystem.global_instance().get_options().ivy_settings,
'-ivy', ivyxml,
'-confs', ','.join(self.get_options().confs)
]
result = self.runjava(classpath=self.tool_classpath('dependency-update-checker'),
main=self._IVY_DEPENDENCY_UPDATE_MAIN,
jvm_options=self.get_options().jvm_options,
args=args,
workunit_name='dependency-update-checker',
workunit_labels=[WorkUnitLabel.LINT])
self.context.log.debug('java {main} ... exited with result ({result})'.format(
main=self._IVY_DEPENDENCY_UPDATE_MAIN, result=result))
return result
class IvyUtilsGenerateIvyTest(IvyUtilsTestBase):
# TODO(John Sirois): increase coverage.
# Some examples:
# + multiple confs - via with_sources and with_docs for example
# + excludes
# + classifiers
# + with_artifact
def setUp(self):
super(IvyUtilsGenerateIvyTest, self).setUp()
self.add_to_build_file('src/java/targets',
dedent('''
jar_library(
name='simple',
jars=[
jar('org1', 'name1', 'rev1'),
jar('org2', 'name2', 'rev2', force=True),
]
)
'''))
self.simple = self.target('src/java/targets:simple')
self.ivy_utils = IvyUtils(create_config(), self.create_options(), logging.Logger('test'))
def test_force_override(self):
jars = list(self.simple.payload.jars)
with temporary_file_path() as ivyxml:
self.ivy_utils._generate_ivy([self.simple], jars=jars, excludes=[], ivyxml=ivyxml,
confs=['default'])
doc = ET.parse(ivyxml).getroot()
conf = self.find_single(doc, 'configurations/conf')
self.assert_attributes(conf, name='default')
dependencies = list(doc.findall('dependencies/dependency'))
self.assertEqual(2, len(dependencies))
dep1 = dependencies[0]
self.assert_attributes(dep1, org='org1', name='name1', rev='rev1')
conf = self.find_single(dep1, 'conf')
self.assert_attributes(conf, name='default', mapped='default')
dep2 = dependencies[1]
self.assert_attributes(dep2, org='org2', name='name2', rev='rev2', force='true')
conf = self.find_single(dep1, 'conf')
self.assert_attributes(conf, name='default', mapped='default')
override = self.find_single(doc, 'dependencies/override')
self.assert_attributes(override, org='org2', module='name2', rev='rev2')
def find_single(self, elem, xpath):
results = list(elem.findall(xpath))
self.assertEqual(1, len(results))
return results[0]
def assert_attributes(self, elem, **kwargs):
self.assertEqual(dict(**kwargs), dict(elem.attrib))
def test_find_new_symlinks(self):
map1 = {'foo': 'bar'}
map2 = {}
diff_map = IvyUtils._find_new_symlinks(map1, map2)
self.assertEquals({}, diff_map)
diff_map = IvyUtils._find_new_symlinks(map2, map1)
self.assertEquals({'foo': 'bar'}, diff_map)
def test_find_new_symlinks(self):
map1 = {"foo": "bar"}
map2 = {}
diff_map = IvyUtils._find_new_symlinks(map1, map2)
self.assertEquals({}, diff_map)
diff_map = IvyUtils._find_new_symlinks(map2, map1)
self.assertEquals({"foo": "bar"}, diff_map)
def test_symlink_cachepath(self):
self.maxDiff = None
with temporary_dir() as mock_cache_dir:
with temporary_dir() as symlink_dir:
with temporary_dir() as classpath_dir:
input_path = os.path.join(classpath_dir, 'inpath')
output_path = os.path.join(classpath_dir, 'classpath')
existing_symlink_map = {}
foo_path = os.path.join(mock_cache_dir, 'foo.jar')
with open(foo_path, 'w') as foo:
foo.write("test jar contents")
with open(input_path, 'w') as inpath:
inpath.write(foo_path)
result_map = IvyUtils.symlink_cachepath(mock_cache_dir, input_path, symlink_dir,
output_path, existing_symlink_map)
symlink_foo_path = os.path.join(symlink_dir, 'foo.jar')
self.assertEquals(
{
foo_path : symlink_foo_path,
os.path.realpath(foo_path) : symlink_foo_path
},
result_map)
with open(output_path, 'r') as outpath:
self.assertEquals(symlink_foo_path, outpath.readline())
self.assertTrue(os.path.islink(symlink_foo_path))
self.assertTrue(os.path.exists(symlink_foo_path))
# Now add an additional path to the existing map
bar_path = os.path.join(mock_cache_dir, 'bar.jar')
with open(bar_path, 'w') as bar:
bar.write("test jar contents2")
with open(input_path, 'w') as inpath:
inpath.write(os.pathsep.join([foo_path, bar_path]))
existing_symlink_map = result_map
result_map = IvyUtils.symlink_cachepath(mock_cache_dir, input_path, symlink_dir,
output_path, existing_symlink_map)
symlink_bar_path = os.path.join(symlink_dir, 'bar.jar')
self.assertEquals(
{
foo_path : symlink_foo_path,
os.path.realpath(foo_path) : symlink_foo_path,
bar_path : symlink_bar_path,
os.path.realpath(bar_path) : symlink_bar_path,
},
result_map)
with open(output_path, 'r') as outpath:
self.assertEquals(symlink_foo_path + os.pathsep + symlink_bar_path, outpath.readline())
self.assertTrue(os.path.islink(symlink_foo_path))
self.assertTrue(os.path.exists(symlink_foo_path))
self.assertTrue(os.path.islink(symlink_bar_path))
self.assertTrue(os.path.exists(symlink_bar_path))
# Reverse the ordering and make sure order is preserved in the output path
with open(input_path, 'w') as inpath:
inpath.write(os.pathsep.join([bar_path, foo_path]))
IvyUtils.symlink_cachepath(mock_cache_dir, input_path, symlink_dir,
output_path, result_map)
with open(output_path, 'r') as outpath:
self.assertEquals(symlink_bar_path + os.pathsep + symlink_foo_path, outpath.readline())
def test_force_override(self):
jars = list(self.a.payload.jars)
with temporary_file_path() as ivyxml:
init_subsystem(JarDependencyManagement)
IvyUtils.generate_ivy([self.a], jars=jars, excludes=[], ivyxml=ivyxml, confs=['default'])
doc = ET.parse(ivyxml).getroot()
conf = self.find_single(doc, 'configurations/conf')
self.assert_attributes(conf, name='default')
dependencies = list(doc.findall('dependencies/dependency'))
self.assertEqual(2, len(dependencies))
dep1 = dependencies[0]
self.assert_attributes(dep1, org='org1', name='name1', rev='rev1')
conf = self.find_single(dep1, 'conf')
self.assert_attributes(conf, name='default', mapped='default')
dep2 = dependencies[1]
self.assert_attributes(dep2, org='org2', name='name2', rev='rev2', force='true')
conf = self.find_single(dep1, 'conf')
self.assert_attributes(conf, name='default', mapped='default')
override = self.find_single(doc, 'dependencies/override')
self.assert_attributes(override, org='org2', module='name2', rev='rev2')
def test_force_override(self):
jars = list(self.a.payload.jars)
with temporary_file_path() as ivyxml:
IvyUtils.generate_ivy([self.a], jars=jars, excludes=[], ivyxml=ivyxml, confs=["default"])
doc = ET.parse(ivyxml).getroot()
conf = self.find_single(doc, "configurations/conf")
self.assert_attributes(conf, name="default")
dependencies = list(doc.findall("dependencies/dependency"))
self.assertEqual(2, len(dependencies))
dep1 = dependencies[0]
self.assert_attributes(dep1, org="org1", name="name1", rev="rev1")
conf = self.find_single(dep1, "conf")
self.assert_attributes(conf, name="default", mapped="default")
dep2 = dependencies[1]
self.assert_attributes(dep2, org="org2", name="name2", rev="rev2", force="true")
conf = self.find_single(dep1, "conf")
self.assert_attributes(conf, name="default", mapped="default")
override = self.find_single(doc, "dependencies/override")
self.assert_attributes(override, org="org2", module="name2", rev="rev2")
def _prepare_ivy_xml(self, targets, ivyxml, hash_name):
jars, global_excludes = IvyUtils.calculate_classpath(targets)
# Don't pass global excludes to ivy when using soft excludes.
if self.soft_excludes:
global_excludes = []
IvyUtils.generate_ivy(targets, jars, global_excludes | self.global_excludes, ivyxml, self.confs,
hash_name, self.pinned_artifacts)
def test_missing_ivy_report(self):
self.set_options_for_scope(IvySubsystem.options_scope, cache_dir='DOES_NOT_EXIST', use_nailgun=False)
# Hack to initialize Ivy subsystem
self.context()
with self.assertRaises(IvyUtils.IvyResolveReportError):
IvyUtils.parse_xml_report('INVALID_REPORT_UNIQUE_NAME', 'default')
def _exec_ivy(self, ivy, executor, confs, ivyxml, args, workunit_name):
try:
IvyUtils.exec_ivy(ivy, confs, ivyxml, args,
jvm_options=self.get_options().jvm_options,
executor=executor,
workunit_name=workunit_name,
workunit_factory=self.context.new_workunit)
except IvyUtils.IvyError as e:
raise self.Error('Ivy resolve failed: {}'.format(e))
def test_fetch_applies_mutable(self):
with temporary_dir() as temp_dir:
ivyxml = os.path.join(temp_dir, "ivy.xml")
IvyUtils.generate_fetch_ivy(
[JarDependency("org-f", "name-f", "rev-f", mutable=True)], ivyxml, ("default",), "some-name"
)
with open(ivyxml) as f:
self.assertIn('changing="true"', f.read())
def test_missing_ivy_report(self):
self.set_options_for_scope(IvySubsystem.options_scope,
cache_dir='DOES_NOT_EXIST',
execution_strategy=NailgunTask.ExecutionStrategy.subprocess)
with self.assertRaises(IvyUtils.IvyResolveReportError):
IvyUtils.parse_xml_report('default', IvyUtils.xml_report_path('INVALID_CACHE_DIR',
'INVALID_REPORT_UNIQUE_NAME',
'default'))
def test_missing_ivy_report(self):
self.set_options_for_scope(IvySubsystem.options_scope,
cache_dir='DOES_NOT_EXIST',
use_nailgun=False)
with self.assertRaises(IvyUtils.IvyResolveReportError):
IvyUtils.parse_xml_report('default', IvyUtils.xml_report_path('INVALID_CACHE_DIR',
'INVALID_REPORT_UNIQUE_NAME',
'default'))
def test_fetch_ivy_xml_requests_url_for_dependency_containing_url(self):
with temporary_dir() as temp_dir:
ivyxml = os.path.join(temp_dir, "ivy.xml")
IvyUtils.generate_fetch_ivy(
[JarDependency("org-f", "name-f", "rev-f", url="an-url")], ivyxml, ("default",), "some-name"
)
with open(ivyxml) as f:
self.assertIn("an-url", f.read())
def test_resolve_conflict_conflict(self):
v1_force = JarDependency('org.example', 'foo', '1', force=True)
v2_force = JarDependency('org.example', 'foo', '2', force=True)
with self.assertRaises(IvyUtils.IvyResolveConflictingDepsError):
IvyUtils._resolve_conflict(v1_force, v2_force)
with self.assertRaises(IvyUtils.IvyResolveConflictingDepsError):
IvyUtils._resolve_conflict(v2_force, v1_force)
def test_missing_ivy_report(self):
self.set_options_for_scope(IvySubsystem.options_scope,
cache_dir='DOES_NOT_EXIST',
use_nailgun=False)
# Hack to initialize Ivy subsystem
self.context()
with self.assertRaises(IvyUtils.IvyResolveReportError):
IvyUtils.parse_xml_report('INVALID_CACHE_DIR', 'INVALID_REPORT_UNIQUE_NAME', 'default')
def test_fetch_applies_mutable(self):
with temporary_dir() as temp_dir:
ivyxml = os.path.join(temp_dir, 'ivy.xml')
IvyUtils.generate_fetch_ivy([JarDependency('org-f', 'name-f', 'rev-f', mutable=True)],
ivyxml,
('default',),
'some-name')
with open(ivyxml) as f:
self.assertIn('changing="true"', f.read())
def test_missing_ivy_report(self):
self.set_options_for_scope(IvySubsystem.options_scope,
cache_dir='DOES_NOT_EXIST')
with self.assertRaises(IvyUtils.IvyResolveReportError):
IvyUtils.parse_xml_report(
'default',
IvyUtils.xml_report_path('INVALID_CACHE_DIR',
'INVALID_REPORT_UNIQUE_NAME',
'default'))
def test_fetch_applies_mutable(self):
with temporary_dir() as temp_dir:
ivyxml = os.path.join(temp_dir, 'ivy.xml')
IvyUtils.generate_fetch_ivy([JarDependency('org-f', 'name-f', 'rev-f', mutable=True)],
ivyxml,
('default',),
'some-name')
with open(ivyxml, 'r') as f:
self.assertIn('changing="true"', f.read())
def test_fetch_requests_classifiers(self):
with temporary_dir() as temp_dir:
ivyxml = os.path.join(temp_dir, 'ivy.xml')
IvyUtils.generate_fetch_ivy([JarDependency('org-f', 'name-f', 'rev-f', classifier='a-classifier')],
ivyxml,
('default',),
'some-name')
with open(ivyxml, 'r') as f:
self.assertIn('a-classifier', f.read())
def test_fetch_ivy_xml_requests_url_for_dependency_containing_url(self):
with temporary_dir() as temp_dir:
ivyxml = os.path.join(temp_dir, 'ivy.xml')
IvyUtils.generate_fetch_ivy([JarDependency('org-f', 'name-f', 'rev-f', url='an-url')],
ivyxml,
('default',),
'some-name')
with open(ivyxml, 'r') as f:
self.assertIn('an-url', f.read())
def test_fetch_requests_classifiers(self):
with temporary_dir() as temp_dir:
ivyxml = os.path.join(temp_dir, 'ivy.xml')
IvyUtils.generate_fetch_ivy([JarDependency('org-f', 'name-f', 'rev-f', classifier='a-classifier')],
ivyxml,
('default',),
'some-name')
with open(ivyxml) as f:
self.assertIn('a-classifier', f.read())
def test_fetch_ivy_xml_requests_url_for_dependency_containing_url(self):
with temporary_dir() as temp_dir:
ivyxml = os.path.join(temp_dir, 'ivy.xml')
IvyUtils.generate_fetch_ivy([JarDependency('org-f', 'name-f', 'rev-f', url='an-url')],
ivyxml,
('default',),
'some-name')
with open(ivyxml) as f:
self.assertIn('an-url', f.read())
def test_missing_ivy_report(self):
self.set_options_for_scope(IvySubsystem.options_scope, cache_dir="DOES_NOT_EXIST", use_nailgun=False)
# Hack to initialize Ivy subsystem
self.context()
with self.assertRaises(IvyUtils.IvyResolveReportError):
IvyUtils.parse_xml_report(
"default", IvyUtils.xml_report_path("INVALID_CACHE_DIR", "INVALID_REPORT_UNIQUE_NAME", "default")
)
def test_symlink_cachepath(self):
self.maxDiff = None
with temporary_dir() as mock_cache_dir:
with temporary_dir() as symlink_dir:
with temporary_dir() as classpath_dir:
input_path = os.path.join(classpath_dir, 'inpath')
output_path = os.path.join(classpath_dir, 'classpath')
foo_path = os.path.join(mock_cache_dir, 'foo.jar')
with open(foo_path, 'w') as foo:
foo.write("test jar contents")
with open(input_path, 'w') as inpath:
inpath.write(foo_path)
result_map = IvyUtils.symlink_cachepath(mock_cache_dir, input_path, symlink_dir,
output_path)
symlink_foo_path = os.path.join(symlink_dir, 'foo.jar')
self.assertEquals(
{
os.path.realpath(foo_path): symlink_foo_path
},
result_map)
with open(output_path, 'r') as outpath:
self.assertEquals(symlink_foo_path, outpath.readline())
self.assertTrue(os.path.islink(symlink_foo_path))
self.assertTrue(os.path.exists(symlink_foo_path))
# Now add an additional path to the existing map
bar_path = os.path.join(mock_cache_dir, 'bar.jar')
with open(bar_path, 'w') as bar:
bar.write("test jar contents2")
with open(input_path, 'w') as inpath:
inpath.write(os.pathsep.join([foo_path, bar_path]))
result_map = IvyUtils.symlink_cachepath(mock_cache_dir, input_path, symlink_dir,
output_path)
symlink_bar_path = os.path.join(symlink_dir, 'bar.jar')
self.assertEquals(
{
os.path.realpath(foo_path): symlink_foo_path,
os.path.realpath(bar_path): symlink_bar_path,
},
result_map)
with open(output_path, 'r') as outpath:
self.assertEquals(symlink_foo_path + os.pathsep + symlink_bar_path, outpath.readline())
self.assertTrue(os.path.islink(symlink_foo_path))
self.assertTrue(os.path.exists(symlink_foo_path))
self.assertTrue(os.path.islink(symlink_bar_path))
self.assertTrue(os.path.exists(symlink_bar_path))
# Reverse the ordering and make sure order is preserved in the output path
with open(input_path, 'w') as inpath:
inpath.write(os.pathsep.join([bar_path, foo_path]))
IvyUtils.symlink_cachepath(mock_cache_dir, input_path, symlink_dir, output_path)
with open(output_path, 'r') as outpath:
self.assertEquals(symlink_bar_path + os.pathsep + symlink_foo_path, outpath.readline())
def __init__(self, *args, **kwargs):
super(Provides, self).__init__(*args, **kwargs)
self.ivy_utils = IvyUtils(config=self.context.config,
options=self.context.options,
log=self.context.log)
self.confs = self.context.config.getlist('ivy',
'confs',
default=['default'])
self.target_roots = self.context.target_roots
self.transitive = self.context.options.provides_transitive
self.also_write_to_stdout = self.context.options.provides_also_write_to_stdout or False
def test_fetch_applies_mutable(self):
with temporary_dir() as temp_dir:
ivyxml = os.path.join(temp_dir, "ivy.xml")
IvyUtils.generate_fetch_ivy(
[JarDependency("org-f", "name-f", "rev-f", mutable=True)],
ivyxml,
("default",),
"some-name",
)
with open(ivyxml, "r") as f:
self.assertIn('changing="true"', f.read())
def test_fetch_requests_classifiers(self):
with temporary_dir() as temp_dir:
ivyxml = os.path.join(temp_dir, "ivy.xml")
IvyUtils.generate_fetch_ivy(
[JarDependency("org-f", "name-f", "rev-f", classifier="a-classifier")],
ivyxml,
("default",),
"some-name",
)
with open(ivyxml, "r") as f:
self.assertIn("a-classifier", f.read())
def test_fetch_ivy_xml_requests_url_for_dependency_containing_url(self):
with temporary_dir() as temp_dir:
ivyxml = os.path.join(temp_dir, "ivy.xml")
IvyUtils.generate_fetch_ivy(
[JarDependency("org-f", "name-f", "rev-f", url="an-url")],
ivyxml,
("default",),
"some-name",
)
with open(ivyxml, "r") as f:
self.assertIn("an-url", f.read())
def test_fetch_requests_classifiers(self):
with temporary_dir() as temp_dir:
ivyxml = os.path.join(temp_dir, "ivy.xml")
IvyUtils.generate_fetch_ivy(
[JarDependency("org-f", "name-f", "rev-f", classifier="a-classifier")],
ivyxml,
("default",),
"some-name",
)
with open(ivyxml) as f:
self.assertIn("a-classifier", f.read())
def test_construct_and_load_symlink_map(self):
self.maxDiff = None
with temporary_dir() as mock_cache_dir:
with temporary_dir() as symlink_dir:
with temporary_dir() as classpath_dir:
input_path = os.path.join(classpath_dir, "inpath")
output_path = os.path.join(classpath_dir, "classpath")
foo_path = os.path.join(mock_cache_dir, "foo.jar")
with open(foo_path, "w") as foo:
foo.write("test jar contents")
with open(input_path, "w") as inpath:
inpath.write(foo_path)
result_classpath, result_map = IvyUtils.construct_and_load_symlink_map(
symlink_dir, mock_cache_dir, input_path, output_path
)
symlink_foo_path = os.path.join(symlink_dir, "foo.jar")
self.assertEquals([symlink_foo_path], result_classpath)
self.assertEquals({os.path.realpath(foo_path): symlink_foo_path}, result_map)
with open(output_path, "r") as outpath:
self.assertEquals(symlink_foo_path, outpath.readline())
self.assertTrue(os.path.islink(symlink_foo_path))
self.assertTrue(os.path.exists(symlink_foo_path))
# Now add an additional path to the existing map
bar_path = os.path.join(mock_cache_dir, "bar.jar")
with open(bar_path, "w") as bar:
bar.write("test jar contents2")
with open(input_path, "w") as inpath:
inpath.write(os.pathsep.join([foo_path, bar_path]))
result_classpath, result_map = IvyUtils.construct_and_load_symlink_map(
symlink_dir, mock_cache_dir, input_path, output_path
)
symlink_bar_path = os.path.join(symlink_dir, "bar.jar")
self.assertEquals(
{os.path.realpath(foo_path): symlink_foo_path, os.path.realpath(bar_path): symlink_bar_path},
result_map,
)
self.assertEquals([symlink_foo_path, symlink_bar_path], result_classpath)
with open(output_path, "r") as outpath:
self.assertEquals(symlink_foo_path + os.pathsep + symlink_bar_path, outpath.readline())
self.assertTrue(os.path.islink(symlink_foo_path))
self.assertTrue(os.path.exists(symlink_foo_path))
self.assertTrue(os.path.islink(symlink_bar_path))
self.assertTrue(os.path.exists(symlink_bar_path))
# Reverse the ordering and make sure order is preserved in the output path
with open(input_path, "w") as inpath:
inpath.write(os.pathsep.join([bar_path, foo_path]))
IvyUtils.construct_and_load_symlink_map(symlink_dir, mock_cache_dir, input_path, output_path)
with open(output_path, "r") as outpath:
self.assertEquals(symlink_bar_path + os.pathsep + symlink_foo_path, outpath.readline())
def _generate_ivy_report(self, targets):
def make_empty_report(report, organisation, module, conf):
no_deps_xml_template = """
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="ivy-report.xsl"?>
<ivy-report version="1.0">
<info
organisation="%(organisation)s"
module="%(module)s"
revision="latest.integration"
conf="%(conf)s"
confs="%(conf)s"
date="%(timestamp)s"/>
</ivy-report>
"""
no_deps_xml = no_deps_xml_template % dict(organisation=organisation,
module=module,
conf=conf,
timestamp=time.strftime('%Y%m%d%H%M%S'))
with open(report, 'w') as report_handle:
print(no_deps_xml, file=report_handle)
classpath = self.tool_classpath(self._ivy_bootstrap_key, self.create_java_executor())
reports = []
org, name = IvyUtils.identify(targets)
xsl = os.path.join(self._cachedir, 'ivy-report.xsl')
# Xalan needs this dir to exist - ensure that, but do no more - we have no clue where this
# points.
safe_mkdir(self._outdir, clean=False)
for conf in self._confs:
params = dict(org=org, name=name, conf=conf)
xml = IvyUtils.xml_report_path(targets, conf)
if not os.path.exists(xml):
make_empty_report(xml, org, name, conf)
out = os.path.join(self._outdir, '%(org)s-%(name)s-%(conf)s.html' % params)
args = ['-IN', xml, '-XSL', xsl, '-OUT', out]
if 0 != self.runjava(classpath=classpath, main='org.apache.xalan.xslt.Process',
args=args, workunit_name='report'):
raise TaskError
reports.append(out)
css = os.path.join(self._outdir, 'ivy-report.css')
if os.path.exists(css):
os.unlink(css)
shutil.copy(os.path.join(self._cachedir, 'ivy-report.css'), self._outdir)
if self._open:
binary_util.ui_open(*reports)
def _exec_ivy(self,
target_workdir,
targets,
args,
executor=None,
confs=None,
ivy=None,
workunit_name='ivy',
use_soft_excludes=False,
resolve_hash_name=None):
ivy_jvm_options = self.get_options().jvm_options[:]
# Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
ivy_jvm_options.append('-Dsun.io.useCanonCaches=false')
ivy = ivy or Bootstrapper.default_ivy()
ivyxml = os.path.join(target_workdir, 'ivy.xml')
ivy_args = ['-ivy', ivyxml]
confs_to_resolve = confs or ('default', )
ivy_args.append('-confs')
ivy_args.extend(confs_to_resolve)
ivy_args.extend(args)
# TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better
# diagnostics can be had in `IvyUtils.generate_ivy` if this is done.
# See: https://github.com/pantsbuild/pants/issues/2239
try:
jars, excludes = IvyUtils.calculate_classpath(
targets, gather_excludes=not use_soft_excludes)
with IvyUtils.ivy_lock:
IvyUtils.generate_ivy(targets, jars, excludes, ivyxml,
confs_to_resolve, resolve_hash_name)
runner = ivy.runner(jvm_options=ivy_jvm_options,
args=ivy_args,
executor=executor)
try:
result = execute_runner(
runner,
workunit_factory=self.context.new_workunit,
workunit_name=workunit_name)
if result != 0:
raise self.Error(
'Ivy returned {result}. cmd={cmd}'.format(
result=result, cmd=runner.cmd))
except runner.executor.Error as e:
raise self.Error(e)
except IvyUtils.IvyError as e:
raise self.Error('Failed to prepare ivy resolve: {}'.format(e))
def execute(self):
deprecated_conditional(
lambda: True,
removal_version="1.31.0.dev0",
entity_description="The `outdated` goal",
hint_message=
"Contact the Pants team on Slack or [email protected] "
"if you need this functionality.",
)
targets = self.context.targets()
jars, global_excludes = IvyUtils.calculate_classpath(targets)
filtered_jars = [
jar for jar in jars if self._is_update_coordinate(jar.coordinate)
]
sorted_jars = sorted((jar for jar in filtered_jars),
key=lambda x:
(x.org, x.name, x.rev, x.classifier))
ivyxml = os.path.join(self.workdir, "ivy.xml")
IvyUtils.generate_ivy(targets,
jars=sorted_jars,
excludes=global_excludes,
ivyxml=ivyxml,
confs=["default"])
args = [
"-settings",
IvySubsystem.global_instance().get_options().ivy_settings,
"-ivy",
ivyxml,
"-confs",
",".join(self.get_options().confs),
]
result = self.runjava(
classpath=self.tool_classpath("dependency-update-checker"),
main=self._IVY_DEPENDENCY_UPDATE_MAIN,
jvm_options=self.get_options().jvm_options,
args=args,
workunit_name="dependency-update-checker",
workunit_labels=[WorkUnitLabel.LINT],
)
self.context.log.debug(
"java {main} ... exited with result ({result})".format(
main=self._IVY_DEPENDENCY_UPDATE_MAIN, result=result))
return result
def test_resolve_ivy_xml_requests_classifiers(self):
with temporary_dir() as temp_dir:
ivyxml = os.path.join(temp_dir, 'ivy.xml')
jar_dep = JarDependency('org-f', 'name-f', 'rev-f', classifier='a-classifier')
IvyUtils.generate_ivy(
[self.make_target('something', JarLibrary, jars=[jar_dep])],
[jar_dep],
excludes=[],
ivyxml=ivyxml,
confs=('default',),
resolve_hash_name='some-name',
jar_dep_manager=namedtuple('stub_jar_dep_manager', ['resolve_version_conflict'])(lambda x: x))
with open(ivyxml, 'r') as f:
self.assertIn('classifier="a-classifier', f.read())
def test_resolve_ivy_xml_requests_classifiers(self):
with temporary_dir() as temp_dir:
ivyxml = os.path.join(temp_dir, 'ivy.xml')
jar_dep = JarDependency('org-f', 'name-f', 'rev-f', classifier='a-classifier')
IvyUtils.generate_ivy(
[self.make_target('something', JarLibrary, jars=[jar_dep])],
[jar_dep],
excludes=[],
ivyxml=ivyxml,
confs=('default',),
resolve_hash_name='some-name',
jar_dep_manager=namedtuple('stub_jar_dep_manager', ['resolve_version_conflict'])(lambda x: x))
with open(ivyxml) as f:
self.assertIn('classifier="a-classifier', f.read())
def test_resolve_ivy_xml_requests_classifiers(self):
with temporary_dir() as temp_dir:
ivyxml = os.path.join(temp_dir, "ivy.xml")
jar_dep = JarDependency("org-f", "name-f", "rev-f", classifier="a-classifier")
IvyUtils.generate_ivy(
[self.make_target("something", JarLibrary, jars=[jar_dep])],
[jar_dep],
excludes=[],
ivyxml=ivyxml,
confs=("default",),
resolve_hash_name="some-name",
jar_dep_manager=namedtuple("stub_jar_dep_manager", ["resolve_version_conflict"])(lambda x: x),
)
with open(ivyxml) as f:
self.assertIn('classifier="a-classifier', f.read())
def exec_ivy(self,
target_workdir,
targets,
args,
executor=None,
confs=None,
ivy=None,
workunit_name='ivy',
jars=None,
use_soft_excludes=False,
resolve_hash_name=None):
ivy_jvm_options = copy.copy(self.get_options().jvm_options)
# Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
ivy_jvm_options.append('-Dsun.io.useCanonCaches=false')
ivy = ivy or Bootstrapper.default_ivy()
ivyxml = os.path.join(target_workdir, 'ivy.xml')
if not jars:
jars, excludes = IvyUtils.calculate_classpath(
targets, gather_excludes=not use_soft_excludes)
else:
excludes = set()
ivy_args = ['-ivy', ivyxml]
confs_to_resolve = confs or ['default']
ivy_args.append('-confs')
ivy_args.extend(confs_to_resolve)
ivy_args.extend(args)
with IvyUtils.ivy_lock:
IvyUtils.generate_ivy(targets, jars, excludes, ivyxml,
confs_to_resolve, resolve_hash_name)
runner = ivy.runner(jvm_options=ivy_jvm_options,
args=ivy_args,
executor=executor)
try:
result = execute_runner(
runner,
workunit_factory=self.context.new_workunit,
workunit_name=workunit_name)
if result != 0:
raise TaskError('Ivy returned {result}. cmd={cmd}'.format(
result=result, cmd=runner.cmd))
except runner.executor.Error as e:
raise TaskError(e)
def _generate_ivy_jar_products(self, resolve_hash_name):
"""Based on the ivy report, compute a map of conf to lists of IvyInfo objects."""
ivy_products = defaultdict(list)
for conf in self.confs:
ivyinfo = IvyUtils.parse_xml_report(resolve_hash_name, conf)
if ivyinfo:
# TODO(stuhood): Value is a list, previously to accommodate multiple exclusives groups.
ivy_products[conf].append(ivyinfo)
return ivy_products
def test_resolve_ivy_xml_requests_classifiers(self):
with temporary_dir() as temp_dir:
ivyxml = os.path.join(temp_dir, "ivy.xml")
jar_dep = JarDependency("org-f", "name-f", "rev-f", classifier="a-classifier")
IvyUtils.generate_ivy(
[self.make_target("something", JarLibrary, jars=[jar_dep])],
[jar_dep],
excludes=[],
ivyxml=ivyxml,
confs=("default",),
resolve_hash_name="some-name",
jar_dep_manager=namedtuple("stub_jar_dep_manager", ["resolve_version_conflict"])(
lambda x: x
),
)
with open(ivyxml, "r") as f:
self.assertIn('classifier="a-classifier', f.read())
def _populate_ivy_jar_products(self, targets):
"""Populate the build products with an IvyInfo object for each generated ivy report."""
ivy_products = self.context.products.get_data('ivy_jar_products') or defaultdict(list)
for conf in self._confs:
ivyinfo = IvyUtils.parse_xml_report(targets, conf)
if ivyinfo:
# Value is a list, to accommodate multiple exclusives groups.
ivy_products[conf].append(ivyinfo)
self.context.products.safe_create_data('ivy_jar_products', lambda: ivy_products)
def _generate_ivy_jar_products(self, targets):
"""Based on the ivy report, compute a map of conf to lists of IvyInfo objects."""
ivy_products = defaultdict(list)
for conf in self.confs:
ivyinfo = IvyUtils.parse_xml_report(targets, conf)
if ivyinfo:
# TODO(stuhood): Value is a list, previously to accommodate multiple exclusives groups.
ivy_products[conf].append(ivyinfo)
return ivy_products
def setUp(self):
super(IvyUtilsGenerateIvyTest, self).setUp()
self.add_to_build_file(
'src/java/targets',
dedent('''
jar_library(
name='simple',
jars=[
jar('org1', 'name1', 'rev1'),
jar('org2', 'name2', 'rev2', force=True),
]
)
'''))
self.simple = self.target('src/java/targets:simple')
self.ivy_utils = IvyUtils(create_config(), self.create_options(),
logging.Logger('test'))
def setUp(self):
super(IvyUtilsGenerateIvyTest, self).setUp()
self.add_to_build_file(
'src/java/targets',
dedent("""
jar_library(
name='simple',
jars=[
jar('org1', 'name1', 'rev1'),
jar('org2', 'name2', 'rev2', force=True),
]
)
"""))
self.simple = self.target('src/java/targets:simple')
context = self.context()
self.ivy_utils = IvyUtils(context.config, logging.Logger('test'))
def __init__(self, context, workdir):
super(Provides, self).__init__(context, workdir)
self.ivy_utils = IvyUtils(config=context.config,
options=context.options,
log=context.log)
self.confs = context.config.getlist('ivy', 'confs', default=['default'])
self.target_roots = context.target_roots
self.transitive = context.options.provides_transitive
self.also_write_to_stdout = context.options.provides_also_write_to_stdout or False
def __init__(self, *args, **kwargs):
super(Provides, self).__init__(*args, **kwargs)
self.ivy_utils = IvyUtils(config=self.context.config,
options=self.context.options,
log=self.context.log)
self.confs = self.context.config.getlist('ivy', 'confs', default=['default'])
self.target_roots = self.context.target_roots
self.transitive = self.get_options().transitive
self.also_write_to_stdout = self.get_options().also_write_to_stdout
def _exec_ivy(
self,
target_workdir,
targets,
args,
executor=None,
confs=None,
ivy=None,
workunit_name="ivy",
use_soft_excludes=False,
resolve_hash_name=None,
):
ivy_jvm_options = self.get_options().jvm_options[:]
# Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng.
ivy_jvm_options.append("-Dsun.io.useCanonCaches=false")
ivy = ivy or Bootstrapper.default_ivy()
ivyxml = os.path.join(target_workdir, "ivy.xml")
ivy_args = ["-ivy", ivyxml]
confs_to_resolve = confs or ("default",)
ivy_args.append("-confs")
ivy_args.extend(confs_to_resolve)
ivy_args.extend(args)
# TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better
# diagnostics can be had in `IvyUtils.generate_ivy` if this is done.
# See: https://github.com/pantsbuild/pants/issues/2239
try:
jars, excludes = IvyUtils.calculate_classpath(targets, gather_excludes=not use_soft_excludes)
with IvyUtils.ivy_lock:
IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name)
runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor)
try:
result = execute_runner(
runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name
)
if result != 0:
raise self.Error("Ivy returned {result}. cmd={cmd}".format(result=result, cmd=runner.cmd))
except runner.executor.Error as e:
raise self.Error(e)
except IvyUtils.IvyError as e:
raise self.Error("Failed to prepare ivy resolve: {}".format(e))
def __init__(self, *args, **kwargs):
super(IvyResolve, self).__init__(*args, **kwargs)
self._ivy_bootstrapper = Bootstrapper.instance()
self._cachedir = self._ivy_bootstrapper.ivy_cache_dir
self._confs = self.context.config.getlist(self._CONFIG_SECTION,
'confs',
default=['default'])
self._classpath_dir = os.path.join(self.workdir, 'mapped')
self._outdir = self.get_options().outdir or os.path.join(
self.workdir, 'reports')
self._open = self.get_options().open
self._report = self._open or self.get_options().report
self._ivy_utils = IvyUtils(config=self.context.config,
log=self.context.log)
# Typically this should be a local cache only, since classpaths aren't portable.
self.setup_artifact_cache()
def _populate_ivy_jar_products(self, targets):
"""Populate the build products with an IvyInfo object for each generated ivy report."""
ivy_products = self.context.products.get_data(
'ivy_jar_products') or defaultdict(list)
for conf in self._confs:
ivyinfo = IvyUtils.parse_xml_report(targets, conf)
if ivyinfo:
# TODO(stuhood): Value is a list, previously to accommodate multiple exclusives groups.
ivy_products[conf].append(ivyinfo)
self.context.products.safe_create_data('ivy_jar_products',
lambda: ivy_products)
def execute(self):
targets = self.context.targets()
jars, global_excludes = IvyUtils.calculate_classpath(targets)
filtered_jars = [
jar for jar in jars if self._is_update_coordinate(jar.coordinate)
]
sorted_jars = sorted((jar for jar in filtered_jars),
key=lambda x:
(x.org, x.name, x.rev, x.classifier))
ivyxml = os.path.join(self.workdir, "ivy.xml")
IvyUtils.generate_ivy(targets,
jars=sorted_jars,
excludes=global_excludes,
ivyxml=ivyxml,
confs=["default"])
args = [
"-settings",
IvySubsystem.global_instance().get_options().ivy_settings,
"-ivy",
ivyxml,
"-confs",
",".join(self.get_options().confs),
]
result = self.runjava(
classpath=self.tool_classpath("dependency-update-checker"),
main=self._IVY_DEPENDENCY_UPDATE_MAIN,
jvm_options=self.get_options().jvm_options,
args=args,
workunit_name="dependency-update-checker",
workunit_labels=[WorkUnitLabel.LINT],
)
self.context.log.debug(
"java {main} ... exited with result ({result})".format(
main=self._IVY_DEPENDENCY_UPDATE_MAIN, result=result))
return result
def _collect_existing_reports(self, confs, resolve_hash_name):
report_missing = False
report_paths = []
for conf in confs:
report_path = IvyUtils.xml_report_path(self.ivy_cache_dir,
resolve_hash_name, conf)
if not os.path.exists(report_path):
report_missing = True
break
else:
report_paths.append(report_path)
return report_missing, report_paths
def _exec_ivy(self,
target_workdir,
targets,
args,
confs,
executor=None,
ivy=None,
workunit_name='ivy',
use_soft_excludes=False,
resolve_hash_name=None,
pinned_artifacts=None):
# TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better
# diagnostics can be had in `IvyUtils.generate_ivy` if this is done.
# See: https://github.com/pantsbuild/pants/issues/2239
jars, global_excludes = IvyUtils.calculate_classpath(targets)
# Don't pass global excludes to ivy when using soft excludes.
if use_soft_excludes:
global_excludes = []
with IvyUtils.ivy_lock:
ivyxml = os.path.join(target_workdir, 'ivy.xml')
try:
IvyUtils.generate_ivy(targets, jars, global_excludes, ivyxml,
confs, resolve_hash_name,
pinned_artifacts)
except IvyUtils.IvyError as e:
raise self.Error('Failed to prepare ivy resolve: {}'.format(e))
try:
IvyUtils.exec_ivy(ivy,
confs,
ivyxml,
args,
jvm_options=self.get_options().jvm_options,
executor=executor,
workunit_name=workunit_name,
workunit_factory=self.context.new_workunit)
except IvyUtils.IvyError as e:
raise self.Error('Ivy resolve failed: {}'.format(e))
def test_force_override(self):
jars = list(self.a.payload.jars)
with temporary_file_path() as ivyxml:
with subsystem_instance(JarDependencyManagement):
IvyUtils.generate_ivy([self.a],
jars=jars,
excludes=[],
ivyxml=ivyxml,
confs=['default'])
doc = ET.parse(ivyxml).getroot()
conf = self.find_single(doc, 'configurations/conf')
self.assert_attributes(conf, name='default')
dependencies = list(doc.findall('dependencies/dependency'))
self.assertEqual(2, len(dependencies))
dep1 = dependencies[0]
self.assert_attributes(dep1, org='org1', name='name1', rev='rev1')
conf = self.find_single(dep1, 'conf')
self.assert_attributes(conf, name='default', mapped='default')
dep2 = dependencies[1]
self.assert_attributes(dep2,
org='org2',
name='name2',
rev='rev2',
force='true')
conf = self.find_single(dep1, 'conf')
self.assert_attributes(conf, name='default', mapped='default')
override = self.find_single(doc, 'dependencies/override')
self.assert_attributes(override,
org='org2',
module='name2',
rev='rev2')
def test_force_override(self):
jars = list(self.a.payload.jars)
with temporary_file_path() as ivyxml:
init_subsystem(JarDependencyManagement)
IvyUtils.generate_ivy([self.a],
jars=jars,
excludes=[],
ivyxml=ivyxml,
confs=["default"])
doc = ET.parse(ivyxml).getroot()
conf = self.find_single(doc, "configurations/conf")
self.assert_attributes(conf, name="default")
dependencies = list(doc.findall("dependencies/dependency"))
self.assertEqual(2, len(dependencies))
dep1 = dependencies[0]
self.assert_attributes(dep1, org="org1", name="name1", rev="rev1")
conf = self.find_single(dep1, "conf")
self.assert_attributes(conf, name="default", mapped="default")
dep2 = dependencies[1]
self.assert_attributes(dep2,
org="org2",
name="name2",
rev="rev2",
force="true")
conf = self.find_single(dep1, "conf")
self.assert_attributes(conf, name="default", mapped="default")
override = self.find_single(doc, "dependencies/override")
self.assert_attributes(override,
org="org2",
module="name2",
rev="rev2")
def test_resove_conflict_no_conflicts(self):
v1 = JarDependency('org.example', 'foo', '1', force=False)
v1_force = JarDependency('org.example', 'foo', '1', force=True)
v2 = JarDependency('org.example', 'foo', '2', force=False)
# If neither version is forced, use the latest version.
self.assertIs(v2, IvyUtils._resolve_conflict(v1, v2))
self.assertIs(v2, IvyUtils._resolve_conflict(v2, v1))
# If an earlier version is forced, use the forced version.
self.assertIs(v1_force, IvyUtils._resolve_conflict(v1_force, v2))
self.assertIs(v1_force, IvyUtils._resolve_conflict(v2, v1_force))
# If the same version is forced, use the forced version.
self.assertIs(v1_force, IvyUtils._resolve_conflict(v1, v1_force))
self.assertIs(v1_force, IvyUtils._resolve_conflict(v1_force, v1))
# If the same force is in play in multiple locations, allow it.
self.assertIs(v1_force, IvyUtils._resolve_conflict(v1_force, v1_force))
def execute(self):
safe_mkdir(self.workdir)
targets = self.context.targets()
for conf in self.confs:
outpath = os.path.join(
self.workdir,
'{0}.{1}.provides'.format(IvyUtils.identify(targets)[1], conf))
if self.transitive:
outpath += '.transitive'
ivy_jar_products = self.context.products.get_data(
'ivy_jar_products') or {}
# This product is a list for historical reasons (exclusives groups) but in practice should
# have either 0 or 1 entries.
ivy_info_list = ivy_jar_products.get(conf)
if ivy_info_list:
assert len(ivy_info_list) == 1, (
'The values in ivy_jar_products should always be length 1,'
' since we no longer have exclusives groups.')
ivy_info = ivy_info_list[0]
else:
ivy_info = None
jar_paths = OrderedSet()
for root in self.target_roots:
jar_paths.update(self.get_jar_paths(ivy_info, root, conf))
with open(outpath, 'w') as outfile:
def do_write(s):
outfile.write(s)
if self.also_write_to_stdout:
sys.stdout.write(s)
for jar in jar_paths:
do_write('# from jar %s\n' % jar)
for line in self.list_jar(jar):
if line.endswith('.class'):
class_name = line[:-6].replace('/', '.')
do_write(class_name)
do_write('\n')
self.context.log.info('Wrote provides information to %s' % outpath)
def test_resove_conflict(self):
v1 = Mock()
v1.force = False
v1.rev = "1"
v1_force = Mock()
v1_force.force = True
v1_force.rev = "1"
v2 = Mock()
v2.force = False
v2.rev = "2"
# If neither version is forced, use the latest version
self.assertIs(v2, IvyUtils._resolve_conflict(v1, v2))
self.assertIs(v2, IvyUtils._resolve_conflict(v2, v1))
# If an earlier version is forced, use the forced version
self.assertIs(v1_force, IvyUtils._resolve_conflict(v1_force, v2))
self.assertIs(v1_force, IvyUtils._resolve_conflict(v2, v1_force))
# If the same version is forced, use the forced version
self.assertIs(v1_force, IvyUtils._resolve_conflict(v1, v1_force))
self.assertIs(v1_force, IvyUtils._resolve_conflict(v1_force, v1))
def parse_ivy_report(self, rel_path):
path = os.path.join('tests/python/pants_test/backend/jvm/tasks',
rel_path)
ivy_info = IvyUtils._parse_xml_report(conf='default', path=path)
self.assertIsNotNone(ivy_info)
return ivy_info
def test_resolve_conflict_missing_versions(self):
v1 = JarDependency('org.example', 'foo', None, force=False)
v2 = JarDependency('org.example', 'foo', '2', force=False)
self.assertIs(v2, IvyUtils._resolve_conflict(v1, v2))
self.assertIs(v2, IvyUtils._resolve_conflict(v2, v1))
