def execute(self): safe_mkdir(self.workdir) targets = self.context.targets() for conf in self.confs: outpath = os.path.join(self.workdir, '{0}.{1}.provides'.format(IvyUtils.identify(targets)[1], conf)) if self.transitive: outpath += '.transitive' ivyinfo = IvyUtils.parse_xml_report(self.context.target_roots, conf) jar_paths = OrderedSet() for root in self.target_roots: jar_paths.update(self.get_jar_paths(ivyinfo, root, conf)) with open(outpath, 'w') as outfile: def do_write(s): outfile.write(s) if self.also_write_to_stdout: sys.stdout.write(s) for jar in jar_paths: do_write('# from jar %s\n' % jar) for line in self.list_jar(jar): if line.endswith('.class'): class_name = line[:-6].replace('/', '.') do_write(class_name) do_write('\n') self.context.log.info('Wrote provides information to %s' % outpath)
def execute(self): targets = self.context.targets() jars, global_excludes = IvyUtils.calculate_classpath(targets) filtered_jars = [jar for jar in jars if self._is_update_coordinate(jar.coordinate)] sorted_jars = sorted((jar for jar in filtered_jars), key=lambda x: (x.org, x.name, x.rev, x.classifier)) ivyxml = os.path.join(self.workdir, 'ivy.xml') IvyUtils.generate_ivy(targets, jars=sorted_jars, excludes=global_excludes, ivyxml=ivyxml, confs=['default']) args = [ '-settings', IvySubsystem.global_instance().get_options().ivy_settings, '-ivy', ivyxml, '-confs', ','.join(self.get_options().confs) ] result = self.runjava(classpath=self.tool_classpath('dependency-update-checker'), main=self._IVY_DEPENDENCY_UPDATE_MAIN, jvm_options=self.get_options().jvm_options, args=args, workunit_name='dependency-update-checker', workunit_labels=[WorkUnitLabel.LINT]) self.context.log.debug('java {main} ... exited with result ({result})'.format( main=self._IVY_DEPENDENCY_UPDATE_MAIN, result=result)) return result
class IvyUtilsGenerateIvyTest(IvyUtilsTestBase): # TODO(John Sirois): increase coverage. # Some examples: # + multiple confs - via with_sources and with_docs for example # + excludes # + classifiers # + with_artifact def setUp(self): super(IvyUtilsGenerateIvyTest, self).setUp() self.add_to_build_file('src/java/targets', dedent(''' jar_library( name='simple', jars=[ jar('org1', 'name1', 'rev1'), jar('org2', 'name2', 'rev2', force=True), ] ) ''')) self.simple = self.target('src/java/targets:simple') self.ivy_utils = IvyUtils(create_config(), self.create_options(), logging.Logger('test')) def test_force_override(self): jars = list(self.simple.payload.jars) with temporary_file_path() as ivyxml: self.ivy_utils._generate_ivy([self.simple], jars=jars, excludes=[], ivyxml=ivyxml, confs=['default']) doc = ET.parse(ivyxml).getroot() conf = self.find_single(doc, 'configurations/conf') self.assert_attributes(conf, name='default') dependencies = list(doc.findall('dependencies/dependency')) self.assertEqual(2, len(dependencies)) dep1 = dependencies[0] self.assert_attributes(dep1, org='org1', name='name1', rev='rev1') conf = self.find_single(dep1, 'conf') self.assert_attributes(conf, name='default', mapped='default') dep2 = dependencies[1] self.assert_attributes(dep2, org='org2', name='name2', rev='rev2', force='true') conf = self.find_single(dep1, 'conf') self.assert_attributes(conf, name='default', mapped='default') override = self.find_single(doc, 'dependencies/override') self.assert_attributes(override, org='org2', module='name2', rev='rev2') def find_single(self, elem, xpath): results = list(elem.findall(xpath)) self.assertEqual(1, len(results)) return results[0] def assert_attributes(self, elem, **kwargs): self.assertEqual(dict(**kwargs), dict(elem.attrib))
def test_find_new_symlinks(self): map1 = {'foo': 'bar'} map2 = {} diff_map = IvyUtils._find_new_symlinks(map1, map2) self.assertEquals({}, diff_map) diff_map = IvyUtils._find_new_symlinks(map2, map1) self.assertEquals({'foo': 'bar'}, diff_map)
def test_find_new_symlinks(self): map1 = {"foo": "bar"} map2 = {} diff_map = IvyUtils._find_new_symlinks(map1, map2) self.assertEquals({}, diff_map) diff_map = IvyUtils._find_new_symlinks(map2, map1) self.assertEquals({"foo": "bar"}, diff_map)
def test_symlink_cachepath(self): self.maxDiff = None with temporary_dir() as mock_cache_dir: with temporary_dir() as symlink_dir: with temporary_dir() as classpath_dir: input_path = os.path.join(classpath_dir, 'inpath') output_path = os.path.join(classpath_dir, 'classpath') existing_symlink_map = {} foo_path = os.path.join(mock_cache_dir, 'foo.jar') with open(foo_path, 'w') as foo: foo.write("test jar contents") with open(input_path, 'w') as inpath: inpath.write(foo_path) result_map = IvyUtils.symlink_cachepath(mock_cache_dir, input_path, symlink_dir, output_path, existing_symlink_map) symlink_foo_path = os.path.join(symlink_dir, 'foo.jar') self.assertEquals( { foo_path : symlink_foo_path, os.path.realpath(foo_path) : symlink_foo_path }, result_map) with open(output_path, 'r') as outpath: self.assertEquals(symlink_foo_path, outpath.readline()) self.assertTrue(os.path.islink(symlink_foo_path)) self.assertTrue(os.path.exists(symlink_foo_path)) # Now add an additional path to the existing map bar_path = os.path.join(mock_cache_dir, 'bar.jar') with open(bar_path, 'w') as bar: bar.write("test jar contents2") with open(input_path, 'w') as inpath: inpath.write(os.pathsep.join([foo_path, bar_path])) existing_symlink_map = result_map result_map = IvyUtils.symlink_cachepath(mock_cache_dir, input_path, symlink_dir, output_path, existing_symlink_map) symlink_bar_path = os.path.join(symlink_dir, 'bar.jar') self.assertEquals( { foo_path : symlink_foo_path, os.path.realpath(foo_path) : symlink_foo_path, bar_path : symlink_bar_path, os.path.realpath(bar_path) : symlink_bar_path, }, result_map) with open(output_path, 'r') as outpath: self.assertEquals(symlink_foo_path + os.pathsep + symlink_bar_path, outpath.readline()) self.assertTrue(os.path.islink(symlink_foo_path)) self.assertTrue(os.path.exists(symlink_foo_path)) self.assertTrue(os.path.islink(symlink_bar_path)) self.assertTrue(os.path.exists(symlink_bar_path)) # Reverse the ordering and make sure order is preserved in the output path with open(input_path, 'w') as inpath: inpath.write(os.pathsep.join([bar_path, foo_path])) IvyUtils.symlink_cachepath(mock_cache_dir, input_path, symlink_dir, output_path, result_map) with open(output_path, 'r') as outpath: self.assertEquals(symlink_bar_path + os.pathsep + symlink_foo_path, outpath.readline())
def test_force_override(self): jars = list(self.a.payload.jars) with temporary_file_path() as ivyxml: init_subsystem(JarDependencyManagement) IvyUtils.generate_ivy([self.a], jars=jars, excludes=[], ivyxml=ivyxml, confs=['default']) doc = ET.parse(ivyxml).getroot() conf = self.find_single(doc, 'configurations/conf') self.assert_attributes(conf, name='default') dependencies = list(doc.findall('dependencies/dependency')) self.assertEqual(2, len(dependencies)) dep1 = dependencies[0] self.assert_attributes(dep1, org='org1', name='name1', rev='rev1') conf = self.find_single(dep1, 'conf') self.assert_attributes(conf, name='default', mapped='default') dep2 = dependencies[1] self.assert_attributes(dep2, org='org2', name='name2', rev='rev2', force='true') conf = self.find_single(dep1, 'conf') self.assert_attributes(conf, name='default', mapped='default') override = self.find_single(doc, 'dependencies/override') self.assert_attributes(override, org='org2', module='name2', rev='rev2')
def test_force_override(self): jars = list(self.a.payload.jars) with temporary_file_path() as ivyxml: IvyUtils.generate_ivy([self.a], jars=jars, excludes=[], ivyxml=ivyxml, confs=["default"]) doc = ET.parse(ivyxml).getroot() conf = self.find_single(doc, "configurations/conf") self.assert_attributes(conf, name="default") dependencies = list(doc.findall("dependencies/dependency")) self.assertEqual(2, len(dependencies)) dep1 = dependencies[0] self.assert_attributes(dep1, org="org1", name="name1", rev="rev1") conf = self.find_single(dep1, "conf") self.assert_attributes(conf, name="default", mapped="default") dep2 = dependencies[1] self.assert_attributes(dep2, org="org2", name="name2", rev="rev2", force="true") conf = self.find_single(dep1, "conf") self.assert_attributes(conf, name="default", mapped="default") override = self.find_single(doc, "dependencies/override") self.assert_attributes(override, org="org2", module="name2", rev="rev2")
def _prepare_ivy_xml(self, targets, ivyxml, hash_name): jars, global_excludes = IvyUtils.calculate_classpath(targets) # Don't pass global excludes to ivy when using soft excludes. if self.soft_excludes: global_excludes = [] IvyUtils.generate_ivy(targets, jars, global_excludes | self.global_excludes, ivyxml, self.confs, hash_name, self.pinned_artifacts)
def test_missing_ivy_report(self): self.set_options_for_scope(IvySubsystem.options_scope, cache_dir='DOES_NOT_EXIST', use_nailgun=False) # Hack to initialize Ivy subsystem self.context() with self.assertRaises(IvyUtils.IvyResolveReportError): IvyUtils.parse_xml_report('INVALID_REPORT_UNIQUE_NAME', 'default')
def _exec_ivy(self, ivy, executor, confs, ivyxml, args, workunit_name): try: IvyUtils.exec_ivy(ivy, confs, ivyxml, args, jvm_options=self.get_options().jvm_options, executor=executor, workunit_name=workunit_name, workunit_factory=self.context.new_workunit) except IvyUtils.IvyError as e: raise self.Error('Ivy resolve failed: {}'.format(e))
def test_fetch_applies_mutable(self): with temporary_dir() as temp_dir: ivyxml = os.path.join(temp_dir, "ivy.xml") IvyUtils.generate_fetch_ivy( [JarDependency("org-f", "name-f", "rev-f", mutable=True)], ivyxml, ("default",), "some-name" ) with open(ivyxml) as f: self.assertIn('changing="true"', f.read())
def test_missing_ivy_report(self): self.set_options_for_scope(IvySubsystem.options_scope, cache_dir='DOES_NOT_EXIST', execution_strategy=NailgunTask.ExecutionStrategy.subprocess) with self.assertRaises(IvyUtils.IvyResolveReportError): IvyUtils.parse_xml_report('default', IvyUtils.xml_report_path('INVALID_CACHE_DIR', 'INVALID_REPORT_UNIQUE_NAME', 'default'))
def test_missing_ivy_report(self): self.set_options_for_scope(IvySubsystem.options_scope, cache_dir='DOES_NOT_EXIST', use_nailgun=False) with self.assertRaises(IvyUtils.IvyResolveReportError): IvyUtils.parse_xml_report('default', IvyUtils.xml_report_path('INVALID_CACHE_DIR', 'INVALID_REPORT_UNIQUE_NAME', 'default'))
def test_fetch_ivy_xml_requests_url_for_dependency_containing_url(self): with temporary_dir() as temp_dir: ivyxml = os.path.join(temp_dir, "ivy.xml") IvyUtils.generate_fetch_ivy( [JarDependency("org-f", "name-f", "rev-f", url="an-url")], ivyxml, ("default",), "some-name" ) with open(ivyxml) as f: self.assertIn("an-url", f.read())
def test_resolve_conflict_conflict(self): v1_force = JarDependency('org.example', 'foo', '1', force=True) v2_force = JarDependency('org.example', 'foo', '2', force=True) with self.assertRaises(IvyUtils.IvyResolveConflictingDepsError): IvyUtils._resolve_conflict(v1_force, v2_force) with self.assertRaises(IvyUtils.IvyResolveConflictingDepsError): IvyUtils._resolve_conflict(v2_force, v1_force)
def test_missing_ivy_report(self): self.set_options_for_scope(IvySubsystem.options_scope, cache_dir='DOES_NOT_EXIST', use_nailgun=False) # Hack to initialize Ivy subsystem self.context() with self.assertRaises(IvyUtils.IvyResolveReportError): IvyUtils.parse_xml_report('INVALID_CACHE_DIR', 'INVALID_REPORT_UNIQUE_NAME', 'default')
def test_fetch_applies_mutable(self): with temporary_dir() as temp_dir: ivyxml = os.path.join(temp_dir, 'ivy.xml') IvyUtils.generate_fetch_ivy([JarDependency('org-f', 'name-f', 'rev-f', mutable=True)], ivyxml, ('default',), 'some-name') with open(ivyxml) as f: self.assertIn('changing="true"', f.read())
def test_missing_ivy_report(self): self.set_options_for_scope(IvySubsystem.options_scope, cache_dir='DOES_NOT_EXIST') with self.assertRaises(IvyUtils.IvyResolveReportError): IvyUtils.parse_xml_report( 'default', IvyUtils.xml_report_path('INVALID_CACHE_DIR', 'INVALID_REPORT_UNIQUE_NAME', 'default'))
def test_fetch_applies_mutable(self): with temporary_dir() as temp_dir: ivyxml = os.path.join(temp_dir, 'ivy.xml') IvyUtils.generate_fetch_ivy([JarDependency('org-f', 'name-f', 'rev-f', mutable=True)], ivyxml, ('default',), 'some-name') with open(ivyxml, 'r') as f: self.assertIn('changing="true"', f.read())
def test_fetch_requests_classifiers(self): with temporary_dir() as temp_dir: ivyxml = os.path.join(temp_dir, 'ivy.xml') IvyUtils.generate_fetch_ivy([JarDependency('org-f', 'name-f', 'rev-f', classifier='a-classifier')], ivyxml, ('default',), 'some-name') with open(ivyxml, 'r') as f: self.assertIn('a-classifier', f.read())
def test_fetch_ivy_xml_requests_url_for_dependency_containing_url(self): with temporary_dir() as temp_dir: ivyxml = os.path.join(temp_dir, 'ivy.xml') IvyUtils.generate_fetch_ivy([JarDependency('org-f', 'name-f', 'rev-f', url='an-url')], ivyxml, ('default',), 'some-name') with open(ivyxml, 'r') as f: self.assertIn('an-url', f.read())
def test_fetch_requests_classifiers(self): with temporary_dir() as temp_dir: ivyxml = os.path.join(temp_dir, 'ivy.xml') IvyUtils.generate_fetch_ivy([JarDependency('org-f', 'name-f', 'rev-f', classifier='a-classifier')], ivyxml, ('default',), 'some-name') with open(ivyxml) as f: self.assertIn('a-classifier', f.read())
def test_fetch_ivy_xml_requests_url_for_dependency_containing_url(self): with temporary_dir() as temp_dir: ivyxml = os.path.join(temp_dir, 'ivy.xml') IvyUtils.generate_fetch_ivy([JarDependency('org-f', 'name-f', 'rev-f', url='an-url')], ivyxml, ('default',), 'some-name') with open(ivyxml) as f: self.assertIn('an-url', f.read())
def test_missing_ivy_report(self): self.set_options_for_scope(IvySubsystem.options_scope, cache_dir="DOES_NOT_EXIST", use_nailgun=False) # Hack to initialize Ivy subsystem self.context() with self.assertRaises(IvyUtils.IvyResolveReportError): IvyUtils.parse_xml_report( "default", IvyUtils.xml_report_path("INVALID_CACHE_DIR", "INVALID_REPORT_UNIQUE_NAME", "default") )
def test_symlink_cachepath(self): self.maxDiff = None with temporary_dir() as mock_cache_dir: with temporary_dir() as symlink_dir: with temporary_dir() as classpath_dir: input_path = os.path.join(classpath_dir, 'inpath') output_path = os.path.join(classpath_dir, 'classpath') foo_path = os.path.join(mock_cache_dir, 'foo.jar') with open(foo_path, 'w') as foo: foo.write("test jar contents") with open(input_path, 'w') as inpath: inpath.write(foo_path) result_map = IvyUtils.symlink_cachepath(mock_cache_dir, input_path, symlink_dir, output_path) symlink_foo_path = os.path.join(symlink_dir, 'foo.jar') self.assertEquals( { os.path.realpath(foo_path): symlink_foo_path }, result_map) with open(output_path, 'r') as outpath: self.assertEquals(symlink_foo_path, outpath.readline()) self.assertTrue(os.path.islink(symlink_foo_path)) self.assertTrue(os.path.exists(symlink_foo_path)) # Now add an additional path to the existing map bar_path = os.path.join(mock_cache_dir, 'bar.jar') with open(bar_path, 'w') as bar: bar.write("test jar contents2") with open(input_path, 'w') as inpath: inpath.write(os.pathsep.join([foo_path, bar_path])) result_map = IvyUtils.symlink_cachepath(mock_cache_dir, input_path, symlink_dir, output_path) symlink_bar_path = os.path.join(symlink_dir, 'bar.jar') self.assertEquals( { os.path.realpath(foo_path): symlink_foo_path, os.path.realpath(bar_path): symlink_bar_path, }, result_map) with open(output_path, 'r') as outpath: self.assertEquals(symlink_foo_path + os.pathsep + symlink_bar_path, outpath.readline()) self.assertTrue(os.path.islink(symlink_foo_path)) self.assertTrue(os.path.exists(symlink_foo_path)) self.assertTrue(os.path.islink(symlink_bar_path)) self.assertTrue(os.path.exists(symlink_bar_path)) # Reverse the ordering and make sure order is preserved in the output path with open(input_path, 'w') as inpath: inpath.write(os.pathsep.join([bar_path, foo_path])) IvyUtils.symlink_cachepath(mock_cache_dir, input_path, symlink_dir, output_path) with open(output_path, 'r') as outpath: self.assertEquals(symlink_bar_path + os.pathsep + symlink_foo_path, outpath.readline())
def __init__(self, *args, **kwargs): super(Provides, self).__init__(*args, **kwargs) self.ivy_utils = IvyUtils(config=self.context.config, options=self.context.options, log=self.context.log) self.confs = self.context.config.getlist('ivy', 'confs', default=['default']) self.target_roots = self.context.target_roots self.transitive = self.context.options.provides_transitive self.also_write_to_stdout = self.context.options.provides_also_write_to_stdout or False
def test_fetch_applies_mutable(self): with temporary_dir() as temp_dir: ivyxml = os.path.join(temp_dir, "ivy.xml") IvyUtils.generate_fetch_ivy( [JarDependency("org-f", "name-f", "rev-f", mutable=True)], ivyxml, ("default",), "some-name", ) with open(ivyxml, "r") as f: self.assertIn('changing="true"', f.read())
def test_fetch_requests_classifiers(self): with temporary_dir() as temp_dir: ivyxml = os.path.join(temp_dir, "ivy.xml") IvyUtils.generate_fetch_ivy( [JarDependency("org-f", "name-f", "rev-f", classifier="a-classifier")], ivyxml, ("default",), "some-name", ) with open(ivyxml, "r") as f: self.assertIn("a-classifier", f.read())
def test_fetch_ivy_xml_requests_url_for_dependency_containing_url(self): with temporary_dir() as temp_dir: ivyxml = os.path.join(temp_dir, "ivy.xml") IvyUtils.generate_fetch_ivy( [JarDependency("org-f", "name-f", "rev-f", url="an-url")], ivyxml, ("default",), "some-name", ) with open(ivyxml, "r") as f: self.assertIn("an-url", f.read())
def test_fetch_requests_classifiers(self): with temporary_dir() as temp_dir: ivyxml = os.path.join(temp_dir, "ivy.xml") IvyUtils.generate_fetch_ivy( [JarDependency("org-f", "name-f", "rev-f", classifier="a-classifier")], ivyxml, ("default",), "some-name", ) with open(ivyxml) as f: self.assertIn("a-classifier", f.read())
def test_construct_and_load_symlink_map(self): self.maxDiff = None with temporary_dir() as mock_cache_dir: with temporary_dir() as symlink_dir: with temporary_dir() as classpath_dir: input_path = os.path.join(classpath_dir, "inpath") output_path = os.path.join(classpath_dir, "classpath") foo_path = os.path.join(mock_cache_dir, "foo.jar") with open(foo_path, "w") as foo: foo.write("test jar contents") with open(input_path, "w") as inpath: inpath.write(foo_path) result_classpath, result_map = IvyUtils.construct_and_load_symlink_map( symlink_dir, mock_cache_dir, input_path, output_path ) symlink_foo_path = os.path.join(symlink_dir, "foo.jar") self.assertEquals([symlink_foo_path], result_classpath) self.assertEquals({os.path.realpath(foo_path): symlink_foo_path}, result_map) with open(output_path, "r") as outpath: self.assertEquals(symlink_foo_path, outpath.readline()) self.assertTrue(os.path.islink(symlink_foo_path)) self.assertTrue(os.path.exists(symlink_foo_path)) # Now add an additional path to the existing map bar_path = os.path.join(mock_cache_dir, "bar.jar") with open(bar_path, "w") as bar: bar.write("test jar contents2") with open(input_path, "w") as inpath: inpath.write(os.pathsep.join([foo_path, bar_path])) result_classpath, result_map = IvyUtils.construct_and_load_symlink_map( symlink_dir, mock_cache_dir, input_path, output_path ) symlink_bar_path = os.path.join(symlink_dir, "bar.jar") self.assertEquals( {os.path.realpath(foo_path): symlink_foo_path, os.path.realpath(bar_path): symlink_bar_path}, result_map, ) self.assertEquals([symlink_foo_path, symlink_bar_path], result_classpath) with open(output_path, "r") as outpath: self.assertEquals(symlink_foo_path + os.pathsep + symlink_bar_path, outpath.readline()) self.assertTrue(os.path.islink(symlink_foo_path)) self.assertTrue(os.path.exists(symlink_foo_path)) self.assertTrue(os.path.islink(symlink_bar_path)) self.assertTrue(os.path.exists(symlink_bar_path)) # Reverse the ordering and make sure order is preserved in the output path with open(input_path, "w") as inpath: inpath.write(os.pathsep.join([bar_path, foo_path])) IvyUtils.construct_and_load_symlink_map(symlink_dir, mock_cache_dir, input_path, output_path) with open(output_path, "r") as outpath: self.assertEquals(symlink_bar_path + os.pathsep + symlink_foo_path, outpath.readline())
def _generate_ivy_report(self, targets): def make_empty_report(report, organisation, module, conf): no_deps_xml_template = """ <?xml version="1.0" encoding="UTF-8"?> <?xml-stylesheet type="text/xsl" href="ivy-report.xsl"?> <ivy-report version="1.0"> <info organisation="%(organisation)s" module="%(module)s" revision="latest.integration" conf="%(conf)s" confs="%(conf)s" date="%(timestamp)s"/> </ivy-report> """ no_deps_xml = no_deps_xml_template % dict(organisation=organisation, module=module, conf=conf, timestamp=time.strftime('%Y%m%d%H%M%S')) with open(report, 'w') as report_handle: print(no_deps_xml, file=report_handle) classpath = self.tool_classpath(self._ivy_bootstrap_key, self.create_java_executor()) reports = [] org, name = IvyUtils.identify(targets) xsl = os.path.join(self._cachedir, 'ivy-report.xsl') # Xalan needs this dir to exist - ensure that, but do no more - we have no clue where this # points. safe_mkdir(self._outdir, clean=False) for conf in self._confs: params = dict(org=org, name=name, conf=conf) xml = IvyUtils.xml_report_path(targets, conf) if not os.path.exists(xml): make_empty_report(xml, org, name, conf) out = os.path.join(self._outdir, '%(org)s-%(name)s-%(conf)s.html' % params) args = ['-IN', xml, '-XSL', xsl, '-OUT', out] if 0 != self.runjava(classpath=classpath, main='org.apache.xalan.xslt.Process', args=args, workunit_name='report'): raise TaskError reports.append(out) css = os.path.join(self._outdir, 'ivy-report.css') if os.path.exists(css): os.unlink(css) shutil.copy(os.path.join(self._cachedir, 'ivy-report.css'), self._outdir) if self._open: binary_util.ui_open(*reports)
def _exec_ivy(self, target_workdir, targets, args, executor=None, confs=None, ivy=None, workunit_name='ivy', use_soft_excludes=False, resolve_hash_name=None): ivy_jvm_options = self.get_options().jvm_options[:] # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append('-Dsun.io.useCanonCaches=false') ivy = ivy or Bootstrapper.default_ivy() ivyxml = os.path.join(target_workdir, 'ivy.xml') ivy_args = ['-ivy', ivyxml] confs_to_resolve = confs or ('default', ) ivy_args.append('-confs') ivy_args.extend(confs_to_resolve) ivy_args.extend(args) # TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better # diagnostics can be had in `IvyUtils.generate_ivy` if this is done. # See: https://github.com/pantsbuild/pants/issues/2239 try: jars, excludes = IvyUtils.calculate_classpath( targets, gather_excludes=not use_soft_excludes) with IvyUtils.ivy_lock: IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name) runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: result = execute_runner( runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name) if result != 0: raise self.Error( 'Ivy returned {result}. cmd={cmd}'.format( result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise self.Error(e) except IvyUtils.IvyError as e: raise self.Error('Failed to prepare ivy resolve: {}'.format(e))
def execute(self): deprecated_conditional( lambda: True, removal_version="1.31.0.dev0", entity_description="The `outdated` goal", hint_message= "Contact the Pants team on Slack or [email protected] " "if you need this functionality.", ) targets = self.context.targets() jars, global_excludes = IvyUtils.calculate_classpath(targets) filtered_jars = [ jar for jar in jars if self._is_update_coordinate(jar.coordinate) ] sorted_jars = sorted((jar for jar in filtered_jars), key=lambda x: (x.org, x.name, x.rev, x.classifier)) ivyxml = os.path.join(self.workdir, "ivy.xml") IvyUtils.generate_ivy(targets, jars=sorted_jars, excludes=global_excludes, ivyxml=ivyxml, confs=["default"]) args = [ "-settings", IvySubsystem.global_instance().get_options().ivy_settings, "-ivy", ivyxml, "-confs", ",".join(self.get_options().confs), ] result = self.runjava( classpath=self.tool_classpath("dependency-update-checker"), main=self._IVY_DEPENDENCY_UPDATE_MAIN, jvm_options=self.get_options().jvm_options, args=args, workunit_name="dependency-update-checker", workunit_labels=[WorkUnitLabel.LINT], ) self.context.log.debug( "java {main} ... exited with result ({result})".format( main=self._IVY_DEPENDENCY_UPDATE_MAIN, result=result)) return result
def test_resolve_ivy_xml_requests_classifiers(self): with temporary_dir() as temp_dir: ivyxml = os.path.join(temp_dir, 'ivy.xml') jar_dep = JarDependency('org-f', 'name-f', 'rev-f', classifier='a-classifier') IvyUtils.generate_ivy( [self.make_target('something', JarLibrary, jars=[jar_dep])], [jar_dep], excludes=[], ivyxml=ivyxml, confs=('default',), resolve_hash_name='some-name', jar_dep_manager=namedtuple('stub_jar_dep_manager', ['resolve_version_conflict'])(lambda x: x)) with open(ivyxml, 'r') as f: self.assertIn('classifier="a-classifier', f.read())
def test_resolve_ivy_xml_requests_classifiers(self): with temporary_dir() as temp_dir: ivyxml = os.path.join(temp_dir, 'ivy.xml') jar_dep = JarDependency('org-f', 'name-f', 'rev-f', classifier='a-classifier') IvyUtils.generate_ivy( [self.make_target('something', JarLibrary, jars=[jar_dep])], [jar_dep], excludes=[], ivyxml=ivyxml, confs=('default',), resolve_hash_name='some-name', jar_dep_manager=namedtuple('stub_jar_dep_manager', ['resolve_version_conflict'])(lambda x: x)) with open(ivyxml) as f: self.assertIn('classifier="a-classifier', f.read())
def test_resolve_ivy_xml_requests_classifiers(self): with temporary_dir() as temp_dir: ivyxml = os.path.join(temp_dir, "ivy.xml") jar_dep = JarDependency("org-f", "name-f", "rev-f", classifier="a-classifier") IvyUtils.generate_ivy( [self.make_target("something", JarLibrary, jars=[jar_dep])], [jar_dep], excludes=[], ivyxml=ivyxml, confs=("default",), resolve_hash_name="some-name", jar_dep_manager=namedtuple("stub_jar_dep_manager", ["resolve_version_conflict"])(lambda x: x), ) with open(ivyxml) as f: self.assertIn('classifier="a-classifier', f.read())
def exec_ivy(self, target_workdir, targets, args, executor=None, confs=None, ivy=None, workunit_name='ivy', jars=None, use_soft_excludes=False, resolve_hash_name=None): ivy_jvm_options = copy.copy(self.get_options().jvm_options) # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append('-Dsun.io.useCanonCaches=false') ivy = ivy or Bootstrapper.default_ivy() ivyxml = os.path.join(target_workdir, 'ivy.xml') if not jars: jars, excludes = IvyUtils.calculate_classpath( targets, gather_excludes=not use_soft_excludes) else: excludes = set() ivy_args = ['-ivy', ivyxml] confs_to_resolve = confs or ['default'] ivy_args.append('-confs') ivy_args.extend(confs_to_resolve) ivy_args.extend(args) with IvyUtils.ivy_lock: IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name) runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: result = execute_runner( runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name) if result != 0: raise TaskError('Ivy returned {result}. cmd={cmd}'.format( result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise TaskError(e)
def _generate_ivy_jar_products(self, resolve_hash_name): """Based on the ivy report, compute a map of conf to lists of IvyInfo objects.""" ivy_products = defaultdict(list) for conf in self.confs: ivyinfo = IvyUtils.parse_xml_report(resolve_hash_name, conf) if ivyinfo: # TODO(stuhood): Value is a list, previously to accommodate multiple exclusives groups. ivy_products[conf].append(ivyinfo) return ivy_products
def test_resolve_ivy_xml_requests_classifiers(self): with temporary_dir() as temp_dir: ivyxml = os.path.join(temp_dir, "ivy.xml") jar_dep = JarDependency("org-f", "name-f", "rev-f", classifier="a-classifier") IvyUtils.generate_ivy( [self.make_target("something", JarLibrary, jars=[jar_dep])], [jar_dep], excludes=[], ivyxml=ivyxml, confs=("default",), resolve_hash_name="some-name", jar_dep_manager=namedtuple("stub_jar_dep_manager", ["resolve_version_conflict"])( lambda x: x ), ) with open(ivyxml, "r") as f: self.assertIn('classifier="a-classifier', f.read())
def _populate_ivy_jar_products(self, targets): """Populate the build products with an IvyInfo object for each generated ivy report.""" ivy_products = self.context.products.get_data('ivy_jar_products') or defaultdict(list) for conf in self._confs: ivyinfo = IvyUtils.parse_xml_report(targets, conf) if ivyinfo: # Value is a list, to accommodate multiple exclusives groups. ivy_products[conf].append(ivyinfo) self.context.products.safe_create_data('ivy_jar_products', lambda: ivy_products)
def _generate_ivy_jar_products(self, targets): """Based on the ivy report, compute a map of conf to lists of IvyInfo objects.""" ivy_products = defaultdict(list) for conf in self.confs: ivyinfo = IvyUtils.parse_xml_report(targets, conf) if ivyinfo: # TODO(stuhood): Value is a list, previously to accommodate multiple exclusives groups. ivy_products[conf].append(ivyinfo) return ivy_products
def setUp(self): super(IvyUtilsGenerateIvyTest, self).setUp() self.add_to_build_file( 'src/java/targets', dedent(''' jar_library( name='simple', jars=[ jar('org1', 'name1', 'rev1'), jar('org2', 'name2', 'rev2', force=True), ] ) ''')) self.simple = self.target('src/java/targets:simple') self.ivy_utils = IvyUtils(create_config(), self.create_options(), logging.Logger('test'))
def setUp(self): super(IvyUtilsGenerateIvyTest, self).setUp() self.add_to_build_file( 'src/java/targets', dedent(""" jar_library( name='simple', jars=[ jar('org1', 'name1', 'rev1'), jar('org2', 'name2', 'rev2', force=True), ] ) """)) self.simple = self.target('src/java/targets:simple') context = self.context() self.ivy_utils = IvyUtils(context.config, logging.Logger('test'))
def __init__(self, context, workdir): super(Provides, self).__init__(context, workdir) self.ivy_utils = IvyUtils(config=context.config, options=context.options, log=context.log) self.confs = context.config.getlist('ivy', 'confs', default=['default']) self.target_roots = context.target_roots self.transitive = context.options.provides_transitive self.also_write_to_stdout = context.options.provides_also_write_to_stdout or False
def __init__(self, *args, **kwargs): super(Provides, self).__init__(*args, **kwargs) self.ivy_utils = IvyUtils(config=self.context.config, options=self.context.options, log=self.context.log) self.confs = self.context.config.getlist('ivy', 'confs', default=['default']) self.target_roots = self.context.target_roots self.transitive = self.get_options().transitive self.also_write_to_stdout = self.get_options().also_write_to_stdout
def _exec_ivy( self, target_workdir, targets, args, executor=None, confs=None, ivy=None, workunit_name="ivy", use_soft_excludes=False, resolve_hash_name=None, ): ivy_jvm_options = self.get_options().jvm_options[:] # Disable cache in File.getCanonicalPath(), makes Ivy work with -symlink option properly on ng. ivy_jvm_options.append("-Dsun.io.useCanonCaches=false") ivy = ivy or Bootstrapper.default_ivy() ivyxml = os.path.join(target_workdir, "ivy.xml") ivy_args = ["-ivy", ivyxml] confs_to_resolve = confs or ("default",) ivy_args.append("-confs") ivy_args.extend(confs_to_resolve) ivy_args.extend(args) # TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better # diagnostics can be had in `IvyUtils.generate_ivy` if this is done. # See: https://github.com/pantsbuild/pants/issues/2239 try: jars, excludes = IvyUtils.calculate_classpath(targets, gather_excludes=not use_soft_excludes) with IvyUtils.ivy_lock: IvyUtils.generate_ivy(targets, jars, excludes, ivyxml, confs_to_resolve, resolve_hash_name) runner = ivy.runner(jvm_options=ivy_jvm_options, args=ivy_args, executor=executor) try: result = execute_runner( runner, workunit_factory=self.context.new_workunit, workunit_name=workunit_name ) if result != 0: raise self.Error("Ivy returned {result}. cmd={cmd}".format(result=result, cmd=runner.cmd)) except runner.executor.Error as e: raise self.Error(e) except IvyUtils.IvyError as e: raise self.Error("Failed to prepare ivy resolve: {}".format(e))
def __init__(self, *args, **kwargs): super(IvyResolve, self).__init__(*args, **kwargs) self._ivy_bootstrapper = Bootstrapper.instance() self._cachedir = self._ivy_bootstrapper.ivy_cache_dir self._confs = self.context.config.getlist(self._CONFIG_SECTION, 'confs', default=['default']) self._classpath_dir = os.path.join(self.workdir, 'mapped') self._outdir = self.get_options().outdir or os.path.join( self.workdir, 'reports') self._open = self.get_options().open self._report = self._open or self.get_options().report self._ivy_utils = IvyUtils(config=self.context.config, log=self.context.log) # Typically this should be a local cache only, since classpaths aren't portable. self.setup_artifact_cache()
def _populate_ivy_jar_products(self, targets): """Populate the build products with an IvyInfo object for each generated ivy report.""" ivy_products = self.context.products.get_data( 'ivy_jar_products') or defaultdict(list) for conf in self._confs: ivyinfo = IvyUtils.parse_xml_report(targets, conf) if ivyinfo: # TODO(stuhood): Value is a list, previously to accommodate multiple exclusives groups. ivy_products[conf].append(ivyinfo) self.context.products.safe_create_data('ivy_jar_products', lambda: ivy_products)
def execute(self): targets = self.context.targets() jars, global_excludes = IvyUtils.calculate_classpath(targets) filtered_jars = [ jar for jar in jars if self._is_update_coordinate(jar.coordinate) ] sorted_jars = sorted((jar for jar in filtered_jars), key=lambda x: (x.org, x.name, x.rev, x.classifier)) ivyxml = os.path.join(self.workdir, "ivy.xml") IvyUtils.generate_ivy(targets, jars=sorted_jars, excludes=global_excludes, ivyxml=ivyxml, confs=["default"]) args = [ "-settings", IvySubsystem.global_instance().get_options().ivy_settings, "-ivy", ivyxml, "-confs", ",".join(self.get_options().confs), ] result = self.runjava( classpath=self.tool_classpath("dependency-update-checker"), main=self._IVY_DEPENDENCY_UPDATE_MAIN, jvm_options=self.get_options().jvm_options, args=args, workunit_name="dependency-update-checker", workunit_labels=[WorkUnitLabel.LINT], ) self.context.log.debug( "java {main} ... exited with result ({result})".format( main=self._IVY_DEPENDENCY_UPDATE_MAIN, result=result)) return result
def _collect_existing_reports(self, confs, resolve_hash_name): report_missing = False report_paths = [] for conf in confs: report_path = IvyUtils.xml_report_path(self.ivy_cache_dir, resolve_hash_name, conf) if not os.path.exists(report_path): report_missing = True break else: report_paths.append(report_path) return report_missing, report_paths
def _exec_ivy(self, target_workdir, targets, args, confs, executor=None, ivy=None, workunit_name='ivy', use_soft_excludes=False, resolve_hash_name=None, pinned_artifacts=None): # TODO(John Sirois): merge the code below into IvyUtils or up here; either way, better # diagnostics can be had in `IvyUtils.generate_ivy` if this is done. # See: https://github.com/pantsbuild/pants/issues/2239 jars, global_excludes = IvyUtils.calculate_classpath(targets) # Don't pass global excludes to ivy when using soft excludes. if use_soft_excludes: global_excludes = [] with IvyUtils.ivy_lock: ivyxml = os.path.join(target_workdir, 'ivy.xml') try: IvyUtils.generate_ivy(targets, jars, global_excludes, ivyxml, confs, resolve_hash_name, pinned_artifacts) except IvyUtils.IvyError as e: raise self.Error('Failed to prepare ivy resolve: {}'.format(e)) try: IvyUtils.exec_ivy(ivy, confs, ivyxml, args, jvm_options=self.get_options().jvm_options, executor=executor, workunit_name=workunit_name, workunit_factory=self.context.new_workunit) except IvyUtils.IvyError as e: raise self.Error('Ivy resolve failed: {}'.format(e))
def test_force_override(self): jars = list(self.a.payload.jars) with temporary_file_path() as ivyxml: with subsystem_instance(JarDependencyManagement): IvyUtils.generate_ivy([self.a], jars=jars, excludes=[], ivyxml=ivyxml, confs=['default']) doc = ET.parse(ivyxml).getroot() conf = self.find_single(doc, 'configurations/conf') self.assert_attributes(conf, name='default') dependencies = list(doc.findall('dependencies/dependency')) self.assertEqual(2, len(dependencies)) dep1 = dependencies[0] self.assert_attributes(dep1, org='org1', name='name1', rev='rev1') conf = self.find_single(dep1, 'conf') self.assert_attributes(conf, name='default', mapped='default') dep2 = dependencies[1] self.assert_attributes(dep2, org='org2', name='name2', rev='rev2', force='true') conf = self.find_single(dep1, 'conf') self.assert_attributes(conf, name='default', mapped='default') override = self.find_single(doc, 'dependencies/override') self.assert_attributes(override, org='org2', module='name2', rev='rev2')
def test_force_override(self): jars = list(self.a.payload.jars) with temporary_file_path() as ivyxml: init_subsystem(JarDependencyManagement) IvyUtils.generate_ivy([self.a], jars=jars, excludes=[], ivyxml=ivyxml, confs=["default"]) doc = ET.parse(ivyxml).getroot() conf = self.find_single(doc, "configurations/conf") self.assert_attributes(conf, name="default") dependencies = list(doc.findall("dependencies/dependency")) self.assertEqual(2, len(dependencies)) dep1 = dependencies[0] self.assert_attributes(dep1, org="org1", name="name1", rev="rev1") conf = self.find_single(dep1, "conf") self.assert_attributes(conf, name="default", mapped="default") dep2 = dependencies[1] self.assert_attributes(dep2, org="org2", name="name2", rev="rev2", force="true") conf = self.find_single(dep1, "conf") self.assert_attributes(conf, name="default", mapped="default") override = self.find_single(doc, "dependencies/override") self.assert_attributes(override, org="org2", module="name2", rev="rev2")
def test_resove_conflict_no_conflicts(self): v1 = JarDependency('org.example', 'foo', '1', force=False) v1_force = JarDependency('org.example', 'foo', '1', force=True) v2 = JarDependency('org.example', 'foo', '2', force=False) # If neither version is forced, use the latest version. self.assertIs(v2, IvyUtils._resolve_conflict(v1, v2)) self.assertIs(v2, IvyUtils._resolve_conflict(v2, v1)) # If an earlier version is forced, use the forced version. self.assertIs(v1_force, IvyUtils._resolve_conflict(v1_force, v2)) self.assertIs(v1_force, IvyUtils._resolve_conflict(v2, v1_force)) # If the same version is forced, use the forced version. self.assertIs(v1_force, IvyUtils._resolve_conflict(v1, v1_force)) self.assertIs(v1_force, IvyUtils._resolve_conflict(v1_force, v1)) # If the same force is in play in multiple locations, allow it. self.assertIs(v1_force, IvyUtils._resolve_conflict(v1_force, v1_force))
def execute(self): safe_mkdir(self.workdir) targets = self.context.targets() for conf in self.confs: outpath = os.path.join( self.workdir, '{0}.{1}.provides'.format(IvyUtils.identify(targets)[1], conf)) if self.transitive: outpath += '.transitive' ivy_jar_products = self.context.products.get_data( 'ivy_jar_products') or {} # This product is a list for historical reasons (exclusives groups) but in practice should # have either 0 or 1 entries. ivy_info_list = ivy_jar_products.get(conf) if ivy_info_list: assert len(ivy_info_list) == 1, ( 'The values in ivy_jar_products should always be length 1,' ' since we no longer have exclusives groups.') ivy_info = ivy_info_list[0] else: ivy_info = None jar_paths = OrderedSet() for root in self.target_roots: jar_paths.update(self.get_jar_paths(ivy_info, root, conf)) with open(outpath, 'w') as outfile: def do_write(s): outfile.write(s) if self.also_write_to_stdout: sys.stdout.write(s) for jar in jar_paths: do_write('# from jar %s\n' % jar) for line in self.list_jar(jar): if line.endswith('.class'): class_name = line[:-6].replace('/', '.') do_write(class_name) do_write('\n') self.context.log.info('Wrote provides information to %s' % outpath)
def test_resove_conflict(self): v1 = Mock() v1.force = False v1.rev = "1" v1_force = Mock() v1_force.force = True v1_force.rev = "1" v2 = Mock() v2.force = False v2.rev = "2" # If neither version is forced, use the latest version self.assertIs(v2, IvyUtils._resolve_conflict(v1, v2)) self.assertIs(v2, IvyUtils._resolve_conflict(v2, v1)) # If an earlier version is forced, use the forced version self.assertIs(v1_force, IvyUtils._resolve_conflict(v1_force, v2)) self.assertIs(v1_force, IvyUtils._resolve_conflict(v2, v1_force)) # If the same version is forced, use the forced version self.assertIs(v1_force, IvyUtils._resolve_conflict(v1, v1_force)) self.assertIs(v1_force, IvyUtils._resolve_conflict(v1_force, v1))
def parse_ivy_report(self, rel_path): path = os.path.join('tests/python/pants_test/backend/jvm/tasks', rel_path) ivy_info = IvyUtils._parse_xml_report(conf='default', path=path) self.assertIsNotNone(ivy_info) return ivy_info
def test_resolve_conflict_missing_versions(self): v1 = JarDependency('org.example', 'foo', None, force=False) v2 = JarDependency('org.example', 'foo', '2', force=False) self.assertIs(v2, IvyUtils._resolve_conflict(v1, v2)) self.assertIs(v2, IvyUtils._resolve_conflict(v2, v1))